forked from chiki2bum2/SniperGold_ML
92 lines
No EOL
3.4 KiB
Python
92 lines
No EOL
3.4 KiB
Python
"""Canonical serialization and content-id computation (spec 8.5, 12.4).
|
|
|
|
The canonical serialization of a tick is the single line
|
|
``ts_ms|bid_u|ask_u|vol\\n``. The chunk content id is SHA-256 over the
|
|
concatenation of these lines in rec_ord order. Bar content ids use the
|
|
fixed bar serialization from spec 12.4.
|
|
|
|
Physical Parquet bytes are never part of the reproducibility contract;
|
|
logical serialization is fixed by this module.
|
|
"""
|
|
|
|
from .util import sha256_bytes
|
|
|
|
_TICK_SER_FMT = "%d|%d|%d|%d\n"
|
|
# 14 canonical bar fields per spec 12.4 (start_ms/end_ms/spread_avg_u are not
|
|
# part of the canonical bar serialization).
|
|
_BAR_FIELDS14 = (0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16)
|
|
_BAR_SER_FMT = "%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d\n"
|
|
|
|
|
|
def serialize_tick(ts_ms, bid_u, ask_u, vol):
|
|
"""Canonical tick line bytes (no spaces)."""
|
|
return (_TICK_SER_FMT % (ts_ms, bid_u, ask_u, vol)).encode("utf-8")
|
|
|
|
|
|
def serialize_bars(row):
|
|
"""Canonical bar line bytes per spec 12.4. ``row`` is a full 17-field
|
|
CBS_V1 row; the 14 canonical fields are selected in fixed order and
|
|
is_final is emitted as 1/0."""
|
|
sel = [row[i] for i in _BAR_FIELDS14]
|
|
bar_idx, period_id, open_u2, high_u2, low_u2, close_u2, tick_count, \
|
|
vol_sum, spread_min_u, spread_max_u, spread_sum_u, is_final, \
|
|
first_src_line, last_src_line = sel
|
|
return (_BAR_SER_FMT % (bar_idx, period_id, open_u2, high_u2, low_u2,
|
|
close_u2, tick_count, vol_sum, spread_min_u,
|
|
spread_max_u, spread_sum_u, 1 if is_final else 0,
|
|
first_src_line, last_src_line)).encode("utf-8")
|
|
|
|
|
|
def tick_chunk_content_id(records):
|
|
"""SHA-256 over canonical serializations of ticks in rec_ord order.
|
|
|
|
``records`` is an iterable of (ts_ms, bid_u, ask_u, vol) tuples already
|
|
ordered by rec_ord.
|
|
"""
|
|
m = []
|
|
for rec in records:
|
|
m.append(serialize_tick(*rec))
|
|
return sha256_bytes(b"".join(m))
|
|
|
|
|
|
def bar_file_content_id(rows):
|
|
"""SHA-256 over canonical bar serializations in ascending bar_idx."""
|
|
m = []
|
|
for row in rows:
|
|
m.append(serialize_bars(row))
|
|
return sha256_bytes(b"".join(m))
|
|
|
|
|
|
def content_id_of_bar_row(row):
|
|
return sha256_bytes(serialize_bars(row))
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source-preservation layer (P3-DATA-ENGINE-004, spec 8.6)
|
|
#
|
|
# The G_TICKSTORY_MT5 source carries a fourth price column ``last`` that is
|
|
# deliberately EXCLUDED from CTS_V1 (schema integrity) and preserved instead
|
|
# in the per-chunk source-preservation sidecar. The canonical serialization
|
|
# of one preservation record is the single line ``src_line|last_u\\n`` in
|
|
# rec_ord order; the preservation content id is SHA-256 over the
|
|
# concatenation. last_u is integer micro-units (PRICE_SCALE), matching the
|
|
# canonical price math (no floats).
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def serialize_preservation(src_line, last_u):
|
|
"""Source-preservation line bytes: ``src_line|last_u\\n``."""
|
|
return ("%d|%d\n" % (src_line, last_u)).encode("utf-8")
|
|
|
|
|
|
def preservation_chunk_content_id(records):
|
|
"""SHA-256 over preservation serializations in rec_ord order.
|
|
|
|
``records`` is an iterable of (src_line, last_u) tuples ordered by
|
|
rec_ord.
|
|
"""
|
|
m = []
|
|
for src_line, last_u in records:
|
|
m.append(serialize_preservation(src_line, last_u))
|
|
return sha256_bytes(b"".join(m))
|
|
|