"""Canonical serialization and content-id computation (spec 8.5, 12.4). The canonical serialization of a tick is the single line ``ts_ms|bid_u|ask_u|vol\\n``. The chunk content id is SHA-256 over the concatenation of these lines in rec_ord order. Bar content ids use the fixed bar serialization from spec 12.4. Physical Parquet bytes are never part of the reproducibility contract; logical serialization is fixed by this module. """ from .util import sha256_bytes _TICK_SER_FMT = "%d|%d|%d|%d\n" # 14 canonical bar fields per spec 12.4 (start_ms/end_ms/spread_avg_u are not # part of the canonical bar serialization). _BAR_FIELDS14 = (0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16) _BAR_SER_FMT = "%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d\n" def serialize_tick(ts_ms, bid_u, ask_u, vol): """Canonical tick line bytes (no spaces).""" return (_TICK_SER_FMT % (ts_ms, bid_u, ask_u, vol)).encode("utf-8") def serialize_bars(row): """Canonical bar line bytes per spec 12.4. ``row`` is a full 17-field CBS_V1 row; the 14 canonical fields are selected in fixed order and is_final is emitted as 1/0.""" sel = [row[i] for i in _BAR_FIELDS14] bar_idx, period_id, open_u2, high_u2, low_u2, close_u2, tick_count, \ vol_sum, spread_min_u, spread_max_u, spread_sum_u, is_final, \ first_src_line, last_src_line = sel return (_BAR_SER_FMT % (bar_idx, period_id, open_u2, high_u2, low_u2, close_u2, tick_count, vol_sum, spread_min_u, spread_max_u, spread_sum_u, 1 if is_final else 0, first_src_line, last_src_line)).encode("utf-8") def tick_chunk_content_id(records): """SHA-256 over canonical serializations of ticks in rec_ord order. ``records`` is an iterable of (ts_ms, bid_u, ask_u, vol) tuples already ordered by rec_ord. """ m = [] for rec in records: m.append(serialize_tick(*rec)) return sha256_bytes(b"".join(m)) def bar_file_content_id(rows): """SHA-256 over canonical bar serializations in ascending bar_idx.""" m = [] for row in rows: m.append(serialize_bars(row)) return sha256_bytes(b"".join(m)) def content_id_of_bar_row(row): return sha256_bytes(serialize_bars(row)) # --------------------------------------------------------------------------- # Source-preservation layer (P3-DATA-ENGINE-004, spec 8.6) # # The G_TICKSTORY_MT5 source carries a fourth price column ``last`` that is # deliberately EXCLUDED from CTS_V1 (schema integrity) and preserved instead # in the per-chunk source-preservation sidecar. The canonical serialization # of one preservation record is the single line ``src_line|last_u\\n`` in # rec_ord order; the preservation content id is SHA-256 over the # concatenation. last_u is integer micro-units (PRICE_SCALE), matching the # canonical price math (no floats). # --------------------------------------------------------------------------- def serialize_preservation(src_line, last_u): """Source-preservation line bytes: ``src_line|last_u\\n``.""" return ("%d|%d\n" % (src_line, last_u)).encode("utf-8") def preservation_chunk_content_id(records): """SHA-256 over preservation serializations in rec_ord order. ``records`` is an iterable of (src_line, last_u) tuples ordered by rec_ord. """ m = [] for src_line, last_u in records: m.append(serialize_preservation(src_line, last_u)) return sha256_bytes(b"".join(m))