SniperGold_ML/engine/canonical.py

"""Canonical serialization and content-id computation (spec 8.5, 12.4).

The canonical serialization of a tick is the single line
``ts_ms|bid_u|ask_u|vol\\n``. The chunk content id is SHA-256 over the
concatenation of these lines in rec_ord order. Bar content ids use the
fixed bar serialization from spec 12.4.

Physical Parquet bytes are never part of the reproducibility contract;
logical serialization is fixed by this module.
"""

from .util import sha256_bytes

_TICK_SER_FMT = "%d|%d|%d|%d\n"
# 14 canonical bar fields per spec 12.4 (start_ms/end_ms/spread_avg_u are not
# part of the canonical bar serialization).
_BAR_FIELDS14 = (0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16)
_BAR_SER_FMT = "%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d|%d\n"


def serialize_tick(ts_ms, bid_u, ask_u, vol):
    """Canonical tick line bytes (no spaces)."""
    return (_TICK_SER_FMT % (ts_ms, bid_u, ask_u, vol)).encode("utf-8")


def serialize_bars(row):
    """Canonical bar line bytes per spec 12.4. ``row`` is a full 17-field
    CBS_V1 row; the 14 canonical fields are selected in fixed order and
    is_final is emitted as 1/0."""
    sel = [row[i] for i in _BAR_FIELDS14]
    bar_idx, period_id, open_u2, high_u2, low_u2, close_u2, tick_count, \
        vol_sum, spread_min_u, spread_max_u, spread_sum_u, is_final, \
        first_src_line, last_src_line = sel
    return (_BAR_SER_FMT % (bar_idx, period_id, open_u2, high_u2, low_u2,
                            close_u2, tick_count, vol_sum, spread_min_u,
                            spread_max_u, spread_sum_u, 1 if is_final else 0,
                            first_src_line, last_src_line)).encode("utf-8")


def tick_chunk_content_id(records):
    """SHA-256 over canonical serializations of ticks in rec_ord order.

    ``records`` is an iterable of (ts_ms, bid_u, ask_u, vol) tuples already
    ordered by rec_ord.
    """
    m = []
    for rec in records:
        m.append(serialize_tick(*rec))
    return sha256_bytes(b"".join(m))


def bar_file_content_id(rows):
    """SHA-256 over canonical bar serializations in ascending bar_idx."""
    m = []
    for row in rows:
        m.append(serialize_bars(row))
    return sha256_bytes(b"".join(m))


def content_id_of_bar_row(row):
    return sha256_bytes(serialize_bars(row))
    


# ---------------------------------------------------------------------------
# Source-preservation layer (P3-DATA-ENGINE-004, spec 8.6)
#
# The G_TICKSTORY_MT5 source carries a fourth price column ``last`` that is
# deliberately EXCLUDED from CTS_V1 (schema integrity) and preserved instead
# in the per-chunk source-preservation sidecar. The canonical serialization
# of one preservation record is the single line ``src_line|last_u\\n`` in
# rec_ord order; the preservation content id is SHA-256 over the
# concatenation. last_u is integer micro-units (PRICE_SCALE), matching the
# canonical price math (no floats).
# ---------------------------------------------------------------------------

def serialize_preservation(src_line, last_u):
    """Source-preservation line bytes: ``src_line|last_u\\n``."""
    return ("%d|%d\n" % (src_line, last_u)).encode("utf-8")


def preservation_chunk_content_id(records):
    """SHA-256 over preservation serializations in rec_ord order.

    ``records`` is an iterable of (src_line, last_u) tuples ordered by
    rec_ord.
    """
    m = []
    for src_line, last_u in records:
        m.append(serialize_preservation(src_line, last_u))
    return sha256_bytes(b"".join(m))