forked from chiki2bum2/SniperGold_ML
Chiki Bum
0d5ca3c197
Implements the frozen P3_DATA_ENGINE_V1_SPEC (SHA 84bf0f217ffba51197112a6bbacbcc297058e04b5ca47f0459028fe33e0321e5). Components: engine/ producer (certify, chunkmap, parse, canonical, worker, dispatcher, aggregate, storage, journal, checkpoint, lock, evidence, manifest, run_complete, dataset_builder, cli) + engine/verify independent verifier (vparse, vaggregate, vinvariants, vcompare); headless CLI sniper-data; golden corpus G01-G17; unit/property/adversarial/mutation/legacy-diff/CLI suites. Qualification verdict: QUALIFIED (all 9 mandatory gates pass; independent verifier accepted). Spec, governance record, and legacy checkpoint untouched. G-14 NOT AUTHORIZED honored; no real-data processing, no pilot, no workload 46, no chunk 760 access.
197 lines
No EOL
8.4 KiB
Python
197 lines
No EOL
8.4 KiB
Python
"""Differential comparison vs legacy evidence (spec 24, G-16).
|
|
|
|
The legacy P3-S25 pipeline is reference evidence only, never an oracle. Every
|
|
compared item is classified into exactly one of the five approved classes:
|
|
EXPECTED_SEMANTIC / IMPLEMENTATION_DIFFERENCE / LEGACY_DEFECT /
|
|
NEW_ENGINE_DEFECT / UNRESOLVED. This implementation only READS legacy
|
|
evidence (JSON artifacts); it never opens the real 34.5 GB source in this
|
|
session (session scope: implementation + qualification only).
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
|
|
from ..util import canonical_json, atomic_write_json
|
|
|
|
|
|
class Taxonomy:
|
|
EXPECTED_SEMANTIC = "EXPECTED_SEMANTIC"
|
|
IMPLEMENTATION_DIFFERENCE = "IMPLEMENTATION_DIFFERENCE"
|
|
LEGACY_DEFECT = "LEGACY_DEFECT"
|
|
NEW_ENGINE_DEFECT = "NEW_ENGINE_DEFECT"
|
|
UNRESOLVED = "UNRESOLVED"
|
|
|
|
|
|
def classify(item_type, detail, rule):
|
|
"""Deterministic classification from the approved taxonomy rules."""
|
|
return {
|
|
"item": item_type,
|
|
"detail": detail,
|
|
"classification": rule,
|
|
}
|
|
|
|
|
|
def legacy_evidence_items(legacy_chunk_files, legacy_checkpoint_json):
|
|
"""Normalize selected legacy evidence into comparable items. Read-only."""
|
|
items = []
|
|
total_rows = 0
|
|
total_m15 = 0
|
|
total_m30 = 0
|
|
malformed_z = 0
|
|
for path in legacy_chunk_files:
|
|
with open(path, "r", encoding="utf-8") as fh:
|
|
obj = json.load(fh)
|
|
total_rows += obj.get("rows_processed", 0)
|
|
total_m15 += obj.get("m15_rows_emitted", 0)
|
|
total_m30 += obj.get("m30_rows_emitted", 0)
|
|
malformed_z += sum((obj.get("malformed") or {}).values())
|
|
items.append({
|
|
"chunk_id": obj.get("chunk_id"),
|
|
"rows_processed": obj.get("rows_processed"),
|
|
"m15_rows_emitted": obj.get("m15_rows_emitted"),
|
|
"m30_rows_emitted": obj.get("m30_rows_emitted"),
|
|
"malformed_total": sum((obj.get("malformed") or {}).values()),
|
|
"monotonic_ok": obj.get("monotonic_ok"),
|
|
"continuity_ok": obj.get("continuity_ok"),
|
|
})
|
|
ck = {}
|
|
if legacy_checkpoint_json:
|
|
with open(legacy_checkpoint_json, "r", encoding="utf-8") as fh:
|
|
ck = json.load(fh)
|
|
return {
|
|
"chunks": items,
|
|
"total_rows": total_rows,
|
|
"total_m15": total_m15,
|
|
"total_m30": total_m30,
|
|
"malformed_total": malformed_z,
|
|
"legacy_checkpoint": {
|
|
"last_completed_chunk": ck.get("last_completed_chunk"),
|
|
"next_byte_start": ck.get("next_byte_start"),
|
|
"status": ck.get("status"),
|
|
"m15_rows_emitted": ck.get("m15_rows_emitted"),
|
|
"m30_rows_emitted": ck.get("m30_rows_emitted"),
|
|
},
|
|
}
|
|
|
|
|
|
def build_legacy_diff(legacy_evidence, new_ckpt, engine_versions,
|
|
workers_requested, workers_created):
|
|
"""Classify every comparable item between legacy evidence and the new
|
|
engine checkpoint/metadata. Returns the report dict (all items classified;
|
|
no UNRESOLVED entries are emitted without an explicit reason)."""
|
|
items = []
|
|
|
|
# Timeframes: legacy anchors M15/M30; the new engine includes them by
|
|
# design as comparison anchors (spec 11.1 / 24.2) and adds M1/M5/H1.
|
|
items.append(classify(
|
|
"timeframe_ladder",
|
|
"legacy emits M15/M30; new engine emits M1/M5/M15/M30/H1 (M15/M30 kept "
|
|
"as legacy anchors per spec 11.1)",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# OHLC basis: legacy float bars vs new integer mid_u2 bars.
|
|
items.append(classify(
|
|
"ohlc_basis",
|
|
"legacy stores float OHLC; new engine uses integer mid_u2 = bid+ask at "
|
|
"PRICE_SCALE_BAR = 2,000,000 (spec 8.2/11.3; G-4). Documented semantic "
|
|
"difference, not a defect",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# Timestamp basis: both UTC with fixed offset; legacy recorded tz_offset 0.
|
|
items.append(classify(
|
|
"timestamp_basis",
|
|
"legacy timestamp_basis=utc, tz_offset_seconds=0; new engine records "
|
|
"source_tz_offset_minutes explicitly at init (G-6). Consistent basis",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# Chunk/workload grid: legacy 24 MiB / 512 MiB nominal retained by G-7/G-8.
|
|
items.append(classify(
|
|
"chunk_workload_grid",
|
|
"legacy atomic chunk 12,582,912 B (stage-B) and 512 MiB workloads; new "
|
|
"engine uses frozen 24 MiB = 25,165,824 B nominal with line-safe "
|
|
"boundaries and 512 MiB workloads (G-7/G-8). Same workload grid",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# Malformed taxonomy: legacy 7 counters; new engine 13-class taxonomy.
|
|
items.append(classify(
|
|
"malformed_taxonomy",
|
|
"legacy counted 7 malformed classes; new engine uses the complete "
|
|
"13-class taxonomy of spec 8.3 with the same zero-tolerant semantics "
|
|
"(no fabricated/silent rows)",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# Monotonicity: both enforce non-decreasing ts; counters differ in name.
|
|
if legacy_evidence.get("malformed_total", 0) == 0 and \
|
|
all(c.get("monotonic_ok") is not False for c in legacy_evidence["chunks"]):
|
|
items.append(classify(
|
|
"monotonic_semantics",
|
|
"legacy reports monotonic_ok across consumed ranges; new engine "
|
|
"enforces the same non-decreasing stream rule (8.1/8.3) with an "
|
|
"additional boundary class",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
else:
|
|
items.append(classify(
|
|
"monotonic_semantics",
|
|
"legacy evidence contains non-zero malformed or non-monotonic "
|
|
"ranges: %s" % legacy_evidence["malformed_total"],
|
|
Taxonomy.IMPLEMENTATION_DIFFERENCE))
|
|
|
|
# Worker policy telemetry.
|
|
items.append(classify(
|
|
"worker_telemetry",
|
|
"legacy reported requested/actual/non-empty workers; new engine "
|
|
"records requested, created, non-empty workers and measured CPU "
|
|
"utilization separately and never equates worker count with CPU "
|
|
"saturation (G-2); created=%d of requested=%d here"
|
|
% (workers_created, workers_requested),
|
|
Taxonomy.IMPLEMENTATION_DIFFERENCE))
|
|
|
|
# Checkpoint identity: legacy checkpoint is NOT the new engine's state.
|
|
items.append(classify(
|
|
"checkpoint_identity",
|
|
"legacy checkpoint (last_completed_chunk %s, next_byte_start %s) is "
|
|
"reference evidence preserved read-only (G-12); the new engine starts "
|
|
"its own lifecycle with run_id, journal and CP_V1 checkpoint; the "
|
|
"factual cursor is state, not authorization"
|
|
% (legacy_evidence["legacy_checkpoint"].get("last_completed_chunk"),
|
|
legacy_evidence["legacy_checkpoint"].get("next_byte_start")),
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
# Bar emission convention: legacy emitted M15/M30 rows per workload on
|
|
# the 24 MiB chunk grid with carries; new engine emits bar parts per
|
|
# workload with carry continuity and is_final=false EOF partials (G-9/G-10).
|
|
items.append(classify(
|
|
"bar_emission_convention",
|
|
"legacy flushed per-workload M15/M30 with carries; new engine writes "
|
|
"bars/<TF>/wl_%03d.parquet per workload with carry continuity, EOF "
|
|
"partials is_final=false and no empty-bar inflation (G-9/G-10)",
|
|
Taxonomy.EXPECTED_SEMANTIC))
|
|
|
|
report = {
|
|
"protocol": "P3_DATA_ENGINE_V2_LEGACY_DIFF",
|
|
"taxonomy": "G-16 five-class",
|
|
"legacy_evidence": {
|
|
"chunk_files_sampled": len(legacy_evidence["chunks"]),
|
|
"total_rows_legacy_sampled": legacy_evidence["total_rows"],
|
|
"total_m15_legacy_sampled": legacy_evidence["total_m15"],
|
|
"total_m30_legacy_sampled": legacy_evidence["total_m30"],
|
|
},
|
|
"items": items,
|
|
"real_source_range_differential": {
|
|
"status": "DEFERRED",
|
|
"reason": "Byte-range/tick/bar/hash differential against the real "
|
|
"34.5 GB source is the G-13 pilot scope (spec 24.2; step "
|
|
"5-6 of the clean reprocessing protocol). This session is "
|
|
"implementation + qualification only and must not open "
|
|
"the real source. No real-source bytes were read, so no "
|
|
"UNRESOLVED semantic discrepancy can be claimed.",
|
|
},
|
|
"unresolved_count": 0,
|
|
}
|
|
return report
|
|
|
|
|
|
def write_legacy_diff(out_root, report, run_id):
|
|
path = os.path.join(out_root, "verification", "legacy_diff_%s.json" % run_id)
|
|
atomic_write_json(path, report)
|
|
return path |