{ "gates": { "adversarial": { "items": [ { "detail": "", "name": "adversarial_mixed_terminators", "pass": true }, { "detail": "", "name": "adversarial_boundary_matrix", "pass": true }, { "detail": "8/10 rejected", "name": "adversarial_malformed_timestamps", "pass": true }, { "detail": "8/8 rejected (5. allows 0 frac digits)", "name": "adversarial_malformed_prices", "pass": true }, { "detail": "rel+nonpos=3 canonical=2", "name": "adversarial_invalid_bid_ask", "pass": true }, { "detail": "", "name": "adversarial_duplicate_data", "pass": true }, { "detail": "full=4 short=2 (verifier count check detects)", "name": "adversarial_dropped_data", "pass": true }, { "detail": "", "name": "adversarial_timestamp_reversal", "pass": true }, { "detail": "", "name": "adversarial_worker_reorder", "pass": true }, { "detail": "", "name": "adversarial_incorrect_carry", "pass": true }, { "detail": "['checkpoint_corrupt:checkpoint_hash mismatch']", "name": "adversarial_altered_prefix", "pass": true }, { "detail": "", "name": "adversarial_repeated_resume", "pass": true }, { "detail": "['already_completed']", "name": "adversarial_already_processed", "pass": true } ], "pass": true }, "cli": { "pass": true, "results": [ { "code": 4, "command": "cli_status_runless_dir", "expected": 4, "stderr_tail": "sniper-data: no run context (state/config.json missing)" }, { "code": 4, "command": "cli_resume_runless_dir", "expected": 4, "stderr_tail": "sniper-data: no run context (state/config.json missing)" }, { "code": 0, "command": "cli_version", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_certify", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_init", "expected": 0, "stderr_tail": "" }, { "code": 3, "command": "cli_ingest", "expected": 3, "stderr_tail": "" }, { "code": 3, "command": "cli_status_before_finalize", "expected": 3, "stderr_tail": "" }, { "code": 0, "command": "cli_verify_golden", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_verify_dataset", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_run_complete_exists", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_manifest", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_build_dataset", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_diff_legacy", "expected": 0, "stderr_tail": "" }, { "code": 0, "command": "cli_status_completed", "expected": 0, "stderr_tail": "" }, { "code": 4, "command": "cli_resume_blocked_on_completed", "expected": 4, "stderr_tail": "sniper-data: resume blocked: already_completed" }, { "code": 0, "command": "cli_pause", "expected": 0, "stderr_tail": "" } ], "run_complete_created": true }, "dataset_builder": { "excluded_no_future": 1, "hash_stable": true, "n_bars": 29, "pass": true }, "golden": { "cases": [ { "case": "G01", "detail": [], "ok": true }, { "case": "G02", "detail": [], "ok": true }, { "case": "G03", "detail": [], "ok": true }, { "case": "G04", "detail": [], "ok": true }, { "case": "G05", "detail": [], "ok": true }, { "case": "G06", "detail": [], "ok": true }, { "case": "G07", "detail": [], "ok": true }, { "case": "G08", "detail": [], "ok": true }, { "case": "G09", "detail": [], "ok": true }, { "case": "G10", "detail": [], "ok": true }, { "case": "G11", "detail": [], "ok": true }, { "case": "G12", "detail": [], "ok": true }, { "case": "G01_equals_G02_ids", "detail": [], "ok": true }, { "case": "G05_boundaries", "detail": [], "ok": true }, { "case": "G06_boundaries", "detail": [], "ok": true }, { "case": "G11_dropped_row", "detail": [], "ok": true }, { "case": "G13_worker_reorder", "detail": [], "ok": true }, { "case": "G14_missing_carry", "detail": [ "checkpoint_corrupt:checkpoint_hash mismatch" ], "ok": true }, { "case": "G14_incorrect_carry", "detail": [], "ok": true }, { "case": "G15_journal_corrupt", "detail": [ "journal_corrupt:journal line unparseable: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte" ], "ok": true }, { "case": "G15_checkpoint_corrupt", "detail": [ "checkpoint_corrupt:checkpoint_hash mismatch" ], "ok": true }, { "case": "G16_repeated_resume", "detail": [], "ok": true }, { "case": "G17_already_transformed", "detail": [ "format='unrecognized' init_refused=True" ], "ok": true } ], "pass": true }, "independent_verifier": { "dataset_verdict": "VERIFIER_ACCEPTED", "evidence_diffs": [], "evidence_recheck": true, "pass": true, "run_complete_verdict": "VERIFIER_ACCEPTED" }, "legacy_diff": { "deferred_status": "DEFERRED", "items_classified": 9, "legacy_untouched": true, "pass": true, "unresolved": 0 }, "mutation": { "detected": 8, "ledger": [ { "class": "flip a byte", "detail": "pid=8539fb064996 base=7cfe1c7330b0", "detected": true, "detection_method": "independent digest/counters differ", "mutation": "M01_byte_flip" }, { "class": "duplicate a row", "detail": "dup=1 rows=4", "detected": true, "detection_method": "MALFORMED_DUPLICATE counter (canonical digest unchanged by design)", "mutation": "M02_duplicate_row" }, { "class": "drop a row", "detail": "rows=3 base=4", "detected": true, "detection_method": "row-count mismatch + independent digest", "mutation": "M03_drop_row" }, { "class": "alter a timestamp", "detail": "pid=8539fb064996", "detected": true, "detection_method": "independent digest differs", "mutation": "M04_alter_timestamp" }, { "class": "altered prefix", "detail": "checkpoint_corrupt:checkpoint_hash mismatch", "detected": true, "detection_method": "resume guard checkpoint_hash", "mutation": "M05_altered_prefix" }, { "class": "incorrect carry", "detail": "FAILED", "detected": true, "detection_method": "FAILED(carry_continuity) state", "mutation": "M06_incorrect_carry" }, { "class": "repeated resume", "detail": "stable=True", "detected": true, "detection_method": "content-id stability (verifier accepts)", "mutation": "M07_repeated_resume" }, { "class": "already-processed state", "detail": "already_completed", "detected": true, "detection_method": "resume guard rejects COMPLETED", "mutation": "M08_already_processed" } ], "pass": true, "total": 8 }, "property": { "items": [ { "detail": "", "name": "deterministic_serialization", "pass": true }, { "detail": "", "name": "content_id_stability_producer_verifier", "pass": true }, { "detail": "", "name": "deterministic_aggregation", "pass": true }, { "detail": "", "name": "aggregation_producer_verifier_equal", "pass": true }, { "detail": "", "name": "no_gaps_no_overlaps", "pass": true }, { "detail": "", "name": "monotonic_source_order", "pass": true }, { "detail": "", "name": "journal_seq_monotonic", "pass": true }, { "detail": "", "name": "checkpoint_next_chunk_monotonic", "pass": true }, { "detail": "stB2='CHUNK_COMMITTED'", "name": "resume_equivalence", "pass": true }, { "detail": "mal=4 rows=4", "name": "no_silent_malformed_transformation", "pass": true }, { "detail": "['0', '131040']", "name": "no_empty_bar_fabrication", "pass": true } ], "pass": true }, "source_grammar": { "e2e": true, "pass": true, "suite_cases": 15 }, "unit": { "note": "29 unit cases (12 areas)", "pass": true } }, "generated_utc": "2026-09-07T10:33:06.186520+00:00", "governance_baseline": "P3-DATA-ENGINE-001 GOV-CLOSE-001 (DESIGN FROZEN) + P3-DE-004 CONTROLLED AMENDMENT (spec 1.1.0, Appendix C); G-14 NOT AUTHORIZED", "interventions": [ { "action": "Corrected the Hinnant civil-days formula; re-verified producer == verifier on all fixtures", "authorization_status": "authorized (implementation)", "decision_basis": "spec 8.1 requires deterministic integer epoch math; both paths must agree exactly", "finding": "days_from_civil month-offset constant used the wrong value (1/-3 instead of 9/-3)", "necessary_or_avoidable": "necessary", "observed_evidence": "parse_timestamp_dotted returned 2021 epoch for a 2023 timestamp; independent table-based verifier disagreed", "result": "Golden G01..G12 and property content-id stability PASS", "state_impact": "None outside engine/util.py", "trigger": "Producer/verifier epoch mismatch caught by dual-path tests" }, { "action": "Producer and verifier serializers now select the 14 canonical fields from full CBS rows", "authorization_status": "authorized (implementation)", "decision_basis": "spec 12.4 field list is authoritative", "finding": "spec 12.4 canonical bar serialization defines exactly 14 fields; start_ms/end_ms/spread_avg_u are not serialized", "necessary_or_avoidable": "necessary", "observed_evidence": "serialize_bars expected 14 fields, storage rows carry 17", "result": "All bar content-id checks PASS", "state_impact": "engine/canonical.py + engine/verify/vaggregate.py", "trigger": "Bar content-id raises on 17-field CBS rows" }, { "action": "Every processed workload now writes a per-TF part file (empty when it contains no finalized bars) as an aggregation-completeness marker; resume skips workloads whose parts exist", "authorization_status": "authorized (implementation)", "decision_basis": "append-only outputs (spec 4.2/11.5) and resume equivalence require exactly-once aggregation per workload", "finding": "Aggregation completeness was tracked only by bar-file presence, but zero-bar workloads wrote no file", "necessary_or_avoidable": "necessary", "observed_evidence": "Pause->resume on a workload with no finalized bars double-counted carry ticks; carry-continuity FAILED", "result": "Repeated-resume and resume-equivalence tests PASS; no bar rows were ever fabricated", "state_impact": "engine/dispatcher.py; documented in implementation map", "trigger": "Resume re-aggregates an already processed workload" }, { "action": "Standardized the line hash over the line text without the trailing newline", "authorization_status": "authorized (implementation)", "decision_basis": "spec 13.1 hash-chained journal; deterministic chain definition required", "finding": "Inconsistent line-hash domain", "necessary_or_avoidable": "necessary", "observed_evidence": "append_commit stored hash of line including trailing \\n; read_journal recomputed without it", "result": "Journal round-trip and resume checks PASS", "state_impact": "engine/journal.py", "trigger": "Journal chain broke at seq 1 during read-back" }, { "action": "Stringify all schema metadata values", "authorization_status": "authorized (implementation)", "decision_basis": "spec 12.3 schema metadata is recorded as text", "finding": "Parquet schema key-value metadata must be strings", "necessary_or_avoidable": "necessary", "observed_evidence": "int metadata values rejected by pyarrow 25", "result": "All parquet write/read paths PASS", "state_impact": "engine/storage.py", "trigger": "pyarrow replace_schema_metadata TypeError (expected bytes, int found)" }, { "action": "Checkpointed carry objects additionally store first_src_line; documented in the implementation map", "authorization_status": "authorized (implementation detail; no spec amendment)", "decision_basis": "spec 11.3/13.5 require the audit anchor and resume equivalence; checkpoint-internal field addition is the minimal deterministic resolution and changes no external contract", "finding": "first_src_line must survive resume to keep CBS_V1 semantics and resume-equivalent content ids", "necessary_or_avoidable": "necessary", "observed_evidence": "A partial bar carried across a resume would lose its first_src_line audit anchor", "result": "Resume-equivalence content ids stable", "state_impact": "engine/aggregate.py (documented)", "trigger": "Spec gap: carry field list (\u00a711.4) omits first_src_line needed by CBS_V1" }, { "action": "Added --out-root to pause/stop; reordered finalize to verify -> COMPLETED -> evidence -> RUN_COMPLETE", "authorization_status": "authorized (implementation)", "decision_basis": "spec 14 exit codes and 22 completion protocol ordering", "finding": "Parser gaps and evidence build before final checkpoint", "necessary_or_avoidable": "necessary", "observed_evidence": "pause returned usage error 2; run-complete re-verification found stale state hashes", "result": "CLI E2E 14/14 PASS with correct exit codes", "state_impact": "engine/cli.py", "trigger": "CLI pause/stop lacked --out-root; verify --mode dataset raced final checkpoint writes" }, { "action": "Amended spec Appendix C (C.1 grammar, C.2 timestamp, C.3 volume/last/versions, C.4 implementation-visible changes, C.5 exclusions); specification SHA updated to effbaf2624cd46137c22a246fdabff4067052807b6b6224be464366bec4745d9", "authorization_status": "authorized (approved amendment package)", "decision_basis": "Owner-authorized controlled amendment (P3-DE-004); Appendix C supersedes the dotted primary for the production source", "finding": "Spec 8.1 primary grammar was the dotted corpus form; the authoritative production grammar is G_TICKSTORY_MT5 (YYYYMMDD,HH:MM:SS,bid,ask,last,volume)", "necessary_or_avoidable": "necessary", "observed_evidence": "First real row '20030505,00:01:03,340.345,340.757,340.345,0'; real source has no header, CRLF, explicit volume column (zeros valid) and a 'last' price column", "result": "Spec 1.1.0 carries the corrected grammar; G-9/G-10/G-14 unchanged; engine version identity bumped (1.1.0 / PARSER_V1.1)", "state_impact": "docs/P3_DATA_ENGINE_V1_SPEC.md only", "trigger": "P3-DE-003 pilot audit: the real six-column Tickstory grammar differs from the dotted corpus grammar" }, { "action": "versions registry + version bumps; config source_grammar identity field + fail-closed validation; parse_compact_datetime; _parse_chunk_tickstory; preservation serialization + per-chunk sidecar writes; certificate grammar_id + compact sniffing; init certificate/config mismatch BLOCK; independent verifier v_parse_chunk_tickstory + preservation id checks; evidence + verify_dataset preservation content-id verification", "authorization_status": "authorized (implementation of the approved amendment)", "decision_basis": "Spec 8.1/8.2/8.6 + Appendix C.3; CTS_V1/CBS_V1 schema identity must stay frozen (G-4/G-5)", "finding": "Need a named G_TICKSTORY_MT5 preset (certificate-pinned), compact civil-math timestamp, mandatory volume semantics and a source-preservation layer", "necessary_or_avoidable": "necessary", "observed_evidence": "Dotted parser cannot recognize compact dates; 'last' has no CTS_V1 mapping (spec 8.6)", "result": "SG01-SG15 + compact E2E PASS; producer == independent verifier on all fixtures; anchor row 20030505,00:01:03 -> ts 1,052,092,863,000 ms (legacy-verified)", "state_impact": "engine/{parse,canonical,certify,config,dispatcher,evidence,storage,versions,worker}.py; engine/verify/{runner,vinvariants,vparse}.py; tests/source_grammar_tests.py (new)", "trigger": "Grammar presets, compact timestamp and the 'last' column required implementation changes" }, { "action": "Return the _err code as the first tuple element; callers return it; checkpoint-load exceptions become exit 4; regression tests cli_status_runless_dir / cli_resume_runless_dir added", "authorization_status": "authorized (implementation)", "decision_basis": "Spec 14.3 exit-code contract (4 = blocked); P3-DE-004 Appendix C.4", "finding": "_load_run_context returned the error integer in the last tuple slot; callers dereferenced a None config (NEW_ENGINE_DEFECT, non-data-affecting)", "necessary_or_avoidable": "necessary", "observed_evidence": "Expected exit code 4 (BLOCKED); actual exit code 1 + TypeError traceback", "result": "CLI E2E regression PASS (exit 4 without traceback)", "state_impact": "engine/cli.py; tests/cli_tests.py", "trigger": "P3-DE-003 independent pilot audit: CLI status/resume on a run-less directory misbehaves" }, { "action": "Restored both import lines; added gates['source_grammar'] (suite_cases + e2e); updated session/SHA/governance/versions/safety/limitations/interventions and the Markdown header", "authorization_status": "authorized (implementation; resume within P3-DE-004 scope)", "decision_basis": "Do not rewrite the driver; restore both imports, add the source_grammar gate, refresh report fields", "finding": "Partial edit; the suite must keep property/adversarial AND add source grammar", "necessary_or_avoidable": "necessary", "observed_evidence": "The previous session's import edit REPLACED 'from tests import property_tests, adversarial_tests' with 'from tests import source_grammar_tests'; property/adversarial gates would NameError and no source_grammar gate existed", "result": "Full qualification discovers and runs every suite including SG01-SG15", "state_impact": "tests/qualification_main.py", "trigger": "Resume-session inspection found a broken partial edit in the qualification driver" } ], "limitations": [ "NOT VERIFIED: byte-range/tick/hash differential vs the real 34.5 GB source (G-13 pilot scope; session forbids opening the source).", "NOT VERIFIED: full 34 GB headless pass (G-14 NOT AUTHORIZED).", "DEFERRED: G-17 HMAC hardening.", "PARTIAL: CLI 'verify --mode range/legacy' live-range modes exercised only on synthetic fixtures.", "PARTIAL: the source-preservation sidecar (spec 8.6) was validated on synthetic six-column fixtures only; no real source bytes were opened.", "source_tz_offset_minutes production value preserved at 0 from the P3-DE-003 evidence chain (spec Appendix C.2); G-6 requires explicit establishment at source initialization in the pilot phase." ], "safety": { "34gb_source_processed": "NO", "chunk_760_plus_touched": "NO", "forecasting_performed": "NO", "g14_authorization_changed": "NO (G-14 remains BLOCKED / NOT AUTHORIZED)", "governance_baseline_altered": "NO (unchanged except the approved specification amendment)", "legacy_checkpoint_modified": "NO", "legacy_pipeline_modified": "NO", "production_checkpoint_created": "NO", "real_data_pilot_executed": "NO", "real_source_ingested": "NO", "research_dataset_generated": "NO", "specification_amendment_applied": "YES (approved P3-DE-004 controlled amendment; Appendix C; SHA verified)", "workload_46_executed": "NO" }, "safety_notes": { "research_dataset_generated": "NO: no real/research dataset was produced. The dataset-builder and build-dataset qualification ran on synthetic fixtures only (permitted by session scope).", "source_grammar_correction": "The P3-DE-004 source-grammar correction (G_TICKSTORY_MT5 six-column preset) is an approved specification amendment, not a legacy defect. Canonical schema/algorithm semantics were not re-opened (G-4/G-5 unchanged)." }, "session": "P3-DATA-ENGINE-004", "specification_doc": "docs/P3_DATA_ENGINE_V1_SPEC.md", "specification_sha256": "effbaf2624cd46137c22a246fdabff4067052807b6b6224be464366bec4745d9", "verdict": "QUALIFIED", "versions": { "algorithm_version": "ALG_V1", "dataset_version": "DS_V1.0.0", "engine_version": "1.1.0", "grammar_registry": [ "tickstory_mt5", "dotted_v1" ], "parser_version": "PARSER_V1.1", "schema": "CTS_V1 / CBS_V1 (unchanged)", "spec_version": "1.1.0 (amended by P3-DATA-ENGINE-004; Appendix C)" } }