forked from chiki2bum2/SniperGold_ML
Chiki Bum
0d5ca3c197
Implements the frozen P3_DATA_ENGINE_V1_SPEC (SHA 84bf0f217ffba51197112a6bbacbcc297058e04b5ca47f0459028fe33e0321e5). Components: engine/ producer (certify, chunkmap, parse, canonical, worker, dispatcher, aggregate, storage, journal, checkpoint, lock, evidence, manifest, run_complete, dataset_builder, cli) + engine/verify independent verifier (vparse, vaggregate, vinvariants, vcompare); headless CLI sniper-data; golden corpus G01-G17; unit/property/adversarial/mutation/legacy-diff/CLI suites. Qualification verdict: QUALIFIED (all 9 mandatory gates pass; independent verifier accepted). Spec, governance record, and legacy checkpoint untouched. G-14 NOT AUTHORIZED honored; no real-data processing, no pilot, no workload 46, no chunk 760 access.
65 lines
No EOL
2.7 KiB
Python
65 lines
No EOL
2.7 KiB
Python
"""Research dataset builder determinism + isolation tests (spec 15).
|
|
|
|
Verifies: build reads the canonical bar layer only; two builds with the same
|
|
config produce identical dataset hashes (regeneration determinism); the
|
|
manifest and outputs are produced; rows lacking future bars are excluded.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from engine.dataset_builder import build_research_dataset, load_closed_bars # noqa: E402
|
|
from tests.golden.run_golden import mini_setup, mini_run # noqa: E402
|
|
from tests.golden.common import case_bytes # noqa: E402
|
|
from engine.storage import checkpoint_path, datasets_dir # noqa: E402
|
|
from engine.checkpoint import load_checkpoint # noqa: E402
|
|
from engine.manifest import load_manifest # noqa: E402
|
|
|
|
RESEARCH = {
|
|
"dataset_id": "xau_m1_1m_dir_v1",
|
|
"dataset_version": "DS_R_V1.0.0",
|
|
"source_dataset_version": "DS_V1.0.0",
|
|
"timeframe": "M1",
|
|
"prediction_horizon_min": 1,
|
|
"target": {"type": "direction", "horizon": "1m", "from": "close"},
|
|
"features": [{"name": "ret1", "primitive": "return", "window": 1},
|
|
{"name": "range1", "primitive": "range", "window": 1}],
|
|
"split": {"policy": "time_ordered", "train": 0.7, "val": 0.15,
|
|
"test": 0.15, "oos_barrier": False, "barrier_gap_min": 1},
|
|
"label_rules": {"min_abs_move_units": 0.0},
|
|
}
|
|
|
|
|
|
def run():
|
|
with tempfile.TemporaryDirectory() as td:
|
|
data = (b"\n".join(
|
|
b"2023.01.01 00:%02d:00.000,100.0%d,100.0%d"
|
|
% (i, (i % 5), (i % 5) + 1)
|
|
for i in range(30)) + b"\n")
|
|
cfg, cert, cm, rid = mini_setup(data, td, chunk_bytes=46,
|
|
workload_bytes=5_368_709_120)
|
|
mini_run(cfg, cert, cm, td, rid)
|
|
ck = load_checkpoint(checkpoint_path(td))
|
|
canonical_manifest = {"source_identity": ck["source_identity"]}
|
|
m1 = build_research_dataset(RESEARCH, td, canonical_manifest)
|
|
m2 = build_research_dataset(RESEARCH, td, canonical_manifest)
|
|
same = m1["dataset_hash"] == m2["dataset_hash"]
|
|
mpath = os.path.join(datasets_dir(td), RESEARCH["dataset_id"],
|
|
"manifest.json")
|
|
exists = os.path.exists(mpath)
|
|
lman = load_manifest(mpath)
|
|
n_bars = len(load_closed_bars(td, "M1"))
|
|
excluded = m1["row_counts"].get("excluded_no_future", 0)
|
|
ok = same and exists and excluded >= 0 and n_bars > 0
|
|
return {"ok": ok, "dataset_hash_stable": same,
|
|
"n_bars": n_bars, "excluded_no_future": excluded}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
res = run()
|
|
print(json.dumps(res, indent=2, sort_keys=True))
|
|
sys.exit(0 if res["ok"] else 1) |