SniperGold_ML/tests/dataset_builder_tests.py

"""Research dataset builder determinism + isolation tests (spec 15).

Verifies: build reads the canonical bar layer only; two builds with the same
config produce identical dataset hashes (regeneration determinism); the
manifest and outputs are produced; rows lacking future bars are excluded.
"""

import json
import os
import sys
import tempfile

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from engine.dataset_builder import build_research_dataset, load_closed_bars  # noqa: E402
from tests.golden.run_golden import mini_setup, mini_run  # noqa: E402
from tests.golden.common import case_bytes  # noqa: E402
from engine.storage import checkpoint_path, datasets_dir  # noqa: E402
from engine.checkpoint import load_checkpoint  # noqa: E402
from engine.manifest import load_manifest  # noqa: E402

RESEARCH = {
    "dataset_id": "xau_m1_1m_dir_v1",
    "dataset_version": "DS_R_V1.0.0",
    "source_dataset_version": "DS_V1.0.0",
    "timeframe": "M1",
    "prediction_horizon_min": 1,
    "target": {"type": "direction", "horizon": "1m", "from": "close"},
    "features": [{"name": "ret1", "primitive": "return", "window": 1},
                 {"name": "range1", "primitive": "range", "window": 1}],
    "split": {"policy": "time_ordered", "train": 0.7, "val": 0.15,
              "test": 0.15, "oos_barrier": False, "barrier_gap_min": 1},
    "label_rules": {"min_abs_move_units": 0.0},
}


def run():
    with tempfile.TemporaryDirectory() as td:
        data = (b"\n".join(
            b"2023.01.01 00:%02d:00.000,100.0%d,100.0%d"
            % (i, (i % 5), (i % 5) + 1)
            for i in range(30)) + b"\n")
        cfg, cert, cm, rid = mini_setup(data, td, chunk_bytes=46,
                                        workload_bytes=5_368_709_120)
        mini_run(cfg, cert, cm, td, rid)
        ck = load_checkpoint(checkpoint_path(td))
        canonical_manifest = {"source_identity": ck["source_identity"]}
        m1 = build_research_dataset(RESEARCH, td, canonical_manifest)
        m2 = build_research_dataset(RESEARCH, td, canonical_manifest)
        same = m1["dataset_hash"] == m2["dataset_hash"]
        mpath = os.path.join(datasets_dir(td), RESEARCH["dataset_id"],
                             "manifest.json")
        exists = os.path.exists(mpath)
        lman = load_manifest(mpath)
        n_bars = len(load_closed_bars(td, "M1"))
        excluded = m1["row_counts"].get("excluded_no_future", 0)
        ok = same and exists and excluded >= 0 and n_bars > 0
        return {"ok": ok, "dataset_hash_stable": same,
                "n_bars": n_bars, "excluded_no_future": excluded}


if __name__ == "__main__":
    res = run()
    print(json.dumps(res, indent=2, sort_keys=True))
    sys.exit(0 if res["ok"] else 1)