ARIMA_SAX_Hybrid_Forecaster/scripts/run_r1r_origins.py

# -*- coding: utf-8 -*-
"""R1-R — REPRODUCIBLE REAL-DATA REPLICATION: deterministic forecast-origin freeze.

PURPOSE
  Freeze a NEW reproducible set of walk-forward forecast-origin lists (R1-R).
  This is NOT a reconstruction of historical R1 and does NOT run any forecasting
  model or compute any performance metric.

ORIGIN-SELECTION RULE (pre-registered, deterministic, model-independent)
    origins_cand = [ o in [train_warmup, n) : bars o..o+H are contiguous ]
    origins      = origins_cand[::stride]       stride = 8 * HORIZON

    M1  : H=15  stride=120  train_warmup=2000
    M5  : H=3   stride=24   train_warmup=2000
    M15 : H=1   stride=8    train_warmup=500

    stride is chosen a priori (8*horizon). Eligibility depends only on data
    contiguity + warmup, never on model outputs, and stride is NOT chosen to
    match the historical R1 counts (489/511/515).

OUTPUTS (results/R1_R/, git-ignored):
    origins_M1.csv  origins_M5.csv  origins_M15.csv
    R1_R_PROTOCOL.md  R1_R_PROTOCOL_MANIFEST.json
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
from datetime import datetime

import numpy as np

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "results", "R1_real_data", "XAUUSDc_M1_raw.json")
OUT = os.path.join(ROOT, "results", "R1_R")
FIT_WINDOW = 300
ATR_PERIOD = 20
TARGET_DEF = "Forward Return / ATR(20) over horizon H"

PROTOCOLS = {
    "M1":  {"H": 15, "train": 2000, "tf": 1,  "mult": 8},
    "M5":  {"H": 3,  "train": 2000, "tf": 5,  "mult": 8},
    "M15": {"H": 1,  "train": 500,  "tf": 15, "mult": 8},
}


def load_raw(path):
    with open(path, "r", encoding="utf-8") as fh:
        return json.load(fh)


def sorted_unique(bars):
    seen = {}
    for b in bars:
        seen[b["time"]] = b
    return [seen[k] for k in sorted(seen.keys())]


def aggregate(bars, k):
    out = []
    for i in range(0, len(bars), k):
        if i + k > len(bars):
            break
        block = bars[i:i + k]
        out.append({
            "time": block[-1]["time"],
            "close": float(block[-1]["close"]),
            "open": float(block[0]["open"]),
            "high": max(float(x["high"]) for x in block),
            "low": min(float(x["low"]) for x in block),
        })
    return out


def valid_orig_mask(times, H, tf_minutes):
    dtv = [datetime.strptime(t, "%Y.%m.%d %H:%M:%S") for t in times]
    deltas = np.zeros(len(times), float)
    for i in range(1, len(times)):
        deltas[i] = (dtv[i] - dtv[i - 1]).total_seconds() / 60.0
    contig = deltas[1:] == float(tf_minutes)
    n = len(contig)
    if n < H:
        return np.zeros(len(times) - H, bool)
    win = np.lib.stride_tricks.sliding_window_view(contig, H)
    return win.all(axis=1)


def sha256_bytes(buf):
    return hashlib.sha256(buf).hexdigest()


def sha256_file(path):
    with open(path, "rb") as fh:
        return hashlib.sha256(fh.read()).hexdigest()


def close_hash(closes):
    return hashlib.sha256(np.asarray(closes, float).astype("<f8").tobytes()).hexdigest()


def git_head():
    try:
        out = subprocess.run(["git", "rev-parse", "--short", "HEAD"],
                             capture_output=True, text=True, cwd=ROOT).stdout.strip()
        if not out:
            out = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True,
                                 text=True, cwd=ROOT).stdout.strip()
        return out
    except Exception:
        return ""


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=OUT)
    ap.add_argument("--code-commit", default="",
                    help="explicit code commit; if empty, uses git HEAD")
    args = ap.parse_args()

    out_dir = args.out
    os.makedirs(out_dir, exist_ok=True)

    m1 = sorted_unique(load_raw(RAW))
    m5 = aggregate(m1, 5)
    m15 = aggregate(m1, 15)
    series = {"M1": m1, "M5": m5, "M15": m15}
    cl = {"M1": [float(b["close"]) for b in m1],
          "M5": [float(b["close"]) for b in m5],
          "M15": [float(b["close"]) for b in m15]}

    code_commit = args.code_commit or git_head()

    data_hashes = {
        "M1_close_sha256": close_hash(cl["M1"]),
        "M5_close_sha256": close_hash(cl["M5"]),
        "M15_close_sha256": close_hash(cl["M15"]),
        "M1_raw_json_sha256": sha256_file(RAW),
    }
    n_bars = {"M1": len(m1), "M5": len(m5), "M15": len(m15)}

    origin_counts = {}
    origin_hashes = {}
    per_res = {}
    for tf, p in PROTOCOLS.items():
        times = [b["time"] for b in series[tf]]
        n = len(times)
        v = valid_orig_mask(times, p["H"], p["tf"])
        cand = [o for o in range(p["train"], n - p["H"]) if v[o]]
        stride = p["mult"] * p["H"]
        origins = cand[::stride]

        cols = ["origin_index", "origin_timestamp", "training_start",
                "training_end", "target_start", "target_end", "horizon"]
        rows = []
        for o in origins:
            rows.append({
                "origin_index": o,
                "origin_timestamp": times[o],
                "training_start": max(0, o - FIT_WINDOW + 1),
                "training_end": o,
                "target_start": o + 1,
                "target_end": o + p["H"],
                "horizon": p["H"],
            })
        lines = [",".join(cols)] + [",".join(str(r[c]) for c in cols) for r in rows]
        csv_text = "\n".join(lines) + "\n"

        csv_path = os.path.join(out_dir, f"origins_{tf}.csv")
        with open(csv_path, "w", encoding="utf-8", newline="") as fh:
            fh.write(csv_text)

        origin_counts[tf] = len(origins)
        origin_hashes[tf] = sha256_bytes(csv_text.encode("utf-8"))
        per_res[tf] = {
            "horizon": p["H"],
            "stride": stride,
            "eligible_count": len(cand),
            "train_warmup": p["train"],
            "origin_count": len(origins),
            "origin_list_sha256": origin_hashes[tf],
        }

    manifest = {
        "experiment": "R1-R",
        "title": "REPRODUCIBLE REAL-DATA REPLICATION (new frozen protocol)",
        "protocol_version": "r1-r-v0.1.0-frozen",
        "code_commit": code_commit,
        "dataset": {
            "symbol": "XAUUSDc",
            "source": "MT5 terminal history (results/R1_real_data/XAUUSDc_M1_raw.json)",
            "M5_M15": "index-aggregated from M1 (deterministic)",
            "n_bars": n_bars,
            "hashes": data_hashes,
        },
        "target": {
            "definition": TARGET_DEF,
            "atr_period": ATR_PERIOD,
            "horizons": {"M1": 15, "M5": 3, "M15": 1},
        },
        "origin_selection_rule": {
            "rule": "origins = origins_cand[::stride] where origins_cand = "
                    "chronologically eligible origins in [train_warmup, n) with "
                    "bars o..o+H contiguous at resolution step; stride = 8*HORIZON.",
            "model_independent": True,
            "not_historical_strides": "stride NOT chosen to reproduce R1 counts 489/511/515",
            "arima_fit_window": FIT_WINDOW,
            "per_resolution": per_res,
        },
        "actual_origin_counts": origin_counts,
        "origin_list_hashes": origin_hashes,
        "outputs": {
            "M1": "results/R1_R/origins_M1.csv",
            "M5": "results/R1_R/origins_M5.csv",
            "M15": "results/R1_R/origins_M15.csv",
            "protocol_md": "results/R1_R/R1_R_PROTOCOL.md",
            "manifest": "results/R1_R/R1_R_PROTOCOL_MANIFEST.json",
        },
        "note": "Origin-only freeze; no forecasting model executed; no performance computed.",
    }
    mpath = os.path.join(out_dir, "R1_R_PROTOCOL_MANIFEST.json")
    with open(mpath, "w", encoding="utf-8", newline="") as fh:
        json.dump(manifest, fh, indent=2)

    print("R1-R origin generation complete. code_commit =", code_commit)
    for tf in ("M1", "M5", "M15"):
        print(f"  {tf}: eligible={per_res[tf]['eligible_count']} "
              f"origins={per_res[tf]['origin_count']} stride={per_res[tf]['stride']} "
              f"hash={origin_hashes[tf][:16]}...")


if __name__ == "__main__":
    main()