ARIMA_SAX_Hybrid_Forecaster/scripts/run_r2h_origins.py

# -*- coding: utf-8 -*-
"""R2-H — HORIZON-BOUNDARY EXPERIMENT: deterministic forecast-origin freeze.

PURPOSE
  Freeze a NEW reproducible set of walk-forward forecast-origin lists for the
  R2-H elapsed-prediction-horizon experiment (XAUUSDc M1 base grid). This is a
  DESIGN artifact. It runs NO forecasting model, computes NO performance metric,
  and produces NO prediction/evidence artifact.

FROZEN ORIGIN-SELECTION RULE (pre-registered, deterministic, model-independent)
  Base grid: M1 (all horizons use the M1 grid; horizon is defined in ELAPSED
  minutes, not bar count).

  For horizon H (minutes):
    1) Candidate origins: every M1 bar index o in [WARMUP, n).
    2) Endpoint: t_end = first valid M1 bar timestamp >= timestamp(o) + H.
       If no such bar exists in the dataset the origin is EXCLUDED
       (target not resolvable).
    3) Gap rule: let delta[i] be the elapsed minutes from bar i-1 to bar i.
       If any delta in (o, t_end] exceeds GAP_TOLERANCE_MINUTES the origin is
       EXCLUDED (materially missing bars / session boundaries are never
       converted into ordinary contiguous observations).
    4) Origin spacing: consecutive accepted origins are separated by at least
       SPACING_MULT x H elapsed minutes (no overlapping target windows).
    5) No human-selected origins; no model-dependent selection; no cap.

FROZEN PARAMETERS
  WARMUP                 = 2000       (M1 bars; sufficient for ARIMA fit
                                        window 300, SAX window 24, drift
                                        lookback 40, ATR 20)
  SPACING_MULT           = 2          (spacing = 2 x H elapsed minutes)
  GAP_TOLERANCE_MINUTES  = 5          (any inter-bar delta > 5 min in the
                                        forward window excludes the origin)
  FIT_WINDOW             = 300        (recorded for information only)
  ATR_PERIOD             = 20         (recorded for information only)

  All exclusions are deterministic and recorded in the summary manifest.

OUTPUTS (per horizon, LF newlines, UTF-8, no BOM)
  origins_H<HHH>.csv  columns:
    origin_index, origin_timestamp, target_end_index, target_end_timestamp,
    horizon_minutes, elapsed_minutes, training_start, training_end
  A per-run params/summary JSON is written alongside (R2_H_ORIGIN_PARAMS.json).
"""
from __future__ import annotations

import argparse
import hashlib
import json
import os
import platform
import sys
from datetime import datetime, timedelta

WARMUP = 2000
SPACING_MULT = 2
GAP_TOLERANCE_MINUTES = 5
FIT_WINDOW = 300
ATR_PERIOD = 20

HORIZONS = {
    "H015": 15,
    "H060": 60,
    "H240": 240,
}

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "results", "R1_real_data", "XAUUSDc_M1_raw.json")
DEFAULT_OUT = os.path.join(ROOT, "results", "R2-H")

CSV_COLS = [
    "origin_index", "origin_timestamp", "target_end_index",
    "target_end_timestamp", "horizon_minutes", "elapsed_minutes",
    "training_start", "training_end",
]


def sha256_file(path):
    with open(path, "rb") as fh:
        return hashlib.sha256(fh.read()).hexdigest()


def sha256_bytes(buf):
    return hashlib.sha256(buf).hexdigest()


def load_bars(path):
    with open(path, "r", encoding="utf-8") as fh:
        raw = json.load(fh)
    times = []
    closes = []
    for b in raw:
        times.append(datetime.strptime(b["time"], "%Y.%m.%d %H:%M:%S"))
        closes.append(float(b["close"]))
    n = len(times)
    for i in range(1, n):
        if not (times[i] > times[i - 1]):
            raise ValueError("non-ascending or duplicate M1 timestamp at index %d" % i)
    return times, closes


def build_deltas(times):
    """delta[i] = elapsed minutes from bar i-1 to bar i; delta[0] = 0."""
    n = len(times)
    deltas = [0.0] * n
    for i in range(1, n):
        deltas[i] = (times[i] - times[i - 1]).total_seconds() / 60.0
    return deltas


def first_index_at_or_after(times, target):
    """First index j with times[j] >= target (binary search on ascending list)."""
    lo, hi = 0, len(times)
    while lo < hi:
        mid = (lo + hi) // 2
        if times[mid] < target:
            lo = mid + 1
        else:
            hi = mid
    return lo


def build_horizon(times, deltas, H):
    """Return (accepted_origins, exclusions) for one elapsed horizon."""
    n = len(times)
    usable = []   # list of (origin_index, t_end_index, elapsed_minutes)
    exclusions = {"no_endpoint": 0, "gap": 0}
    for o in range(WARMUP, n):
        t_o = times[o]
        target = t_o + timedelta(minutes=H)
        j = first_index_at_or_after(times, target)
        if j >= n:
            exclusions["no_endpoint"] += 1
            continue
        # gap rule: max delta in (o, t_end] must be <= tolerance
        max_delta = max(deltas[o + 1:j + 1])
        if max_delta > GAP_TOLERANCE_MINUTES:
            exclusions["gap"] += 1
            continue
        elapsed = (times[j] - t_o).total_seconds() / 60.0
        usable.append((o, j, elapsed))

    # origin spacing in elapsed minutes: accept if >= last_accepted + 2*H
    accepted = []
    last_ts = None
    min_spacing = SPACING_MULT * H
    for (o, j, elapsed) in usable:
        t_o = times[o]
        if last_ts is None or (t_o - last_ts).total_seconds() / 60.0 >= min_spacing:
            accepted.append((o, j, elapsed))
            last_ts = t_o
    return accepted, exclusions


def run(out_dir):
    os.makedirs(out_dir, exist_ok=True)
    times, closes = load_bars(RAW)
    deltas = build_deltas(times)
    n = len(times)

    summary = {
        "design": "R2-H origin freeze",
        "base_grid": "M1",
        "dataset": "results/R1_real_data/XAUUSDc_M1_raw.json",
        "n_M1_bars": n,
        "warmup_M1": WARMUP,
        "spacing_mult": SPACING_MULT,
        "gap_tolerance_minutes": GAP_TOLERANCE_MINUTES,
        "fit_window": FIT_WINDOW,
        "atr_period": ATR_PERIOD,
        "horizons": {},
        "exclusions": {},
    }

    for key, H in HORIZONS.items():
        accepted, exclusions = build_horizon(times, deltas, H)
        rows = []
        for (o, j, elapsed) in accepted:
            rows.append({
                "origin_index": o,
                "origin_timestamp": times[o].strftime("%Y.%m.%d %H:%M:%S"),
                "target_end_index": j,
                "target_end_timestamp": times[j].strftime("%Y.%m.%d %H:%M:%S"),
                "horizon_minutes": H,
                "elapsed_minutes": elapsed,
                "training_start": max(0, o - FIT_WINDOW + 1),
                "training_end": o,
            })
        lines = [",".join(CSV_COLS)] + [
            ",".join(str(r[c]) for c in CSV_COLS) for r in rows
        ]
        csv_text = "\n".join(lines) + "\n"
        csv_path = os.path.join(out_dir, "origins_%s.csv" % key)
        with open(csv_path, "w", encoding="utf-8", newline="") as fh:
            fh.write(csv_text)

        summary["horizons"][key] = {
            "horizon_minutes": H,
            "n_origins": len(rows),
            "n_usable_candidates": len(accepted) + exclusions["gap"] + exclusions["no_endpoint"],
            "sha256": sha256_bytes(csv_text.encode("utf-8")),
        }
        summary["exclusions"][key] = exclusions

    summary["python"] = platform.python_version()
    summary["executable"] = sys.executable

    params_path = os.path.join(out_dir, "R2_H_ORIGIN_PARAMS.json")
    with open(params_path, "w", encoding="utf-8", newline="") as fh:
        json.dump(summary, fh, indent=2, sort_keys=True)

    print("R2-H origin generation complete.")
    for key in HORIZONS:
        h = summary["horizons"][key]
        print("  %s: origins=%d usable_candidates=%d sha256=%s..." %
              (key, h["n_origins"], h["n_usable_candidates"], h["sha256"][:16]))
    print("  params: %s" % params_path)
    return summary


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=DEFAULT_OUT)
    args = ap.parse_args()
    run(args.out)


if __name__ == "__main__":
    main()