ARIMA_SAX_Hybrid_Forecaster/scripts/build_r1_dataset.py

# -*- coding: utf-8 -*-
"""Build the frozen R1 real XAUUSDc M1 dataset from MT5 terminal exports.

Parses the MCP chart-history temp JSON exports (goose_mcp_response_*.txt),
merges by timestamp (dedupe), runs integrity checks, and writes:
  results/R1_real_data/XAUUSDc_M1_raw.json      (frozen raw OHLC+vol+spread)
  results/R1_real_data/XAUUSDc_M1.csv           (frozen wide CSV)
  results/R1_REAL_DATASET_MANIFEST.json         (provenance + hashes)
"""
from __future__ import annotations
import glob
import hashlib
import json
import math
import os
import datetime as dt

import numpy as np

TEMP_GLOB = r"D:\TradingTerminal\MetaTrader 5\MQL5\Files\Temp\goose_mcp_response_*.txt"
OUT_DIR = "results/R1_real_data"
OUT_RAW = os.path.join(OUT_DIR, "XAUUSDc_M1_raw.json")
OUT_CSV = os.path.join(OUT_DIR, "XAUUSDc_M1.csv")
OUT_MANIFEST = "results/R1_REAL_DATASET_MANIFEST.json"


def parse_ts(s):
    return dt.datetime.strptime(s, "%Y.%m.%d %H:%M:%S")


def main():
    os.makedirs(OUT_DIR, exist_ok=True)
    files = sorted(glob.glob(TEMP_GLOB))
    print("temp exports found:", len(files))

    rows = {}
    parse_fail = []
    for fp in files:
        try:
            with open(fp, "r", encoding="utf-8", errors="replace") as fh:
                obj = json.load(fh)
            for r in obj.get("history", []):
                rows[r["time"]] = r
        except Exception as exc:  # noqa: BLE001
            parse_fail.append((os.path.basename(fp), str(exc)))

    print("parse failures:", parse_fail)
    ts = sorted(rows.keys())
    print("unique bars:", len(ts), "| span:", ts[0], "->", ts[-1])

    # ----- integrity checks -----
    ohlc_viol = 0
    nonfinite = 0
    neg_tick = 0
    neg_spread = 0
    for t in ts:
        r = rows[t]
        o, h, l, c = (float(r["open"]), float(r["high"]),
                      float(r["low"]), float(r["close"]))
        if not math.isfinite(o) or not math.isfinite(h) or not math.isfinite(l) or not math.isfinite(c):
            nonfinite += 1
            continue
        if h < max(o, c) - 1e-9 or l > min(o, c) + 1e-9:
            ohlc_viol += 1
        tv = r.get("tick_volume")
        sp = r.get("spread")
        if tv is not None and tv < 0:
            neg_tick += 1
        if sp is not None and sp < 0:
            neg_spread += 1

    # gap estimate (minute bars)
    missing = 0
    prev = parse_ts(ts[0])
    one_min = dt.timedelta(minutes=1)
    for t in ts[1:]:
        cur = parse_ts(t)
        if cur > prev + one_min:
            missing += int((cur - prev - one_min).total_seconds() / 60)
        prev = cur

    # ----- serialize frozen dataset -----
    raw = []
    for t in ts:
        r = rows[t]
        raw.append({"time": t, "open": r["open"], "high": r["high"], "low": r["low"],
                    "close": r["close"], "tick_volume": r.get("tick_volume"),
                    "spread": r.get("spread")})
    with open(OUT_RAW, "w", encoding="utf-8") as fh:
        json.dump(raw, fh)

    # wide CSV
    with open(OUT_CSV, "w", encoding="utf-8", newline="") as fh:
        cols = ["time", "open", "high", "low", "close", "tick_volume", "spread"]
        fh.write(",".join(cols) + "\n")
        for t in ts:
            r = rows[t]
            fh.write(",".join([
                t,
                str(r["open"]), str(r["high"]), str(r["low"]), str(r["close"]),
                str(r.get("tick_volume", "")), str(r.get("spread", "")),
            ]) + "\n")

    close = np.asarray([rows[t]["close"] for t in ts], dtype=float)
    data_hash = hashlib.sha256(close.astype("<f8").tobytes()).hexdigest()
    raw_hash = hashlib.sha256(open(OUT_RAW, "rb").read()).hexdigest()

    manifest = {
        "dataset_id": "R1-XAUUSDc-M1",
        "source": "MT5 terminal chart history (broker symbol XAUUSDc, Gold Spot Cent)",
        "broker_symbol": "XAUUSDc",
        "broker_symbol_note": "exact broker naming retained; not renamed to XAUUSD",
        "timeframe": "M1",
        "timezone": "server trade time (UTC-based per terminal; see MT5 trade server time)",
        "start_timestamp": ts[0],
        "end_timestamp": ts[-1],
        "n_bars": len(ts),
        "integrity": {
            "parse_failures": parse_fail,
            "duplicate_timestamp_count": len(rows) - len(set(rows)),
            "missing_bar_estimate_minutes": missing,
            "ohlc_violations": ohlc_viol,
            "nonfinite_values": nonfinite,
            "negative_tick_volume": neg_tick,
            "negative_spread": neg_spread,
            "chronological_ordering": "ascending-by-construction",
        },
        "dataset_hash_close_only": data_hash,
        "dataset_hash_raw_json": raw_hash,
        "acquisition": "MCP get_chart_history chunks (100k bars each) merged by timestamp; "
                       "see scripts/build_r1_dataset.py",
        "columns": ["time", "open", "high", "low", "close", "tick_volume", "spread"],
        "notes": "Spread is broker-reported points (multiply by point=0.01 for price units). "
                 "Weekend/holiday gaps expected in metals market.",
    }
    with open(OUT_MANIFEST, "w", encoding="utf-8") as fh:
        json.dump(manifest, fh, indent=2)

    print("bars:", len(ts))
    print("span:", ts[0], "->", ts[-1])
    print("ohlc violations:", ohlc_viol, "| nonfinite:", nonfinite,
          "| missing est:", missing, "| neg tick:", neg_tick, "| neg spread:", neg_spread)
    print("data hash (close):", data_hash[:16])
    print("raw json hash:", raw_hash[:16])
    print("wrote:", OUT_RAW, OUT_CSV, OUT_MANIFEST)


if __name__ == "__main__":
    main()