Warrior_EA/Tests/convert_sample_data.py

r"""Runs the SQX bar decoder (research/sqxbars.py, which wraps research/sqx.py) on a COPY
of one SQX history file, so the operator has real decoded sample data to point a manual
backtest/tester run at without touching the SQX install itself.

WHY A COPY, AND WHY THIS SCRIPT DOESN'T EDIT sqxbars.py:
Tests\sample_data\SQX_History\SP500_the5ers\SP500_the5ers_H1.dat is a copy of
SQX_144_2953_win_20260601\user\data\History\SP500_the5ers\SP500_the5ers_H1.dat - the
original was never opened for writing, only read by `cp`. sqxbars.py hardcodes its own
HIST/CACHE module constants (both point at locations OUTSIDE this repo, one of them inside
the live SQX install), so rather than edit those constants in place or call sqxbars.load()
(which reads through HIST/CACHE), this script calls sqxbars.decode() directly with an
explicit path built from the COPY - bypassing HIST/CACHE entirely, so nothing in sqxbars.py
needs to change and the production research scripts (sqx_audit.py, sqx_portfolio.py,
breadth_seasonal.py, ...) are untouched and keep reading the real install.

WHY THIS DOES NOT PRODUCE SCALED PRICES:
sqxbars.load() requires either an explicit `decimals=` or a `ref=(times, closes)` validated
reference series to CALIBRATE the decimal scale - see sqxbars.calibrate()'s own docstring
and sqx.py's calibrate_decimals(): the scale is not recoverable from the file alone, and
guessing it is exactly the "silent wrong scale" trap this project's own notes warn about
twice already (project_sqx_tick_format.md, project_sqx_bar_decoder.md). This script has no
validated reference series available (fills.Book() pulls from this machine's own live
fill-engine data, out of scope for a self-contained sample), so it calls sqxbars.decode()
directly - the raw, un-scaled int64 decode - which proves the parser reads the copied file
correctly (record count, monotonic time, structural sanity) without fabricating a price
scale. The saved .npz keeps the raw integer columns; see the printed note for how the
operator can calibrate them for real use.

Usage: .venv\Scripts\python.exe Tests\convert_sample_data.py
"""
import os
import sys
import datetime as dt
import numpy as np

REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SAMPLE_HIST = os.path.join(REPO_ROOT, "Tests", "sample_data", "SQX_History") + os.sep
SAMPLE_OUT = os.path.join(REPO_ROOT, "Tests", "sample_data", "converted") + os.sep

sys.path.insert(0, os.path.join(REPO_ROOT, "research"))
import sqxbars  # noqa: E402  (path must be extended first)

SYMBOL = "SP500_the5ers"
TIMEFRAME = "H1"


def main():
    os.makedirs(SAMPLE_OUT, exist_ok=True)
    src = os.path.join(SAMPLE_HIST, SYMBOL, f"{SYMBOL}_{TIMEFRAME}.dat")
    if not os.path.exists(src):
        raise FileNotFoundError(
            f"{src} - expected the copied sample .dat here; see this file's docstring.")

    print(f"=== SQX SAMPLE CONVERSION: {SYMBOL} {TIMEFRAME} ===")
    print(f"source (a COPY, not the SQX install): {src}")
    rows, nf = sqxbars.decode(src)
    print(f"decoded {len(rows):,} bar records, {nf} fields per record")
    if len(rows) == 0:
        raise ValueError("decoder produced zero records - the copy may be truncated or corrupt")

    t_ms = rows[:, 0]
    monotonic = bool((np.diff(t_ms) >= 0).all())
    t0 = dt.datetime.fromtimestamp(t_ms[0] / 1000, dt.UTC)
    t1 = dt.datetime.fromtimestamp(t_ms[-1] / 1000, dt.UTC)
    print(f"time span: {t0:%Y-%m-%d %H:%M} .. {t1:%Y-%m-%d %H:%M} UTC")
    print(f"time monotonic: {monotonic}")

    #--- Structural sanity independent of any price scale, per sqxbars.load()'s own check:
    #--- a high below its low means the FIELD ORDER is wrong, which no rescaling could reveal.
    raw_open, raw_high, raw_low, raw_close = rows[:, 1], rows[:, 2], rows[:, 3], rows[:, 4]
    bad_hilo = int((raw_high < raw_low).sum())
    print(f"raw high<low violations: {bad_hilo} of {len(rows):,} "
          f"({'OK' if bad_hilo == 0 else 'SUSPECT FIELD ORDER'})")
    print(f"raw close range: [{raw_close.min():,} .. {raw_close.max():,}] "
          f"(UNSCALED integer units - see this file's docstring for why)")

    out_path = os.path.join(SAMPLE_OUT, f"{SYMBOL}_{TIMEFRAME}_raw.npz")
    np.savez_compressed(out_path, rows=rows,
                         columns=np.array(sqxbars.COLS))
    print(f"\nsaved raw decoded sample -> {out_path}")
    print("NOTE: prices in this file are RAW UNSCALED INTEGERS, not calibrated to a decimal")
    print("point. To get real prices, calibrate against a trusted reference series the way")
    print("sqxbars.calibrate()/load() do - e.g. sqxbars.load(sym, tf, ref=(times_ms, closes))")
    print("with a validated MT5 rate export for the same instrument/timeframe.")
    print(f"\n{SYMBOL} {TIMEFRAME}: {len(rows):,} bars decoded, monotonic={monotonic}, "
          f"high<low violations={bad_hilo} -> " +
          ("PASS" if monotonic and bad_hilo == 0 else "FAIL"))


if __name__ == "__main__":
    main()