SniperGold_ML/ml/p3/p3_common.py

# -*- coding: utf-8 -*-
"""P3 COMMON — shared infrastructure for SniperGold_ML P3 forensic diagnostics.

Reuses the frozen P2.6 pipeline (train_model.py + build_features_p2.py + the
cached corrected feature chunks features_p26_chunk*.npz). No model development.

Responsibilities:
  * load M15 + HTF data (Files\\AlgoForge\\Data, 2017+ window, same as P2.6)
  * load the corrected 19-feature matrix F from the P2.6 chunk cache
  * recompute ATR series, base label (24-bar, 0.75 x ATR), and arbitrary
    horizon/threshold label variants
  * label decomposition: forward return, MFE, MAE, time-to-hit, TP-before-SL,
    SL-before-TP (computed without look-ahead relative to decision bar)
  * purged temporal split (identical to train_p26 / train_model)
  * parse the frozen production model artifact SniperGold_ML_p26_corrected.mqh
    so diagnostics can use the SAME frozen MLP without retraining
  * provenance record: hashes (dataset, feature contract, model, scripts)

P3 discipline: OBSERVE -> HYPOTHESIS -> TEST -> RECORD -> CLASSIFY.
Nothing here selects features, tunes thresholds, or modifies the MLP baseline.
"""
import os
import re
import sys
import glob
import json
import hashlib
import datetime as dt

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
SRC_TM = os.path.normpath(os.path.join(HERE, "..", "..", "..", "SniperGold_ML"))
if SRC_TM not in sys.path:
    sys.path.insert(0, SRC_TM)
import train_model as TM

DATA = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..",
                                     "Files", "AlgoForge", "Data"))
P26_CACHE = os.path.join(DATA, "features_p26_chunk*.npz")
P26_MODEL = r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Include\SniperGold_ML_p26_corrected.mqh"
CONTRACT_PATH = os.path.normpath(os.path.join(
    HERE, "..", "..", "publish", "AlgoForge", "docs", "FEATURE_CONTRACT.md"))

FEAT_NAMES = TM.FEAT_NAMES            # 19 names, contract order
NF = TM.NF                            # 19

# Feature families for the diagnostic group ablation (section 17).
# Diagnostic grouping ONLY — not a production contract change.
FEAT_GROUPS = {
    "A_HTF":            [0, 1, 2],
    "B_Context":        [6, 14, 15, 17],
    "C_Entry":          [7, 8, 9, 10, 11],
    "D_PriceAction":    [12, 13, 16],
    "E_Structural":     [3, 4, 5],
    "F_Confluence":     [18],
}

# ---- provenance constants (P2 / P2.6 checkpoint) --------------------------
P2_COMMIT = "8d330343af688e2de2b2a1d12ce19a582714709a"          # Forge HEAD (P2.6)
P26_TAG = "v20260821_p26"
CONTRACT_HASH_RECORDED = "C44CC6F2B740C32D06F776BD7C3E669DC5A8A6DE0484230544EBFFCF517D38DD"


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for blk in iter(lambda: f.read(1 << 20), b""):
            h.update(blk)
    return h.hexdigest()


def sha256_bytes(b):
    return hashlib.sha256(b).hexdigest()


def load_npz(name):
    z = np.load(os.path.join(DATA, name + ".npz"))
    return (z["time"].astype(np.int64), z["open"].astype(np.float64),
            z["high"].astype(np.float64), z["low"].astype(np.float64),
            z["close"].astype(np.float64), z["tick_volume"].astype(np.float64))


def load_data():
    """M15 bars 2017+ + HTF dict (same window as P2.6)."""
    t, o, h, l, c, v = load_npz("XAUUSD_M15")
    keep = t >= int(dt.datetime(2017, 1, 1, tzinfo=dt.timezone.utc).timestamp())
    t, o, h, l, c, v = t[keep], o[keep], h[keep], l[keep], c[keep], v[keep]
    htf = {}
    for key in ("D1", "H4", "H1"):
        ht_, ho_, hh_, hl_, hc_, hv_ = load_npz("XAUUSD_" + key)
        htf[key] = (hh_, hl_, hc_, ht_)
    return t, o, h, l, c, v, htf


def load_F():
    """Load the corrected 19-feature matrix from the P2.6 chunk cache."""
    parts = []
    for p in sorted(glob.glob(P26_CACHE)):
        z = np.load(p)
        parts.append(z["F"])
    F = np.concatenate(parts, axis=0).astype(np.float64)
    if F.shape[1] != NF:
        raise RuntimeError(f"F cache has {F.shape[1]} cols, expected {NF}")
    return F


def atr_series(h, l, c):
    return np.maximum(TM.atr_series(h, l, c), 1e-9)


def make_label(c, A, horizon=TM.H_LABEL, thr_mult=TM.LABEL_ATR):
    """Directional label over `horizon` bars with thr = thr_mult x ATR.
    Returns int array: +1 / -1 / 0 (no move). Identical to P2.6 contract."""
    n = len(c)
    lab = np.zeros(n, dtype=int)
    for i in range(n - horizon):
        fwd = c[i + horizon] - c[i]
        thr = thr_mult * A[i]
        if fwd >= thr:
            lab[i] = 1
        elif fwd <= -thr:
            lab[i] = -1
    return lab


def label_decompose(c, h, l, A, idxs, horizon=TM.H_LABEL, thr_mult=TM.LABEL_ATR):
    """Path decomposition for the label at decision bars `idxs` (no look-ahead).

    For direction d = sign of eventual label (with thr = thr_mult*A[i]):
      fwd        : c[i+H] - c[i]                     (eventual move)
      mfe        : max favorable excursion over (i, i+H] in ATR units * d
      mae        : max adverse excursion in ATR units * d
      time_to_hit: first bar j in (i, i+H] where favorable excursion >= thr
                   (NaN if never)
      tp_before_sl: 1 if favorable excursion >= thr before adverse excursion
                   crosses thr in the opposite direction (within H)
      sl_before_tp: 1 if adverse excursion <= -thr before favorable >= thr
      no_hit      : 1 if neither TP nor SL reached within H
    All excursions measured from decision close c[i] (bar i closed).
    Returns dict of arrays aligned with idxs.
    """
    n = len(c)
    m = len(idxs)
    fwd = np.full(m, np.nan)
    mfe = np.full(m, np.nan)       # in ATR units, signed by direction
    mae = np.full(m, np.nan)
    t_hit = np.full(m, np.nan)
    tp_before_sl = np.zeros(m, dtype=int)
    sl_before_tp = np.zeros(m, dtype=int)
    no_hit = np.zeros(m, dtype=int)

    for k, i in enumerate(idxs):
        if i + horizon >= n:
            continue
        thr = thr_mult * A[i]
        seg_h = h[i + 1:i + 1 + horizon]
        seg_l = l[i + 1:i + 1 + horizon]
        seg_c = c[i + 1:i + 1 + horizon]
        fwd[k] = c[i + horizon] - c[i]
        d = 1 if (fwd[k] >= thr) else (-1 if fwd[k] <= -thr else 0)
        if d == 0:
            fe = np.maximum(seg_h - c[i], c[i] - seg_l)
            ae = np.maximum(c[i] - seg_l, seg_h - c[i])
            mfe[k] = float(np.max(fe)) / A[i] if len(fe) else np.nan
            mae[k] = float(np.max(ae)) / A[i] if len(ae) else np.nan
            no_hit[k] = 1
            continue
        fe = np.where(d == 1, seg_h - c[i], c[i] - seg_l)   # favorable path
        ae = np.where(d == 1, c[i] - seg_l, seg_h - c[i])   # adverse path
        mfe[k] = float(np.max(fe)) / A[i] if len(fe) else np.nan
        mae[k] = float(np.max(ae)) / A[i] if len(ae) else np.nan
        hit_tp = np.where(fe >= thr)[0]
        hit_sl = np.where(ae >= thr)[0]
        if len(hit_tp) and (not len(hit_sl) or hit_tp[0] < hit_sl[0]):
            tp_before_sl[k] = 1
            t_hit[k] = hit_tp[0] + 1
        elif len(hit_sl) and (not len(hit_tp) or hit_sl[0] < hit_tp[0]):
            sl_before_tp[k] = 1
            t_hit[k] = hit_sl[0] + 1
        else:
            no_hit[k] = 1
    return dict(fwd=fwd, mfe=mfe, mae=mae, t_hit=t_hit,
                tp_before_sl=tp_before_sl, sl_before_tp=sl_before_tp,
                no_hit=no_hit)


def purged_split_idx(midx, split_frac=0.75, gap=TM.H_LABEL):
    split = int(split_frac * len(midx))
    tr_idx, te_idx, pg_idx, split_bar = TM.purged_split(midx, split, gap)
    ok, tr_end, te_start = TM.purge_verify(tr_idx, te_idx, gap)
    if not ok:
        raise RuntimeError(f"purge gap rusak: tr_end={tr_end} te_start={te_start}")
    return tr_idx, te_idx, pg_idx, split_bar


def dataset_hash(F, lab, A, t):
    dh = hashlib.sha256()
    dh.update(F.tobytes())
    dh.update(lab.tobytes())
    dh.update(A.tobytes())
    dh.update(t.tobytes())
    return dh.hexdigest()


def parse_mqh_arrays(path=P26_MODEL):
    """Parse SGML_* arrays from the exported frozen model .mqh.
    Returns dict with mean, std, W1, b1, W2L, b2L, W2S, b2S."""
    txt = open(path, encoding="utf-8").read()

    def arr1(name):
        mm = re.search(re.escape(name) + r"\[[^\]]*\]=\{(.*?)\};", txt, re.S)
        if not mm:
            raise RuntimeError(f"array {name} not found")
        return np.array([float(x) for x in re.findall(
            r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?", mm.group(1))])

    def arr2(name, nrow, ncol):
        # non-greedy: stop at the FIRST `};` that closes the array
        mm = re.search(re.escape(name) + r"\[[^\]]*\]\[[^\]]*\]=\{(.*?)\};", txt, re.S)
        if not mm:
            raise RuntimeError(f"2d array {name} not found")
        rows = re.findall(r"\{([^{}]*)\}", mm.group(1))
        if len(rows) != nrow:
            raise RuntimeError(f"{name}: found {len(rows)} rows, expected {nrow}")
        out = np.zeros((nrow, ncol))
        for ri, r in enumerate(rows):
            vals = [float(x) for x in re.findall(
                r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?", r)]
            if len(vals) != ncol:
                raise RuntimeError(f"{name} row {ri}: {len(vals)} vals, expected {ncol}")
            out[ri] = vals
        return out

    mean = arr1("SGML_MEAN")
    std = arr1("SGML_STD")
    W1 = arr2("SGML_W1", NF, TM.HIDDEN)
    b1 = arr1("SGML_B1")
    W2L = arr1("SGML_W2L")
    b2L = float(re.search(r"SGML_B2L=([-+0-9.eE]+)", txt).group(1))
    W2S = arr1("SGML_W2S")
    b2S = float(re.search(r"SGML_B2S=([-+0-9.eE]+)", txt).group(1))
    return dict(mean=mean, std=std, W1=W1, b1=b1,
                W2L=W2L, b2L=b2L, W2S=W2S, b2S=b2S)


def frozen_forward(M, X):
    """Frozen P2.6 MLP forward on (n,19) rows. X must be raw (not normalized).
    Returns (p_long, p_short)."""
    mean, std = M["mean"], M["std"]
    sd = np.where(std < 1e-9, 1.0, std)
    Z = (X - mean) / sd
    H1 = np.maximum(0.0, Z @ M["W1"] + M["b1"])
    oL = H1 @ M["W2L"] + M["b2L"]
    oS = H1 @ M["W2S"] + M["b2S"]
    return 1.0 / (1.0 + np.exp(-oL)), 1.0 / (1.0 + np.exp(-oS))


def provenance():
    """P3 provenance record (section 19)."""
    p = {
        "P2_COMMIT_SHA": P2_COMMIT,
        "P2.6_SOURCE_SHA": P2_COMMIT,                 # P2.6 artifacts uncommitted; scripts hashed below
        "FEATURE_CONTRACT_SHA_recorded": CONTRACT_HASH_RECORDED,
        "FEATURE_CONTRACT_SHA_file_now": sha256_file(CONTRACT_PATH) if os.path.exists(CONTRACT_PATH) else None,
        "MODEL_SHA": sha256_file(P26_MODEL),
        "train_p26_script_sha": sha256_file(os.path.join(os.path.dirname(HERE), "parity", "train_p26.py")),
        "build_features_p2_script_sha": sha256_file(os.path.join(os.path.dirname(HERE), "parity", "build_features_p2.py")),
        "symbol": "XAUUSD", "timeframe": "M15",
        "window_start": "2017-01-01",
    }
    return p


def save_json(name, obj):
    outdir = os.path.join(HERE, "output")
    os.makedirs(outdir, exist_ok=True)
    path = os.path.join(outdir, name)
    with open(path, "w", encoding="utf-8") as f:
        json.dump(obj, f, indent=2, default=float)
    print(f"  [saved] {path}")
    return path


if __name__ == "__main__":
    t, o, h, l, c, v, htf = load_data()
    F = load_F()
    A = atr_series(h, l, c)
    lab = make_label(c, A)
    print(f"bars={len(c)} F={F.shape} labeled={int((lab != 0).sum())} "
          f"bull={int((lab == 1).sum())} bear={int((lab == -1).sum())}")
    dh = dataset_hash(F, lab, A, t)
    print(f"DATASET_HASH={dh}")
    print(f"(P2.6 recorded prefix e85a0861... match={dh.startswith('e85a0861')})")
    M = parse_mqh_arrays()
    print("frozen model parsed:", {k: (v.shape if hasattr(v, "shape") else v)
                                   for k, v in M.items()})
    print(json.dumps(provenance(), indent=2))