298 lines
12 KiB
Python
298 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""P3 COMMON — shared infrastructure for SniperGold_ML P3 forensic diagnostics.
|
|
|
|
Reuses the frozen P2.6 pipeline (train_model.py + build_features_p2.py + the
|
|
cached corrected feature chunks features_p26_chunk*.npz). No model development.
|
|
|
|
Responsibilities:
|
|
* load M15 + HTF data (Files\\AlgoForge\\Data, 2017+ window, same as P2.6)
|
|
* load the corrected 19-feature matrix F from the P2.6 chunk cache
|
|
* recompute ATR series, base label (24-bar, 0.75 x ATR), and arbitrary
|
|
horizon/threshold label variants
|
|
* label decomposition: forward return, MFE, MAE, time-to-hit, TP-before-SL,
|
|
SL-before-TP (computed without look-ahead relative to decision bar)
|
|
* purged temporal split (identical to train_p26 / train_model)
|
|
* parse the frozen production model artifact SniperGold_ML_p26_corrected.mqh
|
|
so diagnostics can use the SAME frozen MLP without retraining
|
|
* provenance record: hashes (dataset, feature contract, model, scripts)
|
|
|
|
P3 discipline: OBSERVE -> HYPOTHESIS -> TEST -> RECORD -> CLASSIFY.
|
|
Nothing here selects features, tunes thresholds, or modifies the MLP baseline.
|
|
"""
|
|
import os
|
|
import re
|
|
import sys
|
|
import glob
|
|
import json
|
|
import hashlib
|
|
import datetime as dt
|
|
|
|
import numpy as np
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
SRC_TM = os.path.normpath(os.path.join(HERE, "..", "..", "..", "SniperGold_ML"))
|
|
if SRC_TM not in sys.path:
|
|
sys.path.insert(0, SRC_TM)
|
|
import train_model as TM
|
|
|
|
DATA = os.path.normpath(os.path.join(HERE, "..", "..", "..", "..",
|
|
"Files", "AlgoForge", "Data"))
|
|
P26_CACHE = os.path.join(DATA, "features_p26_chunk*.npz")
|
|
P26_MODEL = r"D:\TradingTerminal\HFM Metatrader 5\MQL5\Include\SniperGold_ML_p26_corrected.mqh"
|
|
CONTRACT_PATH = os.path.normpath(os.path.join(
|
|
HERE, "..", "..", "publish", "AlgoForge", "docs", "FEATURE_CONTRACT.md"))
|
|
|
|
FEAT_NAMES = TM.FEAT_NAMES # 19 names, contract order
|
|
NF = TM.NF # 19
|
|
|
|
# Feature families for the diagnostic group ablation (section 17).
|
|
# Diagnostic grouping ONLY — not a production contract change.
|
|
FEAT_GROUPS = {
|
|
"A_HTF": [0, 1, 2],
|
|
"B_Context": [6, 14, 15, 17],
|
|
"C_Entry": [7, 8, 9, 10, 11],
|
|
"D_PriceAction": [12, 13, 16],
|
|
"E_Structural": [3, 4, 5],
|
|
"F_Confluence": [18],
|
|
}
|
|
|
|
# ---- provenance constants (P2 / P2.6 checkpoint) --------------------------
|
|
P2_COMMIT = "8d330343af688e2de2b2a1d12ce19a582714709a" # Forge HEAD (P2.6)
|
|
P26_TAG = "v20260821_p26"
|
|
CONTRACT_HASH_RECORDED = "C44CC6F2B740C32D06F776BD7C3E669DC5A8A6DE0484230544EBFFCF517D38DD"
|
|
|
|
|
|
def sha256_file(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as f:
|
|
for blk in iter(lambda: f.read(1 << 20), b""):
|
|
h.update(blk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def sha256_bytes(b):
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def load_npz(name):
|
|
z = np.load(os.path.join(DATA, name + ".npz"))
|
|
return (z["time"].astype(np.int64), z["open"].astype(np.float64),
|
|
z["high"].astype(np.float64), z["low"].astype(np.float64),
|
|
z["close"].astype(np.float64), z["tick_volume"].astype(np.float64))
|
|
|
|
|
|
def load_data():
|
|
"""M15 bars 2017+ + HTF dict (same window as P2.6)."""
|
|
t, o, h, l, c, v = load_npz("XAUUSD_M15")
|
|
keep = t >= int(dt.datetime(2017, 1, 1, tzinfo=dt.timezone.utc).timestamp())
|
|
t, o, h, l, c, v = t[keep], o[keep], h[keep], l[keep], c[keep], v[keep]
|
|
htf = {}
|
|
for key in ("D1", "H4", "H1"):
|
|
ht_, ho_, hh_, hl_, hc_, hv_ = load_npz("XAUUSD_" + key)
|
|
htf[key] = (hh_, hl_, hc_, ht_)
|
|
return t, o, h, l, c, v, htf
|
|
|
|
|
|
def load_F():
|
|
"""Load the corrected 19-feature matrix from the P2.6 chunk cache."""
|
|
parts = []
|
|
for p in sorted(glob.glob(P26_CACHE)):
|
|
z = np.load(p)
|
|
parts.append(z["F"])
|
|
F = np.concatenate(parts, axis=0).astype(np.float64)
|
|
if F.shape[1] != NF:
|
|
raise RuntimeError(f"F cache has {F.shape[1]} cols, expected {NF}")
|
|
return F
|
|
|
|
|
|
def atr_series(h, l, c):
|
|
return np.maximum(TM.atr_series(h, l, c), 1e-9)
|
|
|
|
|
|
def make_label(c, A, horizon=TM.H_LABEL, thr_mult=TM.LABEL_ATR):
|
|
"""Directional label over `horizon` bars with thr = thr_mult x ATR.
|
|
Returns int array: +1 / -1 / 0 (no move). Identical to P2.6 contract."""
|
|
n = len(c)
|
|
lab = np.zeros(n, dtype=int)
|
|
for i in range(n - horizon):
|
|
fwd = c[i + horizon] - c[i]
|
|
thr = thr_mult * A[i]
|
|
if fwd >= thr:
|
|
lab[i] = 1
|
|
elif fwd <= -thr:
|
|
lab[i] = -1
|
|
return lab
|
|
|
|
|
|
def label_decompose(c, h, l, A, idxs, horizon=TM.H_LABEL, thr_mult=TM.LABEL_ATR):
|
|
"""Path decomposition for the label at decision bars `idxs` (no look-ahead).
|
|
|
|
For direction d = sign of eventual label (with thr = thr_mult*A[i]):
|
|
fwd : c[i+H] - c[i] (eventual move)
|
|
mfe : max favorable excursion over (i, i+H] in ATR units * d
|
|
mae : max adverse excursion in ATR units * d
|
|
time_to_hit: first bar j in (i, i+H] where favorable excursion >= thr
|
|
(NaN if never)
|
|
tp_before_sl: 1 if favorable excursion >= thr before adverse excursion
|
|
crosses thr in the opposite direction (within H)
|
|
sl_before_tp: 1 if adverse excursion <= -thr before favorable >= thr
|
|
no_hit : 1 if neither TP nor SL reached within H
|
|
All excursions measured from decision close c[i] (bar i closed).
|
|
Returns dict of arrays aligned with idxs.
|
|
"""
|
|
n = len(c)
|
|
m = len(idxs)
|
|
fwd = np.full(m, np.nan)
|
|
mfe = np.full(m, np.nan) # in ATR units, signed by direction
|
|
mae = np.full(m, np.nan)
|
|
t_hit = np.full(m, np.nan)
|
|
tp_before_sl = np.zeros(m, dtype=int)
|
|
sl_before_tp = np.zeros(m, dtype=int)
|
|
no_hit = np.zeros(m, dtype=int)
|
|
|
|
for k, i in enumerate(idxs):
|
|
if i + horizon >= n:
|
|
continue
|
|
thr = thr_mult * A[i]
|
|
seg_h = h[i + 1:i + 1 + horizon]
|
|
seg_l = l[i + 1:i + 1 + horizon]
|
|
seg_c = c[i + 1:i + 1 + horizon]
|
|
fwd[k] = c[i + horizon] - c[i]
|
|
d = 1 if (fwd[k] >= thr) else (-1 if fwd[k] <= -thr else 0)
|
|
if d == 0:
|
|
fe = np.maximum(seg_h - c[i], c[i] - seg_l)
|
|
ae = np.maximum(c[i] - seg_l, seg_h - c[i])
|
|
mfe[k] = float(np.max(fe)) / A[i] if len(fe) else np.nan
|
|
mae[k] = float(np.max(ae)) / A[i] if len(ae) else np.nan
|
|
no_hit[k] = 1
|
|
continue
|
|
fe = np.where(d == 1, seg_h - c[i], c[i] - seg_l) # favorable path
|
|
ae = np.where(d == 1, c[i] - seg_l, seg_h - c[i]) # adverse path
|
|
mfe[k] = float(np.max(fe)) / A[i] if len(fe) else np.nan
|
|
mae[k] = float(np.max(ae)) / A[i] if len(ae) else np.nan
|
|
hit_tp = np.where(fe >= thr)[0]
|
|
hit_sl = np.where(ae >= thr)[0]
|
|
if len(hit_tp) and (not len(hit_sl) or hit_tp[0] < hit_sl[0]):
|
|
tp_before_sl[k] = 1
|
|
t_hit[k] = hit_tp[0] + 1
|
|
elif len(hit_sl) and (not len(hit_tp) or hit_sl[0] < hit_tp[0]):
|
|
sl_before_tp[k] = 1
|
|
t_hit[k] = hit_sl[0] + 1
|
|
else:
|
|
no_hit[k] = 1
|
|
return dict(fwd=fwd, mfe=mfe, mae=mae, t_hit=t_hit,
|
|
tp_before_sl=tp_before_sl, sl_before_tp=sl_before_tp,
|
|
no_hit=no_hit)
|
|
|
|
|
|
def purged_split_idx(midx, split_frac=0.75, gap=TM.H_LABEL):
|
|
split = int(split_frac * len(midx))
|
|
tr_idx, te_idx, pg_idx, split_bar = TM.purged_split(midx, split, gap)
|
|
ok, tr_end, te_start = TM.purge_verify(tr_idx, te_idx, gap)
|
|
if not ok:
|
|
raise RuntimeError(f"purge gap rusak: tr_end={tr_end} te_start={te_start}")
|
|
return tr_idx, te_idx, pg_idx, split_bar
|
|
|
|
|
|
def dataset_hash(F, lab, A, t):
|
|
dh = hashlib.sha256()
|
|
dh.update(F.tobytes())
|
|
dh.update(lab.tobytes())
|
|
dh.update(A.tobytes())
|
|
dh.update(t.tobytes())
|
|
return dh.hexdigest()
|
|
|
|
|
|
def parse_mqh_arrays(path=P26_MODEL):
|
|
"""Parse SGML_* arrays from the exported frozen model .mqh.
|
|
Returns dict with mean, std, W1, b1, W2L, b2L, W2S, b2S."""
|
|
txt = open(path, encoding="utf-8").read()
|
|
|
|
def arr1(name):
|
|
mm = re.search(re.escape(name) + r"\[[^\]]*\]=\{(.*?)\};", txt, re.S)
|
|
if not mm:
|
|
raise RuntimeError(f"array {name} not found")
|
|
return np.array([float(x) for x in re.findall(
|
|
r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?", mm.group(1))])
|
|
|
|
def arr2(name, nrow, ncol):
|
|
# non-greedy: stop at the FIRST `};` that closes the array
|
|
mm = re.search(re.escape(name) + r"\[[^\]]*\]\[[^\]]*\]=\{(.*?)\};", txt, re.S)
|
|
if not mm:
|
|
raise RuntimeError(f"2d array {name} not found")
|
|
rows = re.findall(r"\{([^{}]*)\}", mm.group(1))
|
|
if len(rows) != nrow:
|
|
raise RuntimeError(f"{name}: found {len(rows)} rows, expected {nrow}")
|
|
out = np.zeros((nrow, ncol))
|
|
for ri, r in enumerate(rows):
|
|
vals = [float(x) for x in re.findall(
|
|
r"[-+]?[0-9]*\.?[0-9]+(?:[eE][-+]?[0-9]+)?", r)]
|
|
if len(vals) != ncol:
|
|
raise RuntimeError(f"{name} row {ri}: {len(vals)} vals, expected {ncol}")
|
|
out[ri] = vals
|
|
return out
|
|
|
|
mean = arr1("SGML_MEAN")
|
|
std = arr1("SGML_STD")
|
|
W1 = arr2("SGML_W1", NF, TM.HIDDEN)
|
|
b1 = arr1("SGML_B1")
|
|
W2L = arr1("SGML_W2L")
|
|
b2L = float(re.search(r"SGML_B2L=([-+0-9.eE]+)", txt).group(1))
|
|
W2S = arr1("SGML_W2S")
|
|
b2S = float(re.search(r"SGML_B2S=([-+0-9.eE]+)", txt).group(1))
|
|
return dict(mean=mean, std=std, W1=W1, b1=b1,
|
|
W2L=W2L, b2L=b2L, W2S=W2S, b2S=b2S)
|
|
|
|
|
|
def frozen_forward(M, X):
|
|
"""Frozen P2.6 MLP forward on (n,19) rows. X must be raw (not normalized).
|
|
Returns (p_long, p_short)."""
|
|
mean, std = M["mean"], M["std"]
|
|
sd = np.where(std < 1e-9, 1.0, std)
|
|
Z = (X - mean) / sd
|
|
H1 = np.maximum(0.0, Z @ M["W1"] + M["b1"])
|
|
oL = H1 @ M["W2L"] + M["b2L"]
|
|
oS = H1 @ M["W2S"] + M["b2S"]
|
|
return 1.0 / (1.0 + np.exp(-oL)), 1.0 / (1.0 + np.exp(-oS))
|
|
|
|
|
|
def provenance():
|
|
"""P3 provenance record (section 19)."""
|
|
p = {
|
|
"P2_COMMIT_SHA": P2_COMMIT,
|
|
"P2.6_SOURCE_SHA": P2_COMMIT, # P2.6 artifacts uncommitted; scripts hashed below
|
|
"FEATURE_CONTRACT_SHA_recorded": CONTRACT_HASH_RECORDED,
|
|
"FEATURE_CONTRACT_SHA_file_now": sha256_file(CONTRACT_PATH) if os.path.exists(CONTRACT_PATH) else None,
|
|
"MODEL_SHA": sha256_file(P26_MODEL),
|
|
"train_p26_script_sha": sha256_file(os.path.join(os.path.dirname(HERE), "parity", "train_p26.py")),
|
|
"build_features_p2_script_sha": sha256_file(os.path.join(os.path.dirname(HERE), "parity", "build_features_p2.py")),
|
|
"symbol": "XAUUSD", "timeframe": "M15",
|
|
"window_start": "2017-01-01",
|
|
}
|
|
return p
|
|
|
|
|
|
def save_json(name, obj):
|
|
outdir = os.path.join(HERE, "output")
|
|
os.makedirs(outdir, exist_ok=True)
|
|
path = os.path.join(outdir, name)
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(obj, f, indent=2, default=float)
|
|
print(f" [saved] {path}")
|
|
return path
|
|
|
|
|
|
if __name__ == "__main__":
|
|
t, o, h, l, c, v, htf = load_data()
|
|
F = load_F()
|
|
A = atr_series(h, l, c)
|
|
lab = make_label(c, A)
|
|
print(f"bars={len(c)} F={F.shape} labeled={int((lab != 0).sum())} "
|
|
f"bull={int((lab == 1).sum())} bear={int((lab == -1).sum())}")
|
|
dh = dataset_hash(F, lab, A, t)
|
|
print(f"DATASET_HASH={dh}")
|
|
print(f"(P2.6 recorded prefix e85a0861... match={dh.startswith('e85a0861')})")
|
|
M = parse_mqh_arrays()
|
|
print("frozen model parsed:", {k: (v.shape if hasattr(v, "shape") else v)
|
|
for k, v in M.items()})
|
|
print(json.dumps(provenance(), indent=2))
|