//+------------------------------------------------------------------+ //| DipMeta.mqh | //| AnimateDread | //| | //| THE META-LABEL FOR THE DIP-BUY: take this dip, or skip it. | //| | //| The architecture the operator settled on: the SETUP is the | //| strategy, the model only says yes or no to a trade the rules have | //| already found. The dip-buy is the first setup in this EA with a | //| measured positive base rate to say no against (60-77% wins, +10 | //| to +35 bp a trade), so it is the first place a model earns a seat.| //| | //| WHAT IT LEARNS FROM. Every past signal bar becomes one row: the | //| state of the market at that bar (depth of the dip, distance to | //| the 200-bar average, the streak, the bar's own shape, the vol | //| regime, ER/VR, weekday, hour) and the label is what the EA's OWN | //| exit rule would have made of it - stop, mean-cross or time stop, | //| simulated on closed bars. No journal, no file, no Python: the | //| rows are rebuilt from history-so-far on every refit, so the fit | //| is walk-forward by construction and every bar it votes on is out | //| of sample. | //| | //| TWO ALGLIB MODELS, ONE ANSWER. A random decision forest and the | //| MLP (CWarriorNet) are fitted on the same chronological split with | //| the same embargo; each reports its held-out AUC; the score is the | //| mean of whichever are ready. The forest exists because 300 daily | //| dips cannot support even a two-unit MLP at ten rows per weight, | //| and a forest degrades gracefully where the net refuses. ALGLIB's | //| statistics (Pearson / Spearman) print each feature's correlation | //| with the label first, so a fit that "works" can be read against | //| what it could possibly have used. | //| | //| NEVER SERIALISED. DFSerialize crashes on compressed forests and | //| MLPSerialize dies in CSerializer::Stop() - both already met here. | //+------------------------------------------------------------------+ #ifndef WARRIOR_DIP_META_MQH #define WARRIOR_DIP_META_MQH #include "WarriorNet.mqh" #define DIPMETA_FEATURES 15 #define DIPMETA_TREES 100 // forest size #define DIPMETA_R 0.66 // fraction of rows each tree sees (ALGLIB's r) #define DIPMETA_MIN_ROWS 120 // the forest's floor; the MLP sets its own (10 rows/weight) class CDipMeta { private: CWarriorNet m_mlp; CDecisionForestShell m_df; bool m_mlpReady, m_dfReady; double m_dfAuc; int m_nIn; string m_why; static double RankAUC(double &p[], int &l[], const int n); public: CDipMeta(void) : m_mlpReady(false), m_dfReady(false), m_dfAuc(-1.0), m_nIn(0), m_why("not trained") {} ~CDipMeta(void) {} bool Ready(void) const { return m_mlpReady || m_dfReady; } string Why(void) const { return m_why; } double ForestAUC(void) const { return m_dfAuc; } double MlpAUC(void) const { return m_mlpReady ? m_mlp.AUC() : -1.0; } static string FeatureName(const int i); bool Train(CMatrixDouble &xy, const int rows, const int embargo); //--- P(this dip pays), the mean of the ready models; <0 when none is. double Score(double &x[]); }; //+------------------------------------------------------------------+ string CDipMeta::FeatureName(const int i) { switch(i) { //--- THE DIP ITSELF case 0: return "z20"; // how deep, in standard deviations of the 20-bar window case 1: return "dist200_atr"; // where it sits against the 200-bar mean, in ATR case 2: return "ret1_atr"; // the signal bar's own move case 3: return "ret5_atr"; // the five-bar move that produced the dip case 4: return "streak_down"; // consecutive down closes case 5: return "pos20"; // position inside the 20-bar range, 0 = at the low //--- THE SIGNAL BAR'S SHAPE. A wide bar closing at its low is capitulation; closing off the //--- low is a bounce that already began - and in the real trades day-1 direction was the //--- strongest single split (90% win vs 53%). case 6: return "range_atr"; case 7: return "close_in_bar"; case 8: return "gap_atr"; // the open against the previous close //--- THE REGIME. The dip inside a trend paid +117 bp, in chop +33, in reversion +13. case 9: return "vol_rel"; // ATR14 / ATR100 - is this a volatility event case 10: return "er20"; case 11: return "vr60"; case 12: return "regime"; //--- THE CALENDAR. Thu/Fri fills +57/+55 vs Tue +3; Mondays lose on the H4 ensemble. case 13: return "dow"; case 14: return "hour"; } return "?"; } //+------------------------------------------------------------------+ //| Mann-Whitney rank-sum AUC, ties averaged. Same arithmetic as the | //| net's own report, so the two models are scored identically. | //+------------------------------------------------------------------+ double CDipMeta::RankAUC(double &p[], int &l[], const int n) { int pos = 0; for(int i = 0; i < n; i++) if(l[i] == 1) pos++; const int neg = n - pos; if(pos == 0 || neg == 0) return 0.5; int idx[]; ArrayResize(idx, n); for(int i = 0; i < n; i++) idx[i] = i; for(int i = 1; i < n; i++) { const int k = idx[i]; int j = i - 1; while(j >= 0 && p[idx[j]] > p[k]) { idx[j + 1] = idx[j]; j--; } idx[j + 1] = k; } double rankSum = 0.0; int i2 = 0; while(i2 < n) { int j2 = i2; while(j2 + 1 < n && p[idx[j2 + 1]] == p[idx[i2]]) j2++; const double avgRank = 0.5 * ((i2 + 1) + (j2 + 1)); for(int k2 = i2; k2 <= j2; k2++) if(l[idx[k2]] == 1) rankSum += avgRank; i2 = j2 + 1; } return (rankSum - 0.5 * pos * (pos + 1.0)) / ((double)pos * neg); } //+------------------------------------------------------------------+ bool CDipMeta::Train(CMatrixDouble &xy, const int rows, const int embargo) { m_mlpReady = false; m_dfReady = false; m_nIn = DIPMETA_FEATURES; if(rows < DIPMETA_MIN_ROWS) { m_why = StringFormat("%d row(s) is below the %d-row floor - not trained", rows, DIPMETA_MIN_ROWS); return false; } string names[]; ArrayResize(names, m_nIn); for(int f = 0; f < m_nIn; f++) names[f] = FeatureName(f); //--- 1. WHAT IS THERE TO LEARN. Each feature against the label, Pearson and Spearman, over every //--- row. Printed BEFORE the fits so the AUCs below are read against the raw material. double lab[]; ArrayResize(lab, rows); int posAll = 0; for(int r = 0; r < rows; r++) { lab[r] = xy.Get(r, m_nIn); if(lab[r] > 0.5) posAll++; } string corr = ""; double col[]; ArrayResize(col, rows); for(int f = 0; f < m_nIn; f++) { for(int r = 0; r < rows; r++) col[r] = xy.Get(r, f); const double rp = CAlglib::PearsonCorr2(col, lab, rows); const double rs = CAlglib::SpearmanCorr2(col, lab, rows); corr += StringFormat("%s%s %+.3f/%+.3f", (f ? ", " : ""), names[f], rp, rs); } PrintFormat("CDipMeta: %d row(s), %d paid (%.1f%%). Feature vs label, Pearson/Spearman: %s", rows, posAll, 100.0 * posAll / rows, corr); //--- 2. THE SPLIT, identical for both models: the tail is validation, `embargo` rows before it //--- are dropped so no training row shares its outcome window with the first validation row. const int valRows = (int)MathMax(1, MathRound(rows * WARRIOR_NET_VAL_FRAC)); const int gap = (embargo > 0 && embargo < rows / 4) ? embargo : 0; const int trnRows = rows - valRows - gap; if(trnRows < 40) { m_why = "training half too small after the split"; return false; } CMatrixDouble trn(trnRows, m_nIn + 1); for(int r = 0; r < trnRows; r++) for(int c = 0; c <= m_nIn; c++) trn.Set(r, c, xy.Get(r, c)); //--- 3. THE FOREST. int info = 0; CDFReportShell rep; CAlglib::DFBuildRandomDecisionForest(trn, trnRows, m_nIn, 2, DIPMETA_TREES, DIPMETA_R, info, m_df, rep); if(info > 0) { double p[]; int l[]; ArrayResize(p, valRows); ArrayResize(l, valRows); double x[], y[]; ArrayResize(x, m_nIn); int hit = 0, pos = 0; for(int r = 0; r < valRows; r++) { const int src = trnRows + gap + r; for(int c = 0; c < m_nIn; c++) x[c] = xy.Get(src, c); CAlglib::DFProcess(m_df, x, y); p[r] = (ArraySize(y) > 1) ? y[1] : 0.0; l[r] = (int)MathRound(xy.Get(src, m_nIn)); if(l[r] == 1) pos++; if((p[r] >= 0.5 ? 1 : 0) == l[r]) hit++; } m_dfAuc = RankAUC(p, l, valRows); //--- DOES THE TOP THIRD PAY MORE THAN THE BOTTOM THIRD? The AUC says whether the ranking has //--- information; this says whether acting on it would have changed the win rate. int order[]; ArrayResize(order, valRows); for(int i = 0; i < valRows; i++) order[i] = i; for(int i = 1; i < valRows; i++) { const int k = order[i]; int j = i - 1; while(j >= 0 && p[order[j]] > p[k]) { order[j + 1] = order[j]; j--; } order[j + 1] = k; } const int third = valRows / 3; int loWin = 0, hiWin = 0; for(int i = 0; i < third; i++) { if(l[order[i]] == 1) loWin++; if(l[order[valRows - 1 - i]] == 1) hiWin++; } m_dfReady = true; PrintFormat("CDipMeta: FOREST (%d trees) trained on %d / embargo %d / validation %d rows - " "OUT-OF-SAMPLE AUC %.3f, accuracy %.1f%% against a %.1f%% base rate; win rate in the " "model's bottom third %.0f%% vs top third %.0f%% (n %d each).", DIPMETA_TREES, trnRows, gap, valRows, m_dfAuc, 100.0 * hit / valRows, 100.0 * pos / valRows, third > 0 ? 100.0 * loWin / third : 0.0, third > 0 ? 100.0 * hiWin / third : 0.0, third); } else PrintFormat("CDipMeta: DFBuildRandomDecisionForest refused the dataset (info %d).", info); //--- 4. THE MLP, on the same rows; it refuses on its own when the sample cannot carry it. m_mlpReady = m_mlp.Train(xy, rows, m_nIn, names, embargo); if(!m_mlpReady) PrintFormat("CDipMeta: MLP not fitted - %s.", m_mlp.Why()); m_why = Ready() ? "trained" : "no model could be fitted"; return Ready(); } //+------------------------------------------------------------------+ double CDipMeta::Score(double &x[]) { if(!Ready() || ArraySize(x) != m_nIn) return -1.0; double sum = 0.0; int n = 0; if(m_dfReady) { double y[]; CAlglib::DFProcess(m_df, x, y); if(ArraySize(y) > 1) { sum += y[1]; n++; } } if(m_mlpReady) { const double p = m_mlp.Score(x); if(p >= 0.0) { sum += p; n++; } } return (n > 0) ? sum / n : -1.0; } #endif // WARRIOR_DIP_META_MQH