//+------------------------------------------------------------------+ //| PooledGate.mqh | //| Cross-instrument certification for the deploy decision | //+------------------------------------------------------------------+ //| WHY THIS EXISTS. | //| | //| Measured 2026-08-17 on SP500 H4: the OOS window is 4,738 bars and | //| the mean triple-barrier label lifespan is 75.6 bars, so the | //| holdout carries ~63 INDEPENDENT observations (see | //| EffectiveSampleSize). Certifying a 3pp edge at 2 sigma needs | //| ~1,036. The deploy gate was therefore unreachable by arithmetic - | //| not because the models were short, but because the window cannot | //| resolve the question either way. Sixteen times short. | //| | //| Training on more data is one answer; it is not the answer to | //| THIS problem. The bottleneck is CERTIFICATION, and certification | //| pools cleanly across instruments in a way training does not: each | //| symbol keeps its own model, its own derived geometry and its own | //| chance rate, and only the EVIDENCE is combined. | //| | //| WHAT IS POOLED, and what must not be. | //| | //| Not win rates. Symbols have different geometries and therefore | //| different break-evens and different always-long drifts; averaging | //| raw win rates across them is meaningless. What pools is the | //| EXCESS OVER EACH SYMBOL'S OWN CHANCE RATE, combined by inverse- | //| variance weighting - the standard fixed-effects meta-analysis. A | //| symbol with a tighter estimate carries more weight, and each | //| symbol is compared only against itself. | //| | //| excess_i = p_i - c_i var_i = c_i(1-c_i)/n_eff_i | //| w_i = (1/var_i) / SUM(1/var_j) | //| pooled = SUM(w_i * excess_i) | //| | //| THE CORRELATION PROBLEM, handled honestly rather than assumed | //| away. SP500 and NAS100 are ~0.9 correlated; pooling them as | //| independent inflates the evidence. Nothing here can measure that | //| correlation without sharing return series, so instead of guessing | //| it this file BRACKETS it and reports both ends: | //| | //| SE_INDEP = sqrt(1 / SUM(1/var_i)) all independent | //| SE_CORR = SUM(w_i * sqrt(var_i)) all perfectly correlated| //| | //| The truth is between them, always. THE GATE USES SE_CORR - the | //| pessimistic end - so a pass cannot be an artifact of correlated | //| instruments, which on a funded account is the only safe default. | //| The ratio SE_CORR/SE_INDEP is logged as the "diversification | //| credit" the pool would earn if its members were independent, so | //| the cost of that conservatism is visible rather than hidden. | //| | //| ONE FILE PER INSTRUMENT, never a shared append target. Each chart | //| writes only its own record and reads everyone else's, so there is | //| no concurrent-write path to get wrong. Every FileOpen carries | //| FILE_SHARE_READ|FILE_SHARE_WRITE - the omission that produced the | //| tester's 0-trade optimizer-cache corruption. | //+------------------------------------------------------------------+ #property strict //--- Records older than this are ignored. A chart that was stopped days ago is not evidence about the //--- model running now, and a stale pool silently certifying a dead configuration is the failure mode //--- this guards. Long enough to survive an overnight run, short enough that yesterday cannot vote. #define POOL_MAX_AGE_HOURS 12 //--- Minimum distinct instruments before the pooled statistic is allowed to gate anything. Below this //--- the pool is not a pool - it is one symbol with extra steps, and SE_CORR would equal SE_INDEP. #define POOL_MIN_INSTRUMENTS 3 //--- Ceiling on peers read, sizing the fixed arrays. Well above the 24-instrument catalog. #define POOL_MAX_RECORDS 64 //--- Bumped whenever the record layout changes. A reader that finds a different version SKIPS the //--- record rather than misreading its columns - the stale-enum trap in a different costume. #define POOL_RECORD_VERSION 1 #define POOL_DIR "Warrior_EA\\Pool" //+------------------------------------------------------------------+ //| One instrument's contribution to the pooled certificate. | //+------------------------------------------------------------------+ struct SPoolRecord { string symbol; int timeframe; double targetRR; // ratio must match: it fixes the structural break-even double chancePct; // this symbol's own zero-skill rate double winPct; // what the model actually collected double effN; // INDEPENDENT calls, already deflated by label lifespan double lifespanBars; long eraCount; datetime stamp; }; //+------------------------------------------------------------------+ //| Publish this instance's evidence. Called once per era, after the | //| local gate has computed its own numbers. | //| | //| Writes ONLY this instrument's file. The filename carries symbol | //| and timeframe so two charts can never collide, and a chart that | //| restarts overwrites its own record rather than accumulating a | //| history that would let one instrument vote many times. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::PublishPoolRecord(double chancePct, double winPct, double effN) { //--- Nothing measurable to contribute. Writing a placeholder would let a symbol with no evidence //--- dilute the pool's weighting, which is the opposite of what inverse-variance weighting is for. if(chancePct <= 0.0 || chancePct >= 100.0 || winPct < 0.0 || effN < 2.0) return; //--- _Period, not Period(): inside a CExpertBase subclass the bare call resolves to the inherited //--- SETTER bool CExpertBase::Period(ENUM_TIMEFRAMES) rather than the builtin. Same predefined //--- variable AutoTune.mqh uses to build its own per-symbol/timeframe filename. string fn = StringFormat("%s\\%s_%d.csv", POOL_DIR, m_symbol.Name(), (int)_Period); int h = FileOpen(fn, FILE_COMMON | FILE_WRITE | FILE_CSV | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE, ','); if(h == INVALID_HANDLE) { //--- One-shot: a pool that cannot be written is a silent loss of the whole mechanism, and a //--- mechanism that declines to act must announce it (the third quiet no-op this codebase has //--- been bitten by). if(!m_poolWriteWarned) { m_poolWriteWarned = true; Print(ID + StringFormat(": WARNING - cannot write the pooled-gate record %s (error %d). This " "instrument will not contribute to cross-instrument certification and " "the pool will be short one member.", fn, GetLastError())); } return; } FileWrite(h, POOL_RECORD_VERSION, m_symbol.Name(), (int)_Period, BARRIER_TARGET_RR, chancePct, winPct, effN, MeanLabelLifespan(), (long)m_eraCount, (long)TimeCurrent()); FileClose(h); } //+------------------------------------------------------------------+ //| Read every peer's record and combine. Returns the number of | //| instruments that qualified; the pooled figures come back through | //| the out-params. | //| | //| INCLUDES THIS INSTRUMENT: its own file was just written, so the | //| directory scan picks it up like any other and there is no special | //| case to get wrong. | //+------------------------------------------------------------------+ int CExpertSignalAIBase::ReadPooledEvidence(double &pooledExcessPp, double &seIndep, double &seCorr, string &detail) { pooledExcessPp = 0.0; seIndep = 0.0; seCorr = 0.0; detail = ""; SPoolRecord rec[POOL_MAX_RECORDS]; int count = 0; string found; long searchHandle = FileFindFirst(POOL_DIR + "\\*.csv", found, FILE_COMMON); if(searchHandle == INVALID_HANDLE) return 0; do { if(count >= POOL_MAX_RECORDS) break; int h = FileOpen(POOL_DIR + "\\" + found, FILE_COMMON | FILE_READ | FILE_CSV | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE, ','); if(h == INVALID_HANDLE) continue; int ver = (int)FileReadNumber(h); //--- Version mismatch: SKIP, never reinterpret. Reading v2 columns as v1 would produce a //--- plausible-looking record built from the wrong fields, which is worse than one fewer member. if(ver != POOL_RECORD_VERSION) { FileClose(h); continue; } SPoolRecord r; r.symbol = FileReadString(h); r.timeframe = (int)FileReadNumber(h); r.targetRR = FileReadNumber(h); r.chancePct = FileReadNumber(h); r.winPct = FileReadNumber(h); r.effN = FileReadNumber(h); r.lifespanBars = FileReadNumber(h); r.eraCount = (long)FileReadNumber(h); r.stamp = (datetime)(long)FileReadNumber(h); FileClose(h); //--- POOLABILITY. Symbol deliberately does NOT have to match - that is the entire point - but //--- everything that changes what the numbers MEAN does. if(r.timeframe != (int)_Period) continue; // different bar semantics if(MathAbs(r.targetRR - BARRIER_TARGET_RR) > 0.01) continue; // different structural break-even if(r.effN < 2.0 || r.chancePct <= 0.0 || r.chancePct >= 100.0) continue; // no usable estimate if((TimeCurrent() - r.stamp) > POOL_MAX_AGE_HOURS * 3600) continue; // stale: a stopped chart is not evidence about now rec[count++] = r; } while(FileFindNext(searchHandle, found)); FileFindClose(searchHandle); if(count <= 0) return 0; //--- FIXED-EFFECTS COMBINATION. Each symbol is scored against ITS OWN chance rate, so symbols with //--- different geometries and different drifts are directly comparable in this one currency. double sumInvVar = 0.0; for(int i = 0; i < count; i++) { double c = rec[i].chancePct / 100.0; double var = c * (1.0 - c) / rec[i].effN; // in fraction^2 if(var <= 0.0) continue; sumInvVar += 1.0 / var; } if(sumInvVar <= 0.0) return 0; double weighted = 0.0, sumWSd = 0.0; for(int i = 0; i < count; i++) { double c = rec[i].chancePct / 100.0; double var = c * (1.0 - c) / rec[i].effN; if(var <= 0.0) continue; double w = (1.0 / var) / sumInvVar; weighted += w * (rec[i].winPct - rec[i].chancePct); // percentage points //--- Perfectly-correlated bound: the weighted sum of the individual standard deviations, which //--- is what the pooled SE degenerates to when every member moves together. sumWSd += w * MathSqrt(var) * 100.0; detail += StringFormat("%s%s %.0f%%/%.0f%% n%.0f w%.2f", (detail == "" ? "" : " "), rec[i].symbol, rec[i].winPct, rec[i].chancePct, rec[i].effN, w); } pooledExcessPp = weighted; seIndep = 100.0 * MathSqrt(1.0 / sumInvVar); seCorr = sumWSd; return count; } //+------------------------------------------------------------------+ //| The cross-instrument verdict, as one log line and one boolean. | //| | //| Gates on the PESSIMISTIC bound. A pooled edge that clears SE_CORR | //| cannot be explained by the members being correlated, because that | //| bound already assumes they are perfectly correlated. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::PooledGatePasses(string &report) { report = ""; double excess = 0.0, seIndep = 0.0, seCorr = 0.0; string detail = ""; int members = ReadPooledEvidence(excess, seIndep, seCorr, detail); if(members < POOL_MIN_INSTRUMENTS) { report = StringFormat(" | POOL %d/%d instruments - not enough to certify across symbols yet" " (run more charts on this timeframe at the same %.1f:1 ratio; each one is" " worth far more independent evidence than more bars of the same symbol)", members, POOL_MIN_INSTRUMENTS, BARRIER_TARGET_RR); return false; } double bar = EDGE_MIN_SIGMAS * seCorr; bool pass = (excess > bar); //--- The credit the pool WOULD earn if its members were independent. Printed because the gap between //--- the two bounds is the whole cost of refusing to assume independence, and it is the number that //--- says whether adding a genuinely uncorrelated instrument is worth more than another correlated one. double credit = (seIndep > 0.0) ? seCorr / seIndep : 1.0; report = StringFormat(" | POOLED CERTIFICATE across %d instruments: excess %+.2fpp vs a %.2fpp bar" " (%.0f x SE %.2fpp, the ALL-CORRELATED bound; SE would be %.2fpp if the" " instruments were independent, a %.1fx diversification credit this gate" " deliberately declines to claim) -> %s [%s]", members, excess, bar, EDGE_MIN_SIGMAS, seCorr, seIndep, credit, (pass ? "PASSES" : "fails"), detail); return pass; } //+------------------------------------------------------------------+