Warrior_EA/Expert/AIBase/PooledGate.mqh

//+------------------------------------------------------------------+
//|                                                   PooledGate.mqh |
//|          Cross-instrument certification for the deploy decision   |
//+------------------------------------------------------------------+
//| WHY THIS EXISTS.                                                  |
//|                                                                   |
//| Measured 2026-08-17 on SP500 H4: the OOS window is 4,738 bars and |
//| the mean triple-barrier label lifespan is 75.6 bars, so the       |
//| holdout carries ~63 INDEPENDENT observations (see                 |
//| EffectiveSampleSize). Certifying a 3pp edge at 2 sigma needs      |
//| ~1,036. The deploy gate was therefore unreachable by arithmetic - |
//| not because the models were short, but because the window cannot  |
//| resolve the question either way. Sixteen times short.             |
//|                                                                   |
//| Training on more data is one answer; it is not the answer to      |
//| THIS problem. The bottleneck is CERTIFICATION, and certification  |
//| pools cleanly across instruments in a way training does not: each |
//| symbol keeps its own model, its own derived geometry and its own  |
//| chance rate, and only the EVIDENCE is combined.                   |
//|                                                                   |
//| WHAT IS POOLED, and what must not be.                             |
//|                                                                   |
//| Not win rates. Symbols have different geometries and therefore    |
//| different break-evens and different always-long drifts; averaging |
//| raw win rates across them is meaningless. What pools is the       |
//| EXCESS OVER EACH SYMBOL'S OWN CHANCE RATE, combined by inverse-   |
//| variance weighting - the standard fixed-effects meta-analysis. A  |
//| symbol with a tighter estimate carries more weight, and each      |
//| symbol is compared only against itself.                           |
//|                                                                   |
//|   excess_i = p_i - c_i          var_i = c_i(1-c_i)/n_eff_i        |
//|   w_i      = (1/var_i) / SUM(1/var_j)                             |
//|   pooled   = SUM(w_i * excess_i)                                  |
//|                                                                   |
//| THE CORRELATION PROBLEM, handled honestly rather than assumed     |
//| away. SP500 and NAS100 are ~0.9 correlated; pooling them as       |
//| independent inflates the evidence. Nothing here can measure that  |
//| correlation without sharing return series, so instead of guessing |
//| it this file BRACKETS it and reports both ends:                   |
//|                                                                   |
//|   SE_INDEP = sqrt(1 / SUM(1/var_i))        all independent        |
//|   SE_CORR  = SUM(w_i * sqrt(var_i))        all perfectly correlated|
//|                                                                   |
//| The truth is between them, always. THE GATE USES SE_CORR - the    |
//| pessimistic end - so a pass cannot be an artifact of correlated   |
//| instruments, which on a funded account is the only safe default.  |
//| The ratio SE_CORR/SE_INDEP is logged as the "diversification      |
//| credit" the pool would earn if its members were independent, so   |
//| the cost of that conservatism is visible rather than hidden.      |
//|                                                                   |
//| ONE FILE PER INSTRUMENT, never a shared append target. Each chart |
//| writes only its own record and reads everyone else's, so there is |
//| no concurrent-write path to get wrong. Every FileOpen carries     |
//| FILE_SHARE_READ|FILE_SHARE_WRITE - the omission that produced the |
//| tester's 0-trade optimizer-cache corruption.                      |
//+------------------------------------------------------------------+
#property strict

//--- Records older than this are ignored. A chart that was stopped days ago is not evidence about the
//--- model running now, and a stale pool silently certifying a dead configuration is the failure mode
//--- this guards. Long enough to survive an overnight run, short enough that yesterday cannot vote.
#define POOL_MAX_AGE_HOURS 12
//--- Minimum distinct instruments before the pooled statistic is allowed to gate anything. Below this
//--- the pool is not a pool - it is one symbol with extra steps, and SE_CORR would equal SE_INDEP.
#define POOL_MIN_INSTRUMENTS 3
//--- Ceiling on peers read, sizing the fixed arrays. Well above the 24-instrument catalog.
#define POOL_MAX_RECORDS 64
//--- Bumped whenever the record layout changes. A reader that finds a different version SKIPS the
//--- record rather than misreading its columns - the stale-enum trap in a different costume.
#define POOL_RECORD_VERSION 1
#define POOL_DIR "Warrior_EA\\Pool"

//+------------------------------------------------------------------+
//| One instrument's contribution to the pooled certificate.          |
//+------------------------------------------------------------------+
struct SPoolRecord
  {
   string            symbol;
   int               timeframe;
   double            targetRR;      // ratio must match: it fixes the structural break-even
   double            chancePct;     // this symbol's own zero-skill rate
   double            winPct;        // what the model actually collected
   double            effN;          // INDEPENDENT calls, already deflated by label lifespan
   double            lifespanBars;
   long              eraCount;
   datetime          stamp;
  };

//+------------------------------------------------------------------+
//| Publish this instance's evidence. Called once per era, after the  |
//| local gate has computed its own numbers.                          |
//|                                                                   |
//| Writes ONLY this instrument's file. The filename carries symbol   |
//| and timeframe so two charts can never collide, and a chart that   |
//| restarts overwrites its own record rather than accumulating a     |
//| history that would let one instrument vote many times.            |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::PublishPoolRecord(double chancePct, double winPct, double effN)
  {
   //--- Nothing measurable to contribute. Writing a placeholder would let a symbol with no evidence
   //--- dilute the pool's weighting, which is the opposite of what inverse-variance weighting is for.
   if(chancePct <= 0.0 || chancePct >= 100.0 || winPct < 0.0 || effN < 2.0)
      return;
   //--- _Period, not Period(): inside a CExpertBase subclass the bare call resolves to the inherited
   //--- SETTER bool CExpertBase::Period(ENUM_TIMEFRAMES) rather than the builtin. Same predefined
   //--- variable AutoTune.mqh uses to build its own per-symbol/timeframe filename.
   string fn = StringFormat("%s\\%s_%d.csv", POOL_DIR, m_symbol.Name(), (int)_Period);
   int h = FileOpen(fn, FILE_COMMON | FILE_WRITE | FILE_CSV | FILE_ANSI |
                    FILE_SHARE_READ | FILE_SHARE_WRITE, ',');
   if(h == INVALID_HANDLE)
     {
      //--- One-shot: a pool that cannot be written is a silent loss of the whole mechanism, and a
      //--- mechanism that declines to act must announce it (the third quiet no-op this codebase has
      //--- been bitten by).
      if(!m_poolWriteWarned)
        {
         m_poolWriteWarned = true;
         Print(ID + StringFormat(": WARNING - cannot write the pooled-gate record %s (error %d). This "
                                 "instrument will not contribute to cross-instrument certification and "
                                 "the pool will be short one member.", fn, GetLastError()));
        }
      return;
     }
   FileWrite(h, POOL_RECORD_VERSION, m_symbol.Name(), (int)_Period, BARRIER_TARGET_RR,
             chancePct, winPct, effN, MeanLabelLifespan(), (long)m_eraCount, (long)TimeCurrent());
   FileClose(h);
  }

//+------------------------------------------------------------------+
//| Read every peer's record and combine. Returns the number of       |
//| instruments that qualified; the pooled figures come back through  |
//| the out-params.                                                   |
//|                                                                   |
//| INCLUDES THIS INSTRUMENT: its own file was just written, so the   |
//| directory scan picks it up like any other and there is no special |
//| case to get wrong.                                                |
//+------------------------------------------------------------------+
int CExpertSignalAIBase::ReadPooledEvidence(double &pooledExcessPp, double &seIndep, double &seCorr,
                                            string &detail)
  {
   pooledExcessPp = 0.0;
   seIndep = 0.0;
   seCorr = 0.0;
   detail = "";
   SPoolRecord rec[POOL_MAX_RECORDS];
   int count = 0;
   string found;
   long searchHandle = FileFindFirst(POOL_DIR + "\\*.csv", found, FILE_COMMON);
   if(searchHandle == INVALID_HANDLE)
      return 0;
   do
     {
      if(count >= POOL_MAX_RECORDS)
         break;
      int h = FileOpen(POOL_DIR + "\\" + found, FILE_COMMON | FILE_READ | FILE_CSV | FILE_ANSI |
                       FILE_SHARE_READ | FILE_SHARE_WRITE, ',');
      if(h == INVALID_HANDLE)
         continue;
      int ver = (int)FileReadNumber(h);
      //--- Version mismatch: SKIP, never reinterpret. Reading v2 columns as v1 would produce a
      //--- plausible-looking record built from the wrong fields, which is worse than one fewer member.
      if(ver != POOL_RECORD_VERSION)
        {
         FileClose(h);
         continue;
        }
      SPoolRecord r;
      r.symbol       = FileReadString(h);
      r.timeframe    = (int)FileReadNumber(h);
      r.targetRR     = FileReadNumber(h);
      r.chancePct    = FileReadNumber(h);
      r.winPct       = FileReadNumber(h);
      r.effN         = FileReadNumber(h);
      r.lifespanBars = FileReadNumber(h);
      r.eraCount     = (long)FileReadNumber(h);
      r.stamp        = (datetime)(long)FileReadNumber(h);
      FileClose(h);
      //--- POOLABILITY. Symbol deliberately does NOT have to match - that is the entire point - but
      //--- everything that changes what the numbers MEAN does.
      if(r.timeframe != (int)_Period)
         continue;                          // different bar semantics
      if(MathAbs(r.targetRR - BARRIER_TARGET_RR) > 0.01)
         continue;                          // different structural break-even
      if(r.effN < 2.0 || r.chancePct <= 0.0 || r.chancePct >= 100.0)
         continue;                          // no usable estimate
      if((TimeCurrent() - r.stamp) > POOL_MAX_AGE_HOURS * 3600)
         continue;                          // stale: a stopped chart is not evidence about now
      rec[count++] = r;
     }
   while(FileFindNext(searchHandle, found));
   FileFindClose(searchHandle);
   if(count <= 0)
      return 0;
   //--- FIXED-EFFECTS COMBINATION. Each symbol is scored against ITS OWN chance rate, so symbols with
   //--- different geometries and different drifts are directly comparable in this one currency.
   double sumInvVar = 0.0;
   for(int i = 0; i < count; i++)
     {
      double c = rec[i].chancePct / 100.0;
      double var = c * (1.0 - c) / rec[i].effN;      // in fraction^2
      if(var <= 0.0)
         continue;
      sumInvVar += 1.0 / var;
     }
   if(sumInvVar <= 0.0)
      return 0;
   double weighted = 0.0, sumWSd = 0.0;
   for(int i = 0; i < count; i++)
     {
      double c = rec[i].chancePct / 100.0;
      double var = c * (1.0 - c) / rec[i].effN;
      if(var <= 0.0)
         continue;
      double w = (1.0 / var) / sumInvVar;
      weighted += w * (rec[i].winPct - rec[i].chancePct);   // percentage points
      //--- Perfectly-correlated bound: the weighted sum of the individual standard deviations, which
      //--- is what the pooled SE degenerates to when every member moves together.
      sumWSd += w * MathSqrt(var) * 100.0;
      detail += StringFormat("%s%s %.0f%%/%.0f%% n%.0f w%.2f", (detail == "" ? "" : " "),
                             rec[i].symbol, rec[i].winPct, rec[i].chancePct, rec[i].effN, w);
     }
   pooledExcessPp = weighted;
   seIndep = 100.0 * MathSqrt(1.0 / sumInvVar);
   seCorr  = sumWSd;
   return count;
  }

//+------------------------------------------------------------------+
//| The cross-instrument verdict, as one log line and one boolean.    |
//|                                                                   |
//| Gates on the PESSIMISTIC bound. A pooled edge that clears SE_CORR |
//| cannot be explained by the members being correlated, because that |
//| bound already assumes they are perfectly correlated.              |
//+------------------------------------------------------------------+
bool CExpertSignalAIBase::PooledGatePasses(string &report)
  {
   report = "";
   double excess = 0.0, seIndep = 0.0, seCorr = 0.0;
   string detail = "";
   int members = ReadPooledEvidence(excess, seIndep, seCorr, detail);
   if(members < POOL_MIN_INSTRUMENTS)
     {
      report = StringFormat(" | POOL %d/%d instruments - not enough to certify across symbols yet"
                            " (run more charts on this timeframe at the same %.1f:1 ratio; each one is"
                            " worth far more independent evidence than more bars of the same symbol)",
                            members, POOL_MIN_INSTRUMENTS, BARRIER_TARGET_RR);
      return false;
     }
   double bar = EDGE_MIN_SIGMAS * seCorr;
   bool   pass = (excess > bar);
   //--- The credit the pool WOULD earn if its members were independent. Printed because the gap between
   //--- the two bounds is the whole cost of refusing to assume independence, and it is the number that
   //--- says whether adding a genuinely uncorrelated instrument is worth more than another correlated one.
   double credit = (seIndep > 0.0) ? seCorr / seIndep : 1.0;
   report = StringFormat(" | POOLED CERTIFICATE across %d instruments: excess %+.2fpp vs a %.2fpp bar"
                         " (%.0f x SE %.2fpp, the ALL-CORRELATED bound; SE would be %.2fpp if the"
                         " instruments were independent, a %.1fx diversification credit this gate"
                         " deliberately declines to claim) -> %s [%s]",
                         members, excess, bar, EDGE_MIN_SIGMAS, seCorr, seIndep, credit,
                         (pass ? "PASSES" : "fails"), detail);
   return pass;
  }
//+------------------------------------------------------------------+