Warrior_EA/Expert/AIBase/MetaCorpus.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//| Meta-label training corpus - stage S1 of Meta_Labeling_Design.md |
//|                                                                  |
//| The corpus is READ FROM THE SIGNAL DATABASE, not produced by a   |
//| training-time re-implementation of the classic ladders. The DB   |
//| logs every pattern instance the live ladders fire - both sides,  |
//| uncensored, with the filter's net vote and the touchable entry   |
//| price (per-side journaling, 652bf81/195be20) - so a corpus read  |
//| from it is BY CONSTRUCTION identical to what fires live, and the |
//| several hundred lines of condition-mirroring a separate sweep    |
//| would need (a silent-divergence trap) never get written. The     |
//| cost of the trade: corpus coverage equals whatever backtest      |
//| last populated the DB, so a corpus build IS a tester run with    |
//| UseDatabaseRanking on and DB_MaxRowsPerTable raised.             |
//|                                                                  |
//| S1 ships the reader and its report; S2 attaches labels (triple-  |
//| barrier at the EA's own geometry from the candidate's bar) and   |
//| the setup-descriptor features. The report already measures the   |
//| one mapping S2 depends on: DB rows carry GMT timestamps while    |
//| bar history is in server time, so the report tries hour offsets  |
//| 0..+3 and prints the exact-bar match rate of each - measured,    |
//| not assumed (the broker is UTC+2-ish; DST behaviour is exactly   |
//| the kind of fact this project checks empirically).               |
//+------------------------------------------------------------------+
#include "..\..\Structures\tradeRecordStructure.mqh"
#include "..\..\System\PrintVerbose.mqh"

//--- One candidate = one journaled pattern instance. `win` is the DB's stop-and-reverse outcome
//--- (entry-to-reversal), kept for reporting; S2's training label is computed independently from
//--- price history (triple-barrier at the EA's geometry) and does NOT reuse this field.
struct SMetaCandidate
  {
   datetime          gmt;         // journal timestamp (GMT, minute resolution)
   short             family;      // 0=MA 1=RSI 2=MACD 3=Ichimoku
   short             pattern;     // Pattern_N index within the family
   short             side;        // +1 Buy / -1 Sell
   double            netVote;     // the filter's own vote margin when it fired (decision context)
   double            entryPrice;  // touchable side of the spread at fire time
   bool              closed;      // result != NA
   bool              win;         // result == Profit (stop-and-reverse outcome, reporting only)
  };

string MetaFamilyName(const int f)
  {
   switch(f)
     {
      case 0: return "MA";
      case 1: return "RSI";
      case 2: return "MACD";
      case 3: return "Ichimoku";
     }
   return "";
  }

int MetaFamilyPatterns(const int f)
  {
   switch(f)
     {
      case 0: return 4;    // Signals\SignalMA.mqh      m_pattern_count
      case 1: return 4;    // Signals\SignalRSI.mqh
      case 2: return 6;    // Signals\SignalMACD.mqh
      case 3: return 12;   // Signals\SignalIchimoku.mqh
     }
   return 0;
  }

class CMetaCorpus
  {
public:
   SMetaCandidate    m_rows[];
   int               m_count;

                     CMetaCorpus(void) : m_count(0) {}

   //--- Pull every pattern table into m_rows. Missing tables (family disabled in the run that
   //--- populated this DB) are skipped and counted, not treated as errors.
   int               Load(void)
     {
      m_count = 0;
      ArrayResize(m_rows, 0);
      if(!dbm.OpenDatabase())
        {
         Print(__FUNCTION__ + ": signal database is not available - no corpus.");
         return 0;
        }
      int missing = 0;
      TradeRecord rec;
      for(int f = 0; f < 4; f++)
         for(int p = 0; p < MetaFamilyPatterns(f); p++)
            for(int s = 0; s < 2; s++)
              {
               string table = MetaFamilyName(f) + "_Pattern_" + IntegerToString(p) +
                              (s == 0 ? "_Buy" : "_Sell");
               TradeRecord rows[];
               if(!dbm.FetchTradeRecords(table, rec, rows))
                 {
                  missing++;
                  continue;
                 }
               int base = m_count;
               ArrayResize(m_rows, base + ArraySize(rows));
               for(int i = 0; i < ArraySize(rows); i++)
                 {
                  MqlDateTime t;
                  t.year = rows[i].year;
                  t.mon  = rows[i].month;
                  t.day  = rows[i].day;
                  t.hour = rows[i].hour;
                  t.min  = rows[i].minutes;
                  t.sec  = 0;
                  m_rows[base + i].gmt        = StructToTime(t);
                  m_rows[base + i].family     = (short)f;
                  m_rows[base + i].pattern    = (short)p;
                  m_rows[base + i].side       = (short)(rows[i].direction == "Buy" ? 1 : -1);
                  m_rows[base + i].netVote    = rows[i].netVote;
                  m_rows[base + i].entryPrice = rows[i].entryPrice;
                  m_rows[base + i].closed     = (rows[i].result != "NA");
                  m_rows[base + i].win        = (rows[i].result == "Profit");
                 }
               m_count += ArraySize(rows);
              }
      if(missing > 0)
         PrintVerbose(__FUNCTION__ + ": " + IntegerToString(missing) +
                      " pattern table(s) absent (family disabled in the populating run) - skipped.");
      return m_count;
     }

   //--- Corpus state, printed for the log: volume per family, closed fraction, span, and the
   //--- GMT->server-bar offset measurement S2's label plumbing will pin itself to.
   void              Report(void)
     {
      if(m_count == 0)
        {
         Print("MetaCorpus: EMPTY - run a backtest with UseDatabaseRanking=true (and "
               "DB_MaxRowsPerTable raised for a long window) to build the corpus.");
         return;
        }
      datetime lo = m_rows[0].gmt, hi = m_rows[0].gmt;
      int famRows[4] = {0, 0, 0, 0}, famClosed[4] = {0, 0, 0, 0}, famWins[4] = {0, 0, 0, 0};
      for(int i = 0; i < m_count; i++)
        {
         lo = MathMin(lo, m_rows[i].gmt);
         hi = MathMax(hi, m_rows[i].gmt);
         famRows[m_rows[i].family]++;
         if(m_rows[i].closed)
           {
            famClosed[m_rows[i].family]++;
            if(m_rows[i].win)
               famWins[m_rows[i].family]++;
           }
        }
      Print("MetaCorpus: ", m_count, " candidates, ", TimeToString(lo, TIME_DATE), " .. ",
            TimeToString(hi, TIME_DATE));
      //--- The cross-run trap, caught loudly: ProcessSignal's outdated-row guard rejects any
      //--- registration OLDER than a row its table already holds (correct for a live stream,
      //--- append-only-forward across runs). A tester run that starts before the DB's newest row
      //--- will therefore silently register NOTHING for the overlap - an 18-year corpus build
      //--- against leftover recent rows yields a corpus of only the leftover (measured 2026-08-12:
      //--- 3,681 rows, all 2026, from exactly that mistake). Corpus builds start from an empty DB.
      if(MQLInfoInteger(MQL_TESTER) && hi > TimeCurrent())
         Print("MetaCorpus: WARNING - the DB already holds rows newer than this test's start (",
               TimeToString(hi, TIME_DATE), " > ", TimeToString(TimeCurrent(), TIME_DATE),
               "). Historical registrations will be REJECTED by the outdated-row guard; ",
               "wipe the signal DB before a corpus-building backtest.");
      for(int f = 0; f < 4; f++)
        {
         if(famRows[f] == 0)
            continue;
         double wr = famClosed[f] > 0 ? 100.0 * famWins[f] / famClosed[f] : 0.0;
         Print("  ", MetaFamilyName(f), ": ", famRows[f], " rows, ", famClosed[f],
               " closed, S&R win rate ", DoubleToString(wr, 1), "%");
        }
      //--- The offset measurement. A candidate is only labelable if its timestamp resolves to an
      //--- exact bar; rows are journaled at bar opens, so the RIGHT offset should match ~100% and
      //--- wrong ones near-0% on an H1 chart - a clean empirical signature. Measured on every row.
      Print("  GMT->server bar match rate by hour offset (S2 pins to the winner):");
      for(int off = 0; off <= 3; off++)
        {
         int hit = 0;
         for(int i = 0; i < m_count; i++)
            if(iBarShift(_Symbol, _Period, m_rows[i].gmt + off * 3600, true) >= 0)
               hit++;
         Print("    +", off, "h: ", DoubleToString(100.0 * hit / m_count, 1), "% exact");
        }
     }
  };

//--- OnInit hook: load + report under VerboseMode, so a corpus-building backtest is verifiable
//--- straight from its log. Deliberately report-only in S1 - nothing downstream consumes it yet.
void MetaCorpusReport(void)
  {
   CMetaCorpus corpus;
   corpus.Load();
   corpus.Report();
  }

//--- ALWAYS-ON tester guard, deliberately NOT VerboseMode-gated: the stale-DB warning inside
//--- Report() was, and forgetting the wipe silently wasted a full 18-year corpus run - twice.
//--- 52 one-row queries at OnInit; absent tables (family disabled, or a fresh wipe before
//--- AddFilter creates them) probe quietly as "no rows".
void MetaCorpusStaleCheck(void)
  {
   if(!MQLInfoInteger(MQL_TESTER))
      return;
   long newest = 0;
   bool any = false;
   for(int f = 0; f < 4; f++)
      for(int p = 0; p < MetaFamilyPatterns(f); p++)
         for(int s = 0; s < 2; s++)
           {
            string table = MetaFamilyName(f) + "_Pattern_" + IntegerToString(p) +
                           (s == 0 ? "_Buy" : "_Sell");
            long k = 0;
            bool found = false;
            if(dbm.FetchNewestTimeKey(table, k, found, true) && found)
              {
               newest = MathMax(newest, k);
               any = true;
              }
           }
   if(!any)
      return;
   MqlDateTime g;
   TimeGMT(g);
   long nowKey = SignalTimeKey(g.year, g.mon, g.day, g.hour, g.min);
   if(newest >= nowKey)
      Print("MetaCorpus: *** THE SIGNAL DB ALREADY HOLDS ROWS AT/AFTER THIS TEST'S START (key ",
            newest, " >= ", nowKey, "). Every registration in the overlap will be REJECTED by the ",
            "outdated-row guard and this run will build NO corpus. Stop the test, wipe the ",
            "Signals folder, and rerun. ***");
  }
//+------------------------------------------------------------------+