Warrior_EA/Database/WarriorJournal.mqh

//+------------------------------------------------------------------+
//|                                                 WarriorJournal.mqh |
//|                                                     AnimateDread |
//|                                                                  |
//| THE QUEUE. Every filter evaluation is recorded during the vote    |
//| and written to SQLite on the timer, in one transaction.           |
//|                                                                   |
//| THE SHAPE IS THE ORIGINAL ONE and it was right the first time:    |
//| collect during Direction() because that is the only place that    |
//| knows what each filter said, and FLUSH ON THE TIMER because a     |
//| write per evaluation would put SQLite in the tick path. What this |
//| version fixes is the two things the 2025 import got wrong - an    |
//| O(n^2) sort over a bogus date key, and reading whole tables back  |
//| per buffered row - both already solved upstream in 6819bb4 and    |
//| ea2552e. Nothing here re-imports them.                            |
//|                                                                   |
//| THE CONNECTION IS OPENED BY THE TIMER AND CLOSED BY THE TIMER,    |
//| and during a backtest it is not closed at all. That is not an     |
//| optimisation detail, it is the reason this can run in the tester: |
//| open/close per flush is what made it too slow to leave on, and    |
//| "too slow to leave on" is why it was disabled in backtests for    |
//| weeks - which is why no ranking corpus existed to rank from.      |
//+------------------------------------------------------------------+
#ifndef WARRIOR_SIMPLE_JOURNAL_MQH
#define WARRIOR_SIMPLE_JOURNAL_MQH

#include "DatabaseManager.mqh"

//--- One row per filter evaluation. Deliberately small: this is written on every bar by every
//--- module, so a wide struct here is a cost paid millions of times in a decade-long pass.
struct SWarriorFiring
  {
   datetime          when;
   string            table;       // "<filterID>_<pattern>_<Buy|Sell>"
   string            pattern;
   string            direction;
   double            price;
   double            vote;        // this filter's signed contribution, before normalisation
   string            result;      // "Profit" | "Loss" - only resolved firings are ever written
   double            maeR;        // worst excursion against the firing, in units of the stop
   double            mfeR;        // best excursion in its favour, same units
   int               barsHeld;
   double            rMultiple;   // THE OUTCOME THAT MATTERS: +target/stop on a target, -1 on a stop
   //--- WHAT ELSE WAS SAYING THE SAME THING. Stamped after the vote closes, not at Record() time.
   int               agree;
   int               oppose;
   double            barVote;     // the ensemble's normalised result for that bar
   int               regime;      // 0 chop / 1 trend / 2 mean-revert, AT THE FIRING
  };

//--- A FIRING WAITING FOR ITS ANSWER.
//---
//--- A row that records only that a module SPOKE cannot rank anything - which is exactly why
//--- ApplyPatternWeight() sat in this repo with no caller: FetchWinLossCounts() counts
//--- result='Profit' against result='Loss', and nothing ever wrote either. So every firing becomes
//--- a virtual trade and is held here until the market answers it.
//---
//--- THE BARRIERS ARE THE EA'S OWN. Entry, stop and target come from the vote, so a pattern is
//--- scored on the trade it actually implies rather than on some separate notion of "right". A
//--- module whose signal is real but smaller than the stop it would trade behind SHOULD rank badly.
struct SWarriorPending
  {
   datetime          when;
   string            table;
   string            pattern;
   string            direction;   // "Buy" or "Sell"; anything else is never resolved
   double            entry;
   double            stopDist;    // > 0
   double            targetDist;  // > 0
   double            vote;
   int               agree;       // filters voting the SAME way on this bar
   int               oppose;      // filters voting the other way
   double            barVote;     // the ensemble's normalised result
   int               regime;
   double            mae;         // in price, against the firing
   double            mfe;         // in price, in its favour
   int               bars;
  };

//--- The schema. Columns are positional on read, so a new one goes at the END or every existing
//--- database silently re-maps. Broker time, not GMT: rows written live and rows written by a
//--- backfill landed hours apart in the same column when these disagreed.
#define WARRIOR_FIRING_SCHEMA "ts INTEGER, year INTEGER, month INTEGER, day INTEGER, " \
                              "dayOfWeek INTEGER, hour INTEGER, minutes INTEGER, "     \
                              "pattern TEXT, direction TEXT, price REAL, vote REAL, "  \
                              "result TEXT, maeR REAL, mfeR REAL, barsHeld INTEGER, "        \
                              "rMultiple REAL, agree INTEGER, oppose INTEGER, barVote REAL, regime INTEGER"

//--- HOW LONG A FIRING MAY WAIT FOR AN ANSWER, in bars. A firing that has hit neither barrier by
//--- then is DISCARDED rather than scored: it has no outcome, and inventing one - "positive at
//--- the horizon" - would be a second, different definition of winning mixed into one column.
//--- This biases the corpus toward firings that resolve quickly, so a pattern that is right
//--- slowly looks like a pattern with no data. That is the honest cost to pay: the alternative
//--- biases the WIN RATE itself rather than the sample.
#define WARRIOR_RESOLVE_BARS 60

class CWarriorJournal
  {
private:
   CDatabaseManager *m_dbm;
   SWarriorFiring    m_buf[];
   SWarriorPending   m_pend[];
   int               m_resolved;
   int               m_expired;
   bool              m_backtest;
   int               m_written;
   int               m_dropped;

   //--- Table names reach SQL, so they are built from an allow-list rather than interpolated.
   //--- A filter id is author-controlled today; that is not a reason to make it a SQL injection
   //--- surface tomorrow.
   bool              SafeName(const string s) const
     {
      const int n = StringLen(s);
      if(n <= 0 || n > 64)
         return false;
      for(int i = 0; i < n; i++)
        {
         const ushort c = StringGetCharacter(s, i);
         const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
                         (c >= '0' && c <= '9') || c == '_';
         if(!ok)
            return false;
        }
      return true;
     }

public:
                     CWarriorJournal(void) : m_dbm(NULL), m_backtest(false),
                                             m_written(0), m_dropped(0),
                                             m_resolved(0), m_expired(0) {}
                    ~CWarriorJournal(void) {}

   void              Bind(CDatabaseManager *dbm, const bool isBacktest)
                       { m_dbm = dbm; m_backtest = isBacktest; }
   int               Pending(void) const { return ArraySize(m_buf); }
   int               Written(void) const { return m_written; }

   void              Record(const string filterID, const string pattern, const string direction,
                            const double price, const double vote,
                            const double stopDist, const double targetDist);
   void              AdvanceBar(const double high, const double low);
   //--- STAMP THE BAR'S CONFLUENCE onto every firing opened on it. Called once, AFTER the vote
   //--- loop closes, because "how many agreed" is not knowable while the loop is still asking.
   //---
   //--- This is the input the networks in this project have never been given, and the one the
   //--- research says carries the edge: one trigger alone measured -2.15 bp, three or more
   //--- agreeing +1.39 bp out of sample, and opposed -5.80. A journal that records WHAT fired but
   //--- not WHAT ELSE was firing cannot answer the only question worth asking of it.
   void              StampBar(const datetime bar, const int longVotes, const int shortVotes,
                              const double barVote, const int regime)
     {
      const int n = ArraySize(m_pend);
      for(int i = 0; i < n; i++)
        {
         if(m_pend[i].when != bar || m_pend[i].bars > 0)
            continue;                              // only this bar's still-unscored firings
         const bool isLong = (m_pend[i].direction == "Buy");
         m_pend[i].agree   = isLong ? longVotes  : shortVotes;
         m_pend[i].oppose  = isLong ? shortVotes : longVotes;
         m_pend[i].barVote = barVote;
         m_pend[i].regime  = regime;
        }
     }
   bool              Flush(void);
   void              Report(void) const
     {
      //--- RESOLVED AND EXPIRED ARE BOTH REPORTED. A corpus that looks thin can be thin for two
      //--- opposite reasons - few firings, or firings that never answer inside the horizon - and
      //--- the ranking layer behaves completely differently in each case.
      PrintFormat("CWarriorJournal: %d row(s) written, %d resolved, %d expired unanswered, "
                  "%d refused (unsafe name), %d queued, %d still open.",
                  m_written, m_resolved, m_expired, m_dropped, ArraySize(m_buf), ArraySize(m_pend));
     }
  };
//+------------------------------------------------------------------+
void CWarriorJournal::Record(const string filterID, const string pattern, const string direction,
                             const double price, const double vote,
                             const double stopDist, const double targetDist)
  {
   if(m_dbm == NULL || direction == "")
      return;
   if(!SafeName(filterID) || !SafeName(pattern))
     {
      m_dropped++;
      return;
     }
   //--- UNRESOLVABLE FIRINGS ARE NOT RECORDED AT ALL. Without a side there is no barrier to cross,
   //--- and without a positive stop the R units below are a division by nothing. A row that can
   //--- never carry an outcome is not a cheap row, it is a permanent 'NA' diluting every count.
   if((direction != "Buy" && direction != "Sell") || stopDist <= 0.0 || targetDist <= 0.0 ||
      price <= 0.0)
      return;
   const string table = filterID + "_" + pattern + "_" + direction;
   const datetime now = TimeCurrent();
   const int n = ArraySize(m_pend);
   //--- IDENTITY DEDUP, same rule as the original: one row per table per bar. Direction() runs on
   //--- every tick, and without this a quiet bar with 400 ticks writes 400 identical rows and the
   //--- win rate becomes a tick-count weighting.
   for(int i = 0; i < n; i++)
      if(m_pend[i].table == table && m_pend[i].when == now)
         return;
   ArrayResize(m_pend, n + 1);
   m_pend[n].when       = now;
   m_pend[n].table      = table;
   m_pend[n].pattern    = pattern;
   m_pend[n].direction  = direction;
   m_pend[n].entry      = price;
   m_pend[n].stopDist   = stopDist;
   m_pend[n].targetDist = targetDist;
   m_pend[n].vote       = vote;
   m_pend[n].agree      = 0;
   m_pend[n].oppose     = 0;
   m_pend[n].barVote    = 0.0;
   m_pend[n].regime     = 0;
   m_pend[n].mae        = 0.0;
   m_pend[n].mfe        = 0.0;
   //--- -1 = CREATED ON THIS BAR, NOT YET LIVE. Direction() calls Record() and then, in the same
   //--- call, AdvanceBar(High(1), Low(1)) - and High(1)/Low(1) at that moment are the bar that
   //--- just CLOSED, the signal bar, which ended before this entry existed. A pending at 0 would be
   //--- credited with that bar's whole range as its first excursion. Measured 2026-09-12: at a
   //--- crowd-long signal the bar rose into its close, so its high is near the entry (~0 traded MFE)
   //--- and its low is far below (large traded MAE) - every traded-side loss was overstated and the
   //--- crowd fade priced at +0.31R on bars where the live trade made +0.03R. The first AdvanceBar
   //--- after creation only flips this to 0; the next one applies the first REAL bar.
   m_pend[n].bars       = -1;
  }
//+------------------------------------------------------------------+
//| ONE BAR OF THE MARKET'S ANSWER, applied to every open firing.     |
//|                                                                   |
//| Called once per bar by the vote with the CLOSED bar's range. Both |
//| barriers are tested against the same bar, and when a bar spans    |
//| both this scores it a LOSS - the pessimistic reading. Bar data    |
//| cannot say which came first, and a scorer that resolved ties in   |
//| its own favour would report a win rate no live account could      |
//| reproduce. Every stop-and-target study in this repo that looked   |
//| too good was this assumption made the other way.                  |
//+------------------------------------------------------------------+
void CWarriorJournal::AdvanceBar(const double high, const double low)
  {
   if(high <= 0.0 || low <= 0.0 || high < low)
      return;
   int n = ArraySize(m_pend);
   for(int i = n - 1; i >= 0; i--)
     {
      if(m_pend[i].bars < 0)
        { m_pend[i].bars = 0; continue; }         // born this bar: the range passed in predates it
      const bool   isLong = (m_pend[i].direction == "Buy");
      const double entry  = m_pend[i].entry;
      //--- Excursions, signed so that "favour" means the same thing on both sides.
      const double fav    = isLong ? (high - entry) : (entry - low);
      const double adv    = isLong ? (entry - low)  : (high - entry);
      if(fav > m_pend[i].mfe) m_pend[i].mfe = fav;
      if(adv > m_pend[i].mae) m_pend[i].mae = adv;
      m_pend[i].bars++;

      const bool hitStop   = (m_pend[i].mae >= m_pend[i].stopDist);
      const bool hitTarget = (m_pend[i].mfe >= m_pend[i].targetDist);
      string result = "";
      double rMult = 0.0;
      //--- THE R-MULTIPLE IS THE POINT, and the reason both barriers are carried per firing.
      //--- A binary win/loss on a SYMMETRIC barrier makes expectancy identical to win rate by
      //--- construction, which designs out the pattern worth finding: one that is right 30% of the
      //--- time and wins three times what it loses. Scored against the trade's own stop and target,
      //--- a target is +target/stop R and a stop is -1R, so mean R ranks the two correctly.
      if(hitStop)
        { result = "Loss";   rMult = -1.0; }              // pessimistic on a both-touched bar
      else if(hitTarget)
        { result = "Profit"; rMult = m_pend[i].targetDist / m_pend[i].stopDist; }
      else if(m_pend[i].bars >= WARRIOR_RESOLVE_BARS)
        {
         //--- Expired with no answer. Dropped, not scored - see WARRIOR_RESOLVE_BARS.
         m_expired++;
         ArrayRemove(m_pend, i, 1);
         continue;
        }
      if(result == "")
         continue;
      const int w = ArraySize(m_buf);
      ArrayResize(m_buf, w + 1);
      m_buf[w].when      = m_pend[i].when;
      m_buf[w].table     = m_pend[i].table;
      m_buf[w].pattern   = m_pend[i].pattern;
      m_buf[w].direction = m_pend[i].direction;
      m_buf[w].price     = m_pend[i].entry;
      m_buf[w].vote      = m_pend[i].vote;
      m_buf[w].result    = result;
      m_buf[w].maeR      = m_pend[i].mae / m_pend[i].stopDist;
      m_buf[w].mfeR      = m_pend[i].mfe / m_pend[i].stopDist;
      m_buf[w].barsHeld  = m_pend[i].bars;
      m_buf[w].rMultiple = rMult;
      m_buf[w].agree     = m_pend[i].agree;
      m_buf[w].oppose    = m_pend[i].oppose;
      m_buf[w].barVote   = m_pend[i].barVote;
      m_buf[w].regime    = m_pend[i].regime;
      m_resolved++;
      ArrayRemove(m_pend, i, 1);
     }
  }
//+------------------------------------------------------------------+
bool CWarriorJournal::Flush(void)
  {
   const int n = ArraySize(m_buf);
   if(m_dbm == NULL || n <= 0)
      return true;
   if(!m_dbm.BeginTransaction())
     {
      //--- LEFT QUEUED, NOT DROPPED. A failed flush that discarded its buffer would silently thin
      //--- the corpus exactly when the database is under stress - and the gap would look like a
      //--- quiet market rather than a failed write.
      Print("CWarriorJournal: could not begin a transaction; ", n, " row(s) left queued for retry.");
      return false;
     }
   string cols[] = {"ts", "year", "month", "day", "dayOfWeek", "hour", "minutes",
                    "pattern", "direction", "price", "vote",
                    "result", "maeR", "mfeR", "barsHeld", "rMultiple",
                    "agree", "oppose", "barVote", "regime"};
   for(int i = 0; i < n; i++)
     {
      MqlDateTime t;
      TimeToStruct(m_buf[i].when, t);
      m_dbm.CreateTable(m_buf[i].table, WARRIOR_FIRING_SCHEMA);   // no-op when it already exists
      string vals[];
      ArrayResize(vals, 20);
      vals[0] = IntegerToString((long)m_buf[i].when);
      vals[1] = IntegerToString(t.year);
      vals[2] = IntegerToString(t.mon);
      vals[3] = IntegerToString(t.day);
      vals[4] = IntegerToString(t.day_of_week);
      vals[5] = IntegerToString(t.hour);
      vals[6] = IntegerToString(t.min);
      //--- NO QUOTES. InsertTradeRecord() binds these as parameters (PlaceholderList +
      //--- PrepareAndExecuteBound), so a manual "'" makes the apostrophes part of the VALUE:
      //--- every row since this was written stored 'Pattern_0' rather than Pattern_0. It went
      //--- unnoticed because the table NAME carries the real key and nothing ever read these two
      //--- columns back - until `result` joined them and FetchWinLossCounts' result='Profit'
      //--- matched nothing, leaving every weight unset and the whole ranking pass silent.
      vals[7] = m_buf[i].pattern;
      vals[8] = m_buf[i].direction;
      vals[9] = DoubleToString(m_buf[i].price, 8);
      vals[10] = DoubleToString(m_buf[i].vote, 4);
      vals[11] = m_buf[i].result;
      vals[12] = DoubleToString(m_buf[i].maeR, 4);
      vals[13] = DoubleToString(m_buf[i].mfeR, 4);
      vals[14] = IntegerToString(m_buf[i].barsHeld);
      vals[15] = DoubleToString(m_buf[i].rMultiple, 4);
      vals[16] = IntegerToString(m_buf[i].agree);
      vals[17] = IntegerToString(m_buf[i].oppose);
      vals[18] = DoubleToString(m_buf[i].barVote, 4);
      vals[19] = IntegerToString(m_buf[i].regime);
      if(m_dbm.InsertTradeRecord(m_buf[i].table, cols, vals))
         m_written++;
     }
   if(!m_dbm.CommitTransaction())
     {
      m_dbm.RollbackTransaction();
      Print("CWarriorJournal: commit failed, rolled back; ", n, " row(s) left queued for retry.");
      return false;
     }
   ArrayResize(m_buf, 0);
   return true;
  }
#endif // WARRIOR_SIMPLE_JOURNAL_MQH