Warrior_EA/System/AltData.mqh

//+------------------------------------------------------------------+
//|                                                      AltData.mqh |
//|                                                     AnimateDread |
//+------------------------------------------------------------------+
//| Externally collected, publication-stamped feature panel.          |
//|                                                                  |
//| research/altdata (Python) collects sources that are NOT derivable |
//| from any chart series - CFTC COT positioning, the VIX complex,    |
//| macro series - stamps every row with the date it became PUBLICLY  |
//| KNOWABLE, as-of joins them onto calendar days, and exports        |
//| Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv:                |
//|                                                                  |
//|     date;vix_chg5;vix;usd_chg5;cot_spec_net                      |
//|     2010.07.24;0.194000;0.239800;...                             |
//|                                                                  |
//| Only features that survived BOTH the family-wise MI bar AND the   |
//| incremental (conditional-on-trailing-range) test are exported, so |
//| the per-symbol sets differ - see research/altdata/DESIGN.md and   |
//| the screen results (commits 4a56d8d, 6d50e63).                    |
//|                                                                  |
//| LOOKAHEAD CONTRACT: a row dated D contains only values published  |
//| by D 00:00 UTC (the collectors' stamps carry their own >=1-day    |
//| conservative buffers on top - COT: Tuesday report stamped to the  |
//| Saturday after its Friday release). Features(t) returns the last  |
//| row dated <= t, so a bar can never read a value the live run      |
//| would not have had. The same files serve training, the tester and |
//| live - history is reproducible by construction.                   |
//|                                                                  |
//| DEGRADATION CONTRACT (mirrors CCrossAssetPanel): a missing file,  |
//| a gap, a pinned column the file no longer carries - all 0-fill.   |
//| The block is additive context; it must never reject a bar or      |
//| block a trade. The EA runs without Python at all times: these are |
//| plain files, refreshed externally, stale-tolerated here.          |
//|                                                                  |
//| PINNING (mirrors m_crossAssetPairsPinned): the trained model's    |
//| .cfg records the NAME LIST its inputs were built from. On resume  |
//| the pin is applied BEFORE the width sum, so a file that gained    |
//| columns neither shifts existing slots nor changes the width -     |
//| pinned columns are looked up by NAME in the current file.         |
//+------------------------------------------------------------------+
//--- 2026-08-16: raised 16 -> 32. The wire-everything policy put FX symbols at 15
//--- features; at 16 the NEXT added column would have been silently truncated by the
//--- MathMin below - exactly the class of quiet degradation this module must never do.
#define ALTDATA_MAX_FEATURES  32
#define ALTDATA_MAX_PIN_CHARS 1024

//--- Symbol -> safe file-name token, shared by every consumer that builds a path from a
//--- broker symbol (feature files, GEX recorder, TunedPeriods). Broker names can carry
//--- path-hostile characters ("EUR/USD", "#AAPL" variants, exotic suffixes); writing
//--- through them either fails or lands in an unintended subfolder. Deterministic and
//--- collision-tolerant: two symbols differing only in a hostile character map together,
//--- which is acceptable (they would be the same instrument at any sane broker).
string AltDataFileSymbol(string symbol)
  {
   string out = symbol;
   StringReplace(out, "/", "-");
   StringReplace(out, "\\", "-");
   StringReplace(out, ":", "-");
   StringReplace(out, "*", "-");
   StringReplace(out, "?", "-");
   StringReplace(out, "\"", "-");
   StringReplace(out, "<", "-");
   StringReplace(out, ">", "-");
   StringReplace(out, "|", "-");
   return out;
  }

class CAltDataPanel
  {
protected:
   datetime          m_rowTime[];     // row date (server-naive 00:00), ascending
   double            m_vals[];        // rows x m_fileCount, EMPTY_VALUE = blank cell
   int               m_rows;
   int               m_fileCount;     // columns in the loaded file
   //--- Per-file-column MEDIAN over the covered range, used to fill bars the file cannot answer for.
   //--- WHY NOT ZERO, which is what this did until 2026-08-17. Half these columns are LEVELS - vix,
   //--- ivol, mac_y10, mac_cpi, mac_unemp, eia_util - and for a level, 0 is not "no information", it
   //--- is an impossible reading far outside the series' real range (VIX does not visit zero). The
   //--- 0-fill therefore did not degrade the block, it injected a spike at an extreme value.
   //---
   //--- And the spike is not randomly placed: every alt file starts 2010-01-01 while the charts run
   //--- much deeper (USDJPY H4 reaches ~1994, roughly HALF its history), so "alt block is all zeros"
   //--- is exactly the predicate "this bar is older than 2010". The IS/OOS split is chronological, so
   //--- that predicate is ~half of IS and none of OOS: a feature the model can learn in-sample that
   //--- is guaranteed to be useless out-of-sample, plus a bimodal input for the first BatchNorm to
   //--- normalise over. Not a lookahead leak - a distribution corruption, which is quieter.
   //---
   //--- A CONSTANT cannot leak, whatever its source: it takes the same value on every pre-coverage
   //--- bar, so it carries no information about which of those bars won. That is what makes filling
   //--- with a median computed over the covered (later) range legitimate here even though the median
   //--- is "future" relative to the bars it fills - it is one number, not a per-bar signal. The
   //--- median rather than the mean because these series are skewed and a mean would sit off-centre.
   double            m_median[];
   string            m_fileNames[];   // file column names, file order
   string            m_pinnedNames[]; // pinned name list ("" = none): the OUTPUT contract
   int               m_pinnedCount;
   int               m_map[];         // output slot -> file column (-1 = 0-fill)
   string            m_loadedFile;
   string            m_loadedSymbol;          // what Load() was CALLED with - reload uses these
   ENUM_TIMEFRAMES   m_loadedPeriod;          // verbatim, never re-derived from the file path
   datetime          m_lastLoadTry;
   bool              m_loggedMissing;

   void              BuildMap(void)
     {
      int outCount = FeatureCount();
      ArrayResize(m_map, outCount);
      for(int j = 0; j < outCount; j++)
        {
         string want = (m_pinnedCount > 0) ? m_pinnedNames[j] : m_fileNames[j];
         m_map[j] = -1;
         for(int c = 0; c < m_fileCount; c++)
            if(m_fileNames[c] == want)
              {
               m_map[j] = c;
               break;
              }
         if(m_map[j] < 0 && m_pinnedCount > 0)
            PrintFormat("AltData: pinned feature '%s' not present in %s - that input 0-fills "
                        "until the export carries it again.", want, m_loadedFile);
        }
     }

public:
                     CAltDataPanel(void) : m_rows(0), m_fileCount(0), m_pinnedCount(0),
                                           m_loadedFile(""), m_loadedSymbol(""), m_loadedPeriod(PERIOD_CURRENT),
                                           m_lastLoadTry(0), m_loggedMissing(false) {}

   bool              IsLoaded(void)     const { return m_rows > 0; }
   datetime          LastDate(void)     const { return m_rows > 0 ? m_rowTime[m_rows - 1] : 0; }
   //--- The width contract: pinned list is the authority once set; otherwise the file decides.
   int               FeatureCount(void) const { return m_pinnedCount > 0 ? m_pinnedCount : m_fileCount; }

   string            NamesCsv(void) const
     {
      string csv = "";
      int n = FeatureCount();
      for(int j = 0; j < n; j++)
         csv += (j > 0 ? "," : "") + (m_pinnedCount > 0 ? m_pinnedNames[j] : m_fileNames[j]);
      return csv;
     }

   void              SetPinnedNames(string csv)
     {
      m_pinnedCount = (csv == "") ? 0 : StringSplit(csv, ',', m_pinnedNames);
      if(m_pinnedCount > ALTDATA_MAX_FEATURES)
        {
         PrintFormat("AltData: pinned list of %d exceeds the %d-feature cap - truncating. "
                     "This should never happen; check the .cfg.", m_pinnedCount, ALTDATA_MAX_FEATURES);
         m_pinnedCount = ALTDATA_MAX_FEATURES;
        }
      if(m_fileCount > 0)
         BuildMap();
     }

   //--- Parse Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv, FALLING BACK to the
   //--- {SYMBOL}_D1.csv the fetcher maintains. The daily file is timeframe-agnostic by
   //--- construction: rows are as-of daily values joined by published <= bar open, and that
   //--- join is exactly what Features() does per bar - an H4 bar simply reads the same
   //--- daily row its calendar day maps to. (The 2026-08-16 H4 attach trained with ZERO
   //--- alt features because this fallback did not exist and only _D1 files are written.)
   //--- A per-TF file, if some future exporter writes one, still takes precedence.
   //--- Missing BOTH is a NORMAL state (symbol with no catalog row, collector not yet run):
   //--- the panel just reports 0 features / keeps its pinned width and 0-fills.
   bool              Load(string symbol, ENUM_TIMEFRAMES period)
     {
      m_lastLoadTry = TimeCurrent();
      string tf = StringSubstr(EnumToString(period), 7); // PERIOD_D1 -> D1
      string fileSym = AltDataFileSymbol(symbol);
      string name = "Warrior_EA\\AltData\\" + fileSym + "_" + tf + ".csv";
      if(!FileIsExist(name, FILE_COMMON) && tf != "D1")
        {
         string daily = "Warrior_EA\\AltData\\" + fileSym + "_D1.csv";
         if(FileIsExist(daily, FILE_COMMON))
           {
            PrintFormat("AltData: no %s - using the daily file %s (as-of join makes daily "
                        "alt rows valid on any timeframe).", name, daily);
            name = daily;
           }
        }
      if(!FileIsExist(name, FILE_COMMON))
        {
         if(!m_loggedMissing)
           {
            m_loggedMissing = true;
            PrintFormat("AltData: no %s - external block %s.", name,
                        m_pinnedCount > 0 ? "0-fills its pinned width" : "disabled (0 features)");
           }
         return false;
        }
      //--- share flags unconditionally: the exporter may rewrite the file while charts hold it open
      //--- (and the tester agent holds files, see the FileCopy share-flag bug).
      int h = FileOpen(name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltData: %s exists but FileOpen failed (%d).", name, GetLastError());
         return false;
        }
      string header = FileReadString(h);
      string cols[];
      int nCols = StringSplit(header, ';', cols);
      if(nCols < 2 || cols[0] != "date")
        {
         PrintFormat("AltData: %s header unrecognised ('%s') - ignoring file.", name, header);
         FileClose(h);
         return false;
        }
      int fileCount = MathMin(nCols - 1, ALTDATA_MAX_FEATURES);
      ArrayResize(m_fileNames, fileCount);
      for(int c = 0; c < fileCount; c++)
         m_fileNames[c] = cols[c + 1];
      int cap = 8192, rows = 0;
      ArrayResize(m_rowTime, cap);
      ArrayResize(m_vals, cap * fileCount);
      datetime prev = 0;
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringLen(line) < 11)
            continue;
         string f[];
         if(StringSplit(line, ';', f) < fileCount + 1)
            continue;
         datetime t = StringToTime(f[0]);
         if(t <= 0 || t <= prev)   // strictly ascending or the binary search breaks
            continue;
         if(rows >= cap)
           {
            cap = cap * 3 / 2;
            ArrayResize(m_rowTime, cap);
            ArrayResize(m_vals, cap * fileCount);
           }
         m_rowTime[rows] = t;
         for(int c = 0; c < fileCount; c++)
           {
            string cell = f[c + 1];
            m_vals[rows * fileCount + c] = (StringLen(cell) == 0) ? EMPTY_VALUE : StringToDouble(cell);
           }
         prev = t;
         rows++;
        }
      FileClose(h);
      if(rows == 0)
        {
         PrintFormat("AltData: %s parsed to 0 rows - ignoring file.", name);
         return false;
        }
      m_rows = rows;
      m_fileCount = fileCount;
      m_loadedFile = name;
      m_loadedSymbol = symbol;
      m_loadedPeriod = period;
      ArrayResize(m_rowTime, rows);
      ArrayResize(m_vals, rows * fileCount);
      //--- Column medians over the covered range, ignoring blank cells - see m_median. Cheap: a few
      //--- thousand rows x ~15 columns, once per file load.
      ArrayResize(m_median, fileCount);
      ArrayInitialize(m_median, 0.0);
      int blanks = 0;
      for(int c = 0; c < fileCount; c++)
        {
         double col[];
         ArrayResize(col, rows);
         int m = 0;
         for(int rr = 0; rr < rows; rr++)
           {
            double v = m_vals[rr * fileCount + c];
            if(v != EMPTY_VALUE)
               col[m++] = v;
            else
               blanks++;
           }
         if(m <= 0)
            continue;
         ArrayResize(col, m);
         ArraySort(col);
         m_median[c] = (m % 2 == 1) ? col[m / 2] : 0.5 * (col[m / 2 - 1] + col[m / 2]);
        }
      if(blanks > 0)
         PrintFormat("AltData: %s has %d blank cells across %d rows x %d columns - these read as the"
                     " column MEDIAN, not as 0 (see m_median). A level series reported as 0 is an"
                     " impossible value, not a missing one.", name, blanks, rows, fileCount);
      BuildMap();
      PrintFormat("AltData: %s - %d features [%s], %d daily rows %s -> %s%s.",
                  name, FeatureCount(), NamesCsv(), rows,
                  TimeToString(m_rowTime[0], TIME_DATE), TimeToString(LastDate(), TIME_DATE),
                  m_pinnedCount > 0 ? " (pinned name list applied)" : "");
      return true;
     }

   //--- As-of read: the last row dated <= t, mapped through the pin. Always fills FeatureCount()
   //--- slots; anything unavailable is 0.0 (the neutral the sanitize loop would produce anyway).
   void              Features(datetime t, double &out[])
     {
      int n = FeatureCount();
      ArrayResize(out, n);
      ArrayInitialize(out, 0.0);
      if(m_rows == 0 || n == 0)
         return;
      //--- BEFORE THE FILE'S FIRST ROW: median-fill, do not zero-fill. See m_median for why a zero here
      //--- was a spike at an impossible value that coincided exactly with the chronological IS/OOS
      //--- split. Still never rejects the bar - the block stays additive context, as designed.
      if(t < m_rowTime[0])
        {
         for(int j = 0; j < n; j++)
           {
            int c = m_map[j];
            if(c >= 0 && c < ArraySize(m_median))
               out[j] = m_median[c];
           }
         return;
        }
      int lo = 0, hi = m_rows - 1;
      while(lo < hi)
        {
         int mid = (lo + hi + 1) / 2;
         if(m_rowTime[mid] <= t)
            lo = mid;
         else
            hi = mid - 1;
        }
      int base = lo * m_fileCount;
      for(int j = 0; j < n; j++)
        {
         int c = m_map[j];
         if(c < 0)
            continue;
         double v = m_vals[base + c];
         //--- A BLANK CELL is a gap in the collector's series, not a reading of zero - measured
         //--- 2026-08-17: eia_stk_idx1y alone has 181 blanks in 6,073 rows. Same argument as the
         //--- pre-coverage fill above, so the same constant.
         out[j] = (v != EMPTY_VALUE) ? v
                  : ((c < ArraySize(m_median)) ? m_median[c] : 0.0);
        }
     }

   //--- Live freshness: the exporter rewrites the file on its own schedule; reload when the chart
   //--- has moved past the loaded data, at most once an hour. In the tester the newest bar is
   //--- historical and always <= LastDate(), so this never fires there - by design.
   void              EnsureFresh(datetime newestBar)
     {
      if(m_rows == 0 || newestBar <= LastDate() + 86400)
         return;
      if(TimeCurrent() - m_lastLoadTry < 3600)
         return;
      string sym = m_loadedFile; // reload the exact file we came from
      if(sym == "")
         return;
      PrintFormat("AltData: newest bar %s is beyond loaded data (%s) - reloading %s.",
                  TimeToString(newestBar, TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_loadedFile);
      //--- Re-parse in place; a failed reload keeps the old (stale but valid) panel.
      m_lastLoadTry = TimeCurrent();
      int h = FileOpen(m_loadedFile, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return;
      FileClose(h);
      //--- Cheap existence probe passed; reload with the EXACT (symbol, period) Load() was
      //--- called with. The previous version re-derived them by splitting the file path on
      //--- its first '_' - which mis-parsed every symbol containing an underscore (OANDA's
      //--- EUR_USD, US_500 are in the alias catalog) and only knew three timeframes. Never
      //--- reconstruct what you already know.
      if(m_loadedSymbol != "")
         Load(m_loadedSymbol, m_loadedPeriod);
     }
  };
#undef ALTDATA_MAX_FEATURES