Warrior_EA/System/AltData.mqh

//+------------------------------------------------------------------+
//|                                                      AltData.mqh |
//|                                                     AnimateDread |
//| research/altdata (Python) collects sources that are NOT          |
//| derivable from any chart series - CFTC COT positioning, the VIX  |
//| complex, macro series - stamps every row with the date it became |
//| PUBLICLY KNOWABLE, as-of joins them onto calendar days, and      |
//| exports Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv:       |
//+------------------------------------------------------------------+
#ifndef WARRIOR_ALTDATA_MQH
#define WARRIOR_ALTDATA_MQH
#include <Math\Stat\Math.mqh>
//--- 2026-08-16: raised 16 -> 32. The wire-everything policy put FX symbols at 15
//--- features; at 16 the NEXT added column would have been silently truncated by the
//--- MathMin below - exactly the class of quiet degradation this module must never do.
#define ALTDATA_MAX_FEATURES  32
#define ALTDATA_MAX_PIN_CHARS 1024

//--- THE FLEET COLUMN SET, AND WHY IT IS AN INTERSECTION (2026-08-26).
//---
//--- Each exported {SYMBOL}_D1.csv carries whatever series apply to that instrument, so the fleet ran
//--- three different alt-data widths: FX 15 (COT triple), metals/oil 14 (implied-vol pair), SP500 13
//--- (cot_spec_net). The alt block's width joins the model fingerprint, and CTrainPoolReader only
//--- adopts peer rows whose fingerprint AND width match - so those three sets were three separate
//--- training pools. SP500 was alone in its own: "EVERY peer file was REJECTED, so this chart is
//--- training alone", 0 pooled rows, 2279 independent observations against a 600-wide input, the
//--- first layer floored at 16, and the EA's own capacity warning reading "expect overfitting". It
//--- was the one chart with no pool and the one chart being asked to be more precise.
//---
//--- INTERSECTION, NOT UNION, and the reason is not economy. A union would fill a series an
//--- instrument does not have with that column's median, making it CONSTANT for that instrument -
//--- so a pooled model could identify the source instrument from its alt block and would stop
//--- learning the shared mechanism, which is the entire point of pooling. The union is also 6
//--- columns wider, moving the capacity ratio the wrong way. The intersection wins on both axes.
//---
//--- COST, STATED PLAINLY: FX gives up cot_idx_1y/cot_idx_3y/cot_chg_4w, metals and oil give up
//--- ivol_chg5/ivol, SP500 gives up cot_spec_net. Whether any of those six carried retained
//--- information is NOT known - the keep-screen reports a column BITMASK, and no run has yet mapped
//--- its bits back to names. Do not read this constant as evidence they were worthless; it trades
//--- six unproven columns for a pool the starved chart has never had.
#define ALTDATA_FLEET_COLUMNS "vix_chg5,vix,usd_chg5,eia_stk_idx1y,eia_stk_chg4,eia_util,mac_y10,mac_curve,mac_bei,mac_gap,mac_cpi,mac_unemp"

//--- Symbol -> safe file-name token, shared by every consumer that builds a path from a broker
//--- symbol (feature files, GEX recorder, TunedPeriods).
string AltDataFileSymbol(string symbol)
  {
   string out = symbol;
   StringReplace(out, "/", "-");
   StringReplace(out, "\\", "-");
   StringReplace(out, ":", "-");
   StringReplace(out, "*", "-");
   StringReplace(out, "?", "-");
   StringReplace(out, "\"", "-");
   StringReplace(out, "<", "-");
   StringReplace(out, ">", "-");
   StringReplace(out, "|", "-");
   return out;
  }

class CAltDataPanel
  {
protected:
   datetime          m_rowTime[];     // row date (server-naive 00:00), ascending
   double            m_vals[];        // rows x m_fileCount, EMPTY_VALUE = blank cell
   int               m_rows;
   int               m_fileCount;     // columns in the loaded file
   //--- Per-file-column MEDIAN over the covered range, used to fill bars the file cannot answer
   //--- for. WHY NOT ZERO, which is what this did until 2026-08-17.
   double            m_median[];
   string            m_fileNames[];   // file column names, file order
   string            m_pinnedNames[]; // pinned name list ("" = none): the OUTPUT contract
   int               m_pinnedCount;
   int               m_map[];         // output slot -> file column (-1 = 0-fill)
   string            m_loadedFile;
   string            m_loadedSymbol;          // what Load() was CALLED with - reload uses these
   ENUM_TIMEFRAMES   m_loadedPeriod;          // verbatim, never re-derived from the file path
   datetime          m_lastLoadTry;
   bool              m_loggedMissing;

   void              BuildMap(void)
     {
      int outCount = FeatureCount();
      ArrayResize(m_map, outCount);
      for(int j = 0; j < outCount; j++)
        {
         string want = (m_pinnedCount > 0) ? m_pinnedNames[j] : m_fileNames[j];
         m_map[j] = -1;
         for(int c = 0; c < m_fileCount; c++)
            if(m_fileNames[c] == want)
              {
               m_map[j] = c;
               break;
              }
         if(m_map[j] < 0 && m_pinnedCount > 0)
            PrintFormat("AltData: pinned feature '%s' not present in %s - that input 0-fills "
                        "until the export carries it again.", want, m_loadedFile);
        }
     }

public:
                     CAltDataPanel(void) : m_rows(0), m_fileCount(0), m_pinnedCount(0),
                                           m_loadedFile(""), m_loadedSymbol(""), m_loadedPeriod(PERIOD_CURRENT),
                                           m_lastLoadTry(0), m_loggedMissing(false) {}

   datetime          LastDate(void)     const { return m_rows > 0 ? m_rowTime[m_rows - 1] : 0; }
   //--- The width contract: pinned list is the authority once set; otherwise the file decides.
   int               FeatureCount(void) const { return m_pinnedCount > 0 ? m_pinnedCount : m_fileCount; }

   string            NamesCsv(void) const
     {
      string csv = "";
      int n = FeatureCount();
      for(int j = 0; j < n; j++)
         csv += (j > 0 ? "," : "") + (m_pinnedCount > 0 ? m_pinnedNames[j] : m_fileNames[j]);
      return csv;
     }

   void              SetPinnedNames(string csv)
     {
      m_pinnedCount = (csv == "") ? 0 : StringSplit(csv, ',', m_pinnedNames);
      if(m_pinnedCount > ALTDATA_MAX_FEATURES)
        {
         PrintFormat("AltData: pinned list of %d exceeds the %d-feature cap - truncating. "
                     "This should never happen; check the .cfg.", m_pinnedCount, ALTDATA_MAX_FEATURES);
         m_pinnedCount = ALTDATA_MAX_FEATURES;
        }
      if(m_fileCount > 0)
         BuildMap();
     }

   //--- Parse Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv, FALLING BACK to the
   //--- {SYMBOL}_D1.csv the fetcher maintains.
   bool              Load(string symbol, ENUM_TIMEFRAMES period)
     {
      m_lastLoadTry = TimeCurrent();
      string tf = StringSubstr(EnumToString(period), 7); // PERIOD_D1 -> D1
      string fileSym = AltDataFileSymbol(symbol);
      string name = "Warrior_EA\\AltData\\" + fileSym + "_" + tf + ".csv";
      if(!FileIsExist(name, FILE_COMMON) && tf != "D1")
        {
         string daily = "Warrior_EA\\AltData\\" + fileSym + "_D1.csv";
         if(FileIsExist(daily, FILE_COMMON))
           {
            PrintFormat("AltData: no %s - using the daily file %s (as-of join makes daily "
                        "alt rows valid on any timeframe).", name, daily);
            name = daily;
           }
        }
      if(!FileIsExist(name, FILE_COMMON))
        {
         if(!m_loggedMissing)
           {
            m_loggedMissing = true;
            PrintFormat("AltData: no %s - external block %s.", name,
                        m_pinnedCount > 0 ? "0-fills its pinned width" : "disabled (0 features)");
           }
         return false;
        }
      //--- share flags unconditionally: the exporter may rewrite the file while charts hold it open
      //--- (and the tester agent holds files, see the FileCopy share-flag bug).
      int h = FileOpen(name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltData: %s exists but FileOpen failed (%d).", name, GetLastError());
         return false;
        }
      string header = FileReadString(h);
      string cols[];
      int nCols = StringSplit(header, ';', cols);
      if(nCols < 2 || cols[0] != "date")
        {
         PrintFormat("AltData: %s header unrecognised ('%s') - ignoring file.", name, header);
         FileClose(h);
         return false;
        }
      int fileCount = MathMin(nCols - 1, ALTDATA_MAX_FEATURES);
      //--- LOCAL until the parse actually succeeds (rows>0 below) - m_fileNames used to be
      //--- overwritten here unconditionally, before the row loop and before the rows==0 failure
      //--- return, leaving it describing the NEW file's columns while m_fileCount/m_rows/m_vals
      //--- still described the OLD one on a failed re-load (EnsureFresh() calls this live, while the
      //--- exporter may be mid-rewrite of the CSV).
      string newFileNames[];
      ArrayResize(newFileNames, fileCount);
      for(int c = 0; c < fileCount; c++)
         newFileNames[c] = cols[c + 1];
      int cap = 8192, rows = 0;
      ArrayResize(m_rowTime, cap);
      ArrayResize(m_vals, cap * fileCount);
      datetime prev = 0;
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringLen(line) < 11)
            continue;
         string f[];
         if(StringSplit(line, ';', f) < fileCount + 1)
            continue;
         datetime t = StringToTime(f[0]);
         if(t <= 0 || t <= prev)   // strictly ascending or the binary search breaks
            continue;
         if(rows >= cap)
           {
            cap = cap * 3 / 2;
            ArrayResize(m_rowTime, cap);
            ArrayResize(m_vals, cap * fileCount);
           }
         m_rowTime[rows] = t;
         for(int c = 0; c < fileCount; c++)
           {
            string cell = f[c + 1];
            m_vals[rows * fileCount + c] = (StringLen(cell) == 0) ? EMPTY_VALUE : StringToDouble(cell);
           }
         prev = t;
         rows++;
        }
      FileClose(h);
      if(rows == 0)
        {
         PrintFormat("AltData: %s parsed to 0 rows - ignoring file.", name);
         return false;
        }
      m_rows = rows;
      m_fileCount = fileCount;
      ArrayCopy(m_fileNames, newFileNames);
      m_loadedFile = name;
      m_loadedSymbol = symbol;
      m_loadedPeriod = period;
      ArrayResize(m_rowTime, rows);
      ArrayResize(m_vals, rows * fileCount);
      //--- Column medians over the covered range, ignoring blank cells - see m_median. Cheap: a few
      //--- thousand rows x ~15 columns, once per file load.
      ArrayResize(m_median, fileCount);
      ArrayInitialize(m_median, 0.0);
      int blanks = 0;
      for(int c = 0; c < fileCount; c++)
        {
         double col[];
         ArrayResize(col, rows);
         int m = 0;
         for(int rr = 0; rr < rows; rr++)
           {
            double v = m_vals[rr * fileCount + c];
            if(v != EMPTY_VALUE)
               col[m++] = v;
            else
               blanks++;
           }
         if(m <= 0)
            continue;
         ArrayResize(col, m);
         m_median[c] = MathMedian(col);
        }
      if(blanks > 0)
         PrintFormat("AltData: %s has %d blank cells across %d rows x %d columns - these read as the"
                     " column MEDIAN, not as 0 (see m_median). A level series reported as 0 is an"
                     " impossible value, not a missing one.", name, blanks, rows, fileCount);
      BuildMap();
      PrintFormat("AltData: %s - %d features [%s], %d daily rows %s -> %s%s.",
                  name, FeatureCount(), NamesCsv(), rows,
                  TimeToString(m_rowTime[0], TIME_DATE), TimeToString(LastDate(), TIME_DATE),
                  m_pinnedCount > 0 ? " (pinned name list applied)" : "");
      return true;
     }

   //--- As-of read: the last row dated <= t, mapped through the pin. Always fills FeatureCount()
   //--- slots; anything unavailable is 0.0 (the neutral the sanitize loop would produce anyway).
   void              Features(datetime t, double &out[])
     {
      int n = FeatureCount();
      ArrayResize(out, n);
      ArrayInitialize(out, 0.0);
      if(m_rows == 0 || n == 0)
         return;
      //--- BEFORE THE FILE'S FIRST ROW: median-fill, do not zero-fill. See m_median for why a zero here
      //--- was a spike at an impossible value that coincided exactly with the chronological IS/OOS
      //--- split. Still never rejects the bar - the block stays additive context, as designed.
      if(t < m_rowTime[0])
        {
         for(int j = 0; j < n; j++)
           {
            int c = m_map[j];
            if(c >= 0 && c < ArraySize(m_median))
               out[j] = m_median[c];
           }
         return;
        }
      //--- ROW DATES ARE NAIVE, BAR TIMES ARE BROKER TIME. A row dated D means "public from D 00:00
      //--- UTC", so the first bar of each broker day reads it the broker's offset (2-3h) early.
      //--- Shorten a stamp and this becomes a leak: the buffer is the guarantee, not this
      //--- comparison.
      int lo = 0, hi = m_rows - 1;
      while(lo < hi)
        {
         int mid = (lo + hi + 1) / 2;
         if(m_rowTime[mid] <= t)
            lo = mid;
         else
            hi = mid - 1;
        }
      int base = lo * m_fileCount;
      for(int j = 0; j < n; j++)
        {
         int c = m_map[j];
         if(c < 0)
            continue;
         double v = m_vals[base + c];
         //--- A BLANK CELL is a gap in the collector's series, not a reading of zero - measured
         //--- 2026-08-17: eia_stk_idx1y alone has 181 blanks in 6,073 rows. Same argument as the
         //--- pre-coverage fill above, so the same constant.
         out[j] = (v != EMPTY_VALUE) ? v
                  : ((c < ArraySize(m_median)) ? m_median[c] : 0.0);
        }
     }

   //--- Live freshness: the exporter rewrites the file on its own schedule; reload when the chart
   //--- has moved past the loaded data, at most once an hour. In the tester the newest bar is
   //--- historical and always <= LastDate(), so this never fires there - by design.
   void              EnsureFresh(datetime newestBar)
     {
      if(m_rows == 0 || newestBar <= LastDate() + 86400)
         return;
      if(TimeCurrent() - m_lastLoadTry < 3600)
         return;
      string sym = m_loadedFile; // reload the exact file we came from
      if(sym == "")
         return;
      PrintFormat("AltData: newest bar %s is beyond loaded data (%s) - reloading %s.",
                  TimeToString(newestBar, TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_loadedFile);
      //--- Re-parse in place; a failed reload keeps the old (stale but valid) panel.
      m_lastLoadTry = TimeCurrent();
      int h = FileOpen(m_loadedFile, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return;
      FileClose(h);
      //--- Cheap existence probe passed; reload with the EXACT (symbol, period) Load() was called
      //--- with. Never reconstruct what you already know.
      if(m_loadedSymbol != "")
         Load(m_loadedSymbol, m_loadedPeriod);
     }
  };
#undef ALTDATA_MAX_FEATURES
#endif // WARRIOR_ALTDATA_MQH