Warrior_EA/System/AltDataFetch.mqh

//+------------------------------------------------------------------+
//|                                                 AltDataFetch.mqh |
//|                                                     AnimateDread |
//| missing history at attach time and keeps appending forward while |
//| deployed, so online learning never depends on an external Python |
//| process. research/altdata remains the RESEARCH side (screening,  |
//| new-source adjudication); this module maintains the exact same   |
//| files in production.                                             |
//+------------------------------------------------------------------+
#include "AltData.mqh"

#define ALTFETCH_DIR            "Warrior_EA\\AltData\\"
#define ALTFETCH_TIMEOUT_MS     15000
//--- the SPX option chain is ~13 MB; the small feeds' 15 s would time out on a slow link
#define ALTFETCH_GEX_TIMEOUT_MS 90000
#define ALTFETCH_RETRY_SECONDS  3600
#define ALTFETCH_GRID_START     D'2010.01.01'

//--- one raw series: weekly COT positioning, a daily FRED series, or the EIA WPSR block
struct SAltRawSeries
  {
   datetime          date[];       // observation date, ascending
   datetime          published[];
   double            v1[];         // FRED: value | COT: open interest      | EIA: crude stocks ex-SPR
   double            v2[];         // COT: spec long                       | EIA: field production
   double            v3[];         // COT: spec short                      | EIA: refinery utilization %
   int               rows;
  };

//+------------------------------------------------------------------+
//| Append one empty row and return its index.                        |
//|                                                                  |
//| The five arrays are ONE record split across five buffers, so a    |
//| resize missed on any of them reads out of range on the next       |
//| append rather than failing here. Callers fill the values they     |
//| have; the rest stay 0.                                            |
//+------------------------------------------------------------------+
int AltSeriesAppend(SAltRawSeries &s, const datetime date, const datetime published)
  {
   int n = s.rows;
   ArrayResize(s.date, n + 1);
   ArrayResize(s.published, n + 1);
   ArrayResize(s.v1, n + 1);
   ArrayResize(s.v2, n + 1);
   ArrayResize(s.v3, n + 1);
   s.date[n] = date;
   s.published[n] = published;
   s.v1[n] = 0.0;
   s.v2[n] = 0.0;
   s.v3[n] = 0.0;
   s.rows = n + 1;
   return n;
  }
//--- one symbol catalog row: which sources feed the instrument and which feature columns its
//--- {SYM}_D1.csv carries (column order = EA input order, APPEND-ONLY per the .cfg name pin)
struct SAltSymbolSpec
  {
   string            canonical; // catalog key - also names the shared COT raw cache
   string            aliases;   // ';'-separated broker names; matched EXACT first, then as a
                                // PREFIX so suffixed variants (US500.cash, XAUUSDm, EURUSD.r)
                                // resolve without being listed one by one
   string            cotWhere;  // SoQL predicate selecting the CFTC contract; "" = no COT leg
   double            cotSign;   // +1 futures quote matches the chart; -1 inverted (JPY/CAD futures are FX/USD)
   string            features;  // ';'-separated feature ids, in CSV column order
   string            ivolSeries;// FRED id of THIS instrument's own implied-vol index; "" = none
  };

class CAltDataFetch
  {
protected:
   string            m_fredKey;
   bool              m_keyLoaded;
   string            m_eiaKey;
   bool              m_eiaWarned;
   datetime          m_lastGexDate;   // last UTC day whose GEX row is already on disk
   bool              m_gexDateLoaded;
   string            m_blockedHost[];   // 4014 backoff, PER HOST - one missing whitelist entry
   datetime          m_blockedUntil[];  // must not silence the hosts that are whitelisted
   bool              m_urlAlerted;      // full four-line reference printed once per session
   datetime          m_lastAttempt[16]; // per-source retry throttle: 0=VIX 1=USD 2=COT 3=EIA 4=IVOL
                                        // 5=DGS10 6=T10Y2Y 7=T5YIE 8=DFF 9=ECBDFR 10=CPI 11=UNRATE

   //--- ---------------------------------------------------------- utils
   //--- The FredApiKey input is the key's home - it travels with the EA and survives the
   //--- Common\Files wipe that starts every fresh test (the 2026-08-16 incident: keys.txt
   //--- was wiped with the folder, FredKey() returned "" with NO log line, COT updated but
   //--- FRED never ran and the SP500 rebuild - gated on all three raws - never fired).
   //--- keys.txt is only the fallback for a blanked input, re-read on every hourly-throttled
   //--- attempt until a key is found; m_keyLoaded means "warned once", never fail silently.
   string            FredKey(void)
     {
      if(FredApiKey != "")
         return FredApiKey;
      if(m_fredKey != "")
         return m_fredKey;
      int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         if(!m_keyLoaded)
            Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt "
                  "does not exist in Common\\Files - FRED features (VIX, USD index) cannot update, "
                  "so the SP500/XAUUSD feature files will not build. Set the FredApiKey input "
                  "(free key: fred.stlouisfed.org) or create keys.txt with one line: fred=<key>. "
                  "The file is re-checked hourly; no restart needed.");
         m_keyLoaded = true;
         return "";
        }
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringFind(line, "fred=") == 0)
            m_fredKey = StringSubstr(line, 5);
        }
      FileClose(h);
      if(m_fredKey == "" && !m_keyLoaded)
         Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt has "
               "no 'fred=<key>' line - FRED-based features will not update until one is provided "
               "(free key: fred.stlouisfed.org).");
      m_keyLoaded = true;
      return m_fredKey;
     }

   //--- same contract as FredKey: input first, keys.txt "eia=" line as fallback, loud once
   string            EiaKey(void)
     {
      if(EiaApiKey != "")
         return EiaApiKey;
      if(m_eiaKey != "")
         return m_eiaKey;
      int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h != INVALID_HANDLE)
        {
         while(!FileIsEnding(h))
           {
            string line = FileReadString(h);
            if(StringFind(line, "eia=") == 0)
               m_eiaKey = StringSubstr(line, 4);
           }
         FileClose(h);
        }
      if(m_eiaKey == "" && !m_eiaWarned)
        {
         m_eiaWarned = true;
         Print("AltDataFetch: the EiaApiKey input is blank and no 'eia=<key>' line exists in " +
               ALTFETCH_DIR + "keys.txt - the EIA feature columns will stay empty (panel 0-fills; "
               "free key: eia.gov/opendata). Re-checked hourly; no restart needed.");
        }
      return m_eiaKey;
     }

   //--- "https://host/..." -> "https://host" (what the MT5 whitelist dialog wants)
   string            HostOf(string url)
     {
      int p = StringFind(url, "://");
      int q = StringFind(url, "/", p < 0 ? 0 : p + 3);
      return (q < 0) ? url : StringSubstr(url, 0, q);
     }

   //--- URL safe for the JOURNAL: api_key values masked. The 2026-08-16 EIA failure
   //--- printed the first 80 chars of the URL - which included most of the key.
   //--- Every error path that echoes a URL must go through this.
   string            MaskUrl(string url)
     {
      int p = StringFind(url, "api_key=");
      if(p < 0)
         return StringSubstr(url, 0, 100);
      int e = StringFind(url, "&", p);
      string masked = StringSubstr(url, 0, p + 8) + "***";
      if(e > 0)
         masked += StringSubstr(url, e);
      return StringSubstr(masked, 0, 100);
     }

   int               BlockedIndex(string host)
     {
      for(int i = 0; i < ArraySize(m_blockedHost); i++)
         if(m_blockedHost[i] == host)
            return i;
      return -1;
     }

   bool              HttpGet(string url, string &body, int timeoutMs = ALTFETCH_TIMEOUT_MS)
     {
      //--- PER-HOST backoff. An in-flight request cannot be cancelled; refusing to START another
      //--- one is the whole of the remedy.
      if(IsStopped())
         return false;
      string host = HostOf(url);
      int bi = BlockedIndex(host);
      if(bi >= 0 && TimeCurrent() < m_blockedUntil[bi])
         return false;
      char data[], result[];
      string rh;
      ResetLastError();
      //--- identify ourselves: some API gateways reject requests with an empty User-Agent
      //--- at the transport level (candidate cause of the EIA 1003 on first live fetch)
      int status = WebRequest("GET", url, "User-Agent: WarriorEA/1.0\r\n", timeoutMs, data, result, rh);
      if(status == -1)
        {
         int err = GetLastError();
         if(err == 4014)
           {
            //--- Backoff, not a permanent latch: the whitelist takes effect for a RUNNING EA
            //--- the moment the user saves the options dialog, so retrying hourly picks the
            //--- fix up without a re-attach.
            bool firstTimeThisHost = (bi < 0);
            if(bi < 0)
              {
               bi = ArraySize(m_blockedHost);
               ArrayResize(m_blockedHost, bi + 1);
               ArrayResize(m_blockedUntil, bi + 1);
               m_blockedHost[bi] = host;
              }
            m_blockedUntil[bi] = TimeCurrent() + 3600;
            if(firstTimeThisHost)
              {
               //--- ALWAYS name the offender - the whole point of this message is telling
               //--- the user which entry is missing, not that "something" is
               Alert("Warrior EA: web request to ", host, " is blocked (4014) - this exact ",
                     "address is missing from the WebRequest whitelist (or the checkbox is ",
                     "off). Copy-paste line and steps are in the Experts log.");
               Print("AltDataFetch: ==================================================================");
               Print("AltDataFetch: BLOCKED host (add this exact line as its own whitelist entry):");
               Print(host);
               Print("AltDataFetch: MT5 menu: Tools > Options > Expert Advisors ->");
               Print("AltDataFetch: tick 'Allow WebRequest for listed URL', add the line, click OK.");
               Print("AltDataFetch: Takes effect immediately - the EA retries within the hour.");
               if(!m_urlAlerted)
                 {
                  m_urlAlerted = true;
                  Print("AltDataFetch: For reference, the FULL set this EA uses (each its own entry):");
                  Print("https://publicreporting.cftc.gov");
                  Print("https://api.stlouisfed.org");
                  Print("https://api.eia.gov");
                  Print("https://cdn.cboe.com");
                  Print("AltDataFetch: Until fixed it runs normally on cached alt-data files - features");
                  Print("AltDataFetch: stay valid but stop updating past their last downloaded date.");
                 }
               Print("AltDataFetch: ==================================================================");
              }
            else
               Print("AltDataFetch: still blocked (4014): " + host + " - retrying in an hour.");
           }
         else
            PrintFormat("AltDataFetch: WebRequest error %d for %s", err, MaskUrl(url));
         return false;
        }
      if(status != 200)
        {
         PrintFormat("AltDataFetch: HTTP %d for %s%s", status, MaskUrl(url),
                     status >= 1000 ? " (1xxx = MT5 transport-layer failure, not a server "
                     "response - retried on the normal hourly schedule)" : "");
         return false;
        }
      body = CharArrayToString(result, 0, WHOLE_ARRAY, CP_UTF8);
      return true;
     }

   //--- minimal targeted JSON scan: value of "key":"..." (or numeric) at/after position `from`.
   //--- Sufficient for Socrata/FRED responses, which are flat arrays of flat objects.
   string            JsonField(const string &body, string key, int from, int &valueEnd)
     {
      valueEnd = -1;
      string tag = "\"" + key + "\":";
      int p = StringFind(body, tag, from);
      if(p < 0)
         return "";
      p += StringLen(tag);
      if(StringGetCharacter(body, p) == '"')
        {
         int q = StringFind(body, "\"", p + 1);
         if(q < 0)
            return "";
         valueEnd = q + 1;
         return StringSubstr(body, p + 1, q - p - 1);
        }
      int e = p;
      while(e < StringLen(body))
        {
         ushort c = StringGetCharacter(body, e);
         if((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')
            e++;
         else
            break;
        }
      valueEnd = e;
      return StringSubstr(body, p, e - p);
     }

   datetime          SaturdayAfter(datetime reportDate)
     {
      MqlDateTime st;
      TimeToStruct(reportDate, st);
      int days = (6 - st.day_of_week + 7) % 7;   // MQL5: 0=Sunday .. 6=Saturday
      if(days == 0)
         days = 7;
      return reportDate + days * 86400;
     }

   string            UrlEncodePart(string s)
     {
      string out = "";
      for(int i = 0; i < StringLen(s); i++)
        {
         ushort c = StringGetCharacter(s, i);
         //--- '%' MUST come first and MUST be escaped: SoQL like-predicates use it as the
         //--- wildcard ('EURO FX -%'), and an unescaped one turns the rest of the query into
         //--- a malformed percent-escape.
         if(c == '%')       out += "%25";
         else if(c == ' ')  out += "%20";
         else if(c == '\'') out += "%27";
         else if(c == '&')  out += "%26";
         else if(c == '>')  out += "%3E";
         else if(c == '=')  out += "%3D";
         else if(c == '+')  out += "%2B";
         else if(c == '#')  out += "%23";
         else               out += ShortToString(c);
        }
      return out;
     }

   //--- ------------------------------------------ raw cache load/save
   bool              LoadRaw(string name, SAltRawSeries &s, int cols)
     {
      s.rows = 0;
      int h = FileOpen(ALTFETCH_DIR + name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return false;
      FileReadString(h); // header
      int cap = 2048;
      ArrayResize(s.date, cap);
      ArrayResize(s.published, cap);
      ArrayResize(s.v1, cap);
      ArrayResize(s.v2, cap);
      ArrayResize(s.v3, cap);
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         string f[];
         if(StringSplit(line, ';', f) < cols + 2)
            continue;
         if(s.rows >= cap)
           {
            cap = cap * 3 / 2;
            ArrayResize(s.date, cap);
            ArrayResize(s.published, cap);
            ArrayResize(s.v1, cap);
            ArrayResize(s.v2, cap);
            ArrayResize(s.v3, cap);
           }
         s.date[s.rows] = StringToTime(f[0]);
         s.published[s.rows] = StringToTime(f[1]);
         s.v1[s.rows] = StringToDouble(f[2]);
         s.v2[s.rows] = cols > 1 ? StringToDouble(f[3]) : 0.0;
         s.v3[s.rows] = cols > 2 ? StringToDouble(f[4]) : 0.0;
         s.rows++;
        }
      FileClose(h);
      return s.rows > 0;
     }

   //--- Whole-file rewrites go through a temp + FileMove swap: multiple charts share these caches
   //--- (raw_VIXCLS serves every symbol; two timeframes of one symbol share all of them), and a
   //--- reader hitting a truncate-then-write mid-flight parses a torn file.
   bool              SaveRaw(string name, SAltRawSeries &s, int cols, string header)
     {
      string tmp = ALTFETCH_DIR + name + ".tmp";
      int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltDataFetch: cannot write %s (%d)", tmp, GetLastError());
         return false;
        }
      FileWriteString(h, header + "\n");
      for(int i = 0; i < s.rows; i++)
        {
         string line = TimeToString(s.date[i], TIME_DATE) + ";" + TimeToString(s.published[i], TIME_DATE) + ";"
                       + DoubleToString(s.v1[i], 6);
         if(cols > 1)
            line += ";" + DoubleToString(s.v2[i], 6);
         if(cols > 2)
            line += ";" + DoubleToString(s.v3[i], 6);
         FileWriteString(h, line + "\n");
        }
      FileClose(h);
      if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + name, FILE_COMMON | FILE_REWRITE))
        {
         PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - cache not updated this pass.",
                     tmp, name, GetLastError());
         FileDelete(tmp, FILE_COMMON);
         return false;
        }
      return true;
     }

   //--- ------------------------------------------------- FRED source appends observations after
   //--- the cached tail; full history when the cache is empty vmin/vmax: per-series plausibility
   //--- band. Rejected rows are counted and reported, never silently dropped.
   bool              UpdateFred(string seriesId, SAltRawSeries &s, int throttleSlot,
                                int staleDays = 4, double vmin = -1e18, double vmax = 1e18)
     {
      datetime now = TimeCurrent();
      //--- daily series: current until ~2 business days behind (weekends/holidays).
      //--- monthly series pass staleDays ~32 so a 30-day-old latest print does not
      //--- trigger a pointless fetch attempt every hour for weeks.
      if(s.rows > 0 && now - s.date[s.rows - 1] < staleDays * 86400)
         return false;
      if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
         return false;
      m_lastAttempt[throttleSlot] = now;
      string key = FredKey();
      if(key == "")
         return false;
      string url = "https://api.stlouisfed.org/fred/series/observations?series_id=" + seriesId +
                   "&api_key=" + key + "&file_type=json&limit=100000";
      //--- ALWAYS bound the start. 2005 leaves 5y of lookback margin ahead of the grid for the
      //--- longest window (3y COT percentile analog).
      string startDate = (s.rows > 0)
                         ? TimeToString(s.date[s.rows - 1] + 86400, TIME_DATE)
                         : "2005.01.01";
      StringReplace(startDate, ".", "-");      // TimeToString is yyyy.mm.dd; FRED wants ISO dashes
      url += "&observation_start=" + startDate;
      string body;
      if(!HttpGet(url, body))
         return false;
      int pos = 0, added = 0, rejected = 0;
      while(true)
        {
         int e1;
         string d = JsonField(body, "date", pos, e1);
         if(d == "" || e1 < 0)
            break;
         int e2;
         string v = JsonField(body, "value", e1, e2);
         pos = (e2 > e1 ? e2 : e1);
         if(v == "" || v == ".")
            continue;
         datetime dt = StringToTime(d);        // accepts yyyy-mm-dd
         if(dt <= 0 || (s.rows > 0 && dt <= s.date[s.rows - 1]))
            continue;
         double val = StringToDouble(v);
         if(val < vmin || val > vmax || !MathIsValidNumber(val))
           {
            rejected++;
            continue;
           }
         int n = AltSeriesAppend(s, dt, dt + 86400);   // unrevised daily: knowable next day
         s.v1[n] = val;
         added++;
        }
      if(rejected > 0)
         PrintFormat("AltDataFetch: %s REJECTED %d value(s) outside the plausibility band "
                     "[%g, %g] - transport corruption or format drift; investigate if repeated.",
                     seriesId, rejected, vmin, vmax);
      if(added > 0)
        {
         SaveRaw("raw_" + seriesId + ".csv", s, 1, "date;published;value");
         PrintFormat("AltDataFetch: %s +%d rows (through %s)", seriesId, added,
                     TimeToString(s.date[s.rows - 1], TIME_DATE));
        }
      else if(s.rows == 0)
         PrintFormat("AltDataFetch: %s returned a response with ZERO usable observations and "
                     "the cache is empty - series id wrong or API format changed. Features "
                     "depending on it will stay empty (panel 0-fills).", seriesId);
      return added > 0;
     }

   //--- -------------------------------------------------- COT source
   bool              UpdateCot(string cacheName, string cotWhere, SAltRawSeries &s, int throttleSlot)
     {
      datetime now = TimeCurrent();
      //--- weekly: a new Tuesday report exists once the last one is >7d old; allow 3 extra
      //--- days before nagging the API (Friday release + weekend)
      if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
         return false;
      if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
         return false;
      m_lastAttempt[throttleSlot] = now;
      string where = cotWhere;
      if(s.rows > 0)
        {
         string lastDate = TimeToString(s.date[s.rows - 1], TIME_DATE);
         StringReplace(lastDate, ".", "-");    // ISO for the SoQL timestamp comparison
         where += " AND report_date_as_yyyy_mm_dd > '" + lastDate + "'";
        }
      string url = "https://publicreporting.cftc.gov/resource/gpe5-46if.json?%24select=" +
                   "report_date_as_yyyy_mm_dd,open_interest_all,lev_money_positions_long,lev_money_positions_short" +
                   "&%24where=" + UrlEncodePart(where) +
                   "&%24order=report_date_as_yyyy_mm_dd&%24limit=50000";
      string body;
      if(!HttpGet(url, body))
         return false;
      int pos = 0, added = 0;
      while(true)
        {
         int e1;
         string d = JsonField(body, "report_date_as_yyyy_mm_dd", pos, e1);
         if(d == "" || e1 < 0)
            break;
         int e2, e3, e4;
         string oi = JsonField(body, "open_interest_all", e1, e2);
         string ll = JsonField(body, "lev_money_positions_long", e2, e3);
         string ls = JsonField(body, "lev_money_positions_short", e3, e4);
         pos = (e4 > e1 ? e4 : e1);
         datetime dt = StringToTime(StringSubstr(d, 0, 10));
         if(dt <= 0 || oi == "" || ll == "" || ls == "")
            continue;
         if(s.rows > 0 && dt == s.date[s.rows - 1])
           {
            //--- same report date from a second contract-name variant: keep the larger-OI row
            if(StringToDouble(oi) > s.v1[s.rows - 1])
              {
               s.v1[s.rows - 1] = StringToDouble(oi);
               s.v2[s.rows - 1] = StringToDouble(ll);
               s.v3[s.rows - 1] = StringToDouble(ls);
              }
            continue;
           }
         if(s.rows > 0 && dt < s.date[s.rows - 1])
            continue;
         int n = AltSeriesAppend(s, dt, SaturdayAfter(dt));
         s.v1[n] = StringToDouble(oi);
         s.v2[n] = StringToDouble(ll);
         s.v3[n] = StringToDouble(ls);
         added++;
        }
      if(added > 0)
        {
         SaveRaw(cacheName, s, 3, "date;published;oi;lev_long;lev_short");
         PrintFormat("AltDataFetch: %s +%d weekly reports (through %s)", cacheName, added,
                     TimeToString(s.date[s.rows - 1], TIME_DATE));
        }
      else if(s.rows == 0)
         PrintFormat("AltDataFetch: COT query for %s returned ZERO reports on an empty cache - "
                     "the contract-name predicate probably matches nothing at the endpoint "
                     "(the like-predicates for non-SP500/JPY contracts are unverified; see the "
                     "catalog). COT features stay empty (panel 0-fills) until it is corrected.",
                     cacheName);
      return added > 0;
     }

   //--- ---------------------------------------------------- EIA source Weekly Petroleum Status
   //--- Report via the EIA v2 API, three series in one call (2010->now x 3 ~ 2,600 rows, under the
   //--- API's 5000-row page - revisit if series are added).
   bool              UpdateEia(SAltRawSeries &s, int throttleSlot)
     {
      datetime now = TimeCurrent();
      //--- weekly cadence, same nag guard as COT
      if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
         return false;
      if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
         return false;
      m_lastAttempt[throttleSlot] = now;
      string key = EiaKey();
      if(key == "")
         return false;
      string startDate = TimeToString(s.rows > 0 ? s.date[s.rows - 1] + 86400 : ALTFETCH_GRID_START, TIME_DATE);
      StringReplace(startDate, ".", "-");      // TimeToString is yyyy.mm.dd; the API wants ISO dashes
      string url = "https://api.eia.gov/v2/petroleum/sum/sndw/data/?api_key=" + key +
                   "&frequency=weekly&data%5B0%5D=value" +
                   "&facets%5Bseries%5D%5B0%5D=WCESTUS1" +      // crude stocks ex-SPR
                   "&facets%5Bseries%5D%5B1%5D=WCRFPUS2" +      // field production
                   "&facets%5Bseries%5D%5B2%5D=WPULEUS3" +      // refinery utilization %
                   "&sort%5B0%5D%5Bcolumn%5D=period&sort%5B0%5D%5Bdirection%5D=asc" +
                   "&length=5000&start=" + startDate;
      string body;
      if(!HttpGet(url, body))
         return false;
      int pos = 0, added = 0;
      while(true)
        {
         int e1, e2, e3;
         string d = JsonField(body, "period", pos, e1);
         if(d == "" || e1 < 0)
            break;
         string sid = JsonField(body, "series", e1, e2);
         string v = JsonField(body, "value", (e2 > e1 ? e2 : e1), e3);
         pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
         datetime dt = StringToTime(d);           // accepts yyyy-mm-dd
         if(dt <= 0 || sid == "" || v == "")
            continue;
         if(s.rows > 0 && dt < s.date[s.rows - 1])
            continue;
         if(s.rows == 0 || dt > s.date[s.rows - 1])
           {
            int n = AltSeriesAppend(s, dt, dt + 6 * 86400);
            added++;
           }
         int r = s.rows - 1;
         if(sid == "WCESTUS1")      s.v1[r] = StringToDouble(v);
         else if(sid == "WCRFPUS2") s.v2[r] = StringToDouble(v);
         else if(sid == "WPULEUS3") s.v3[r] = StringToDouble(v);
        }
      if(added > 0)
        {
         SaveRaw("raw_EIA_WPSR.csv", s, 3, "date;published;stocks;production;utilization");
         PrintFormat("AltDataFetch: raw_EIA_WPSR.csv +%d weekly reports (through %s)", added,
                     TimeToString(s.date[s.rows - 1], TIME_DATE));
        }
      return added > 0;
     }

   //--- ------------------------------------ GEX forward recorder WHY THIS IS A RECORDER AND NOT A
   //--- FEATURE: option open interest is a SNAPSHOT source. Revisit when the file holds a year or
   //--- so, then screen it like every other candidate.
   string            CboeTicker(string canonical)
     {
      if(canonical == "SP500")  return "_SPX";   // index options - where dealer positioning concentrates
      if(canonical == "NAS100") return "_NDX";
      if(canonical == "US30")   return "_DJX";
      if(canonical == "US2000") return "IWM";
      if(canonical == "XAUUSD") return "GLD";
      if(canonical == "XAGUSD") return "SLV";
      if(canonical == "XTIUSD") return "USO";
      if(canonical == "BTCUSD") return "IBIT";
      return "";                                 // no listed-options proxy worth recording
     }

   //--- last date already in the file, or 0
   datetime          LastGexDate(string file)
     {
      int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return 0;
      string last = "";
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
            last = line;
        }
      FileClose(h);
      if(last == "")
         return 0;
      string f[];
      if(StringSplit(last, ';', f) < 1)
         return 0;
      return StringToTime(f[0]);
     }

   bool              UpdateGex(string canonical)
     {
      string ticker = CboeTicker(canonical);
      if(ticker == "")
         return false;
      //--- Once per UTC day, after the US cash close (21:00 UTC covers both DST regimes),
      //--- and never on a weekend - the chain would just repeat Friday's settled numbers.
      MqlDateTime g;
      TimeToStruct(TimeGMT(), g);
      if(g.day_of_week == 0 || g.day_of_week == 6)
         return false;
      if(g.hour < 21)
         return false;
      string file = ALTFETCH_DIR + "gex_" + canonical + ".csv";
      datetime today = StringToTime(StringFormat("%04d.%02d.%02d", g.year, g.mon, g.day));
      if(!m_gexDateLoaded)
        {
         m_lastGexDate = LastGexDate(file);
         m_gexDateLoaded = true;
        }
      if(m_lastGexDate >= today)
         return false;
      //--- Re-read from DISK before spending the download: two charts of the same symbol
      //--- (e.g. D1 + H4) each run their own fetcher, and a per-instance cached date would
      //--- let the second chart append a duplicate row for the day the first just recorded.
      m_lastGexDate = LastGexDate(file);
      if(m_lastGexDate >= today)
         return false;
      //--- the SPX chain is ~13 MB: this needs a far longer timeout than the small feeds
      string body;
      if(!HttpGet("https://cdn.cboe.com/api/global/delayed_quotes/options/" + ticker + ".json",
                  body, ALTFETCH_GEX_TIMEOUT_MS))
         return false;
      if(StringLen(body) < 1000)
        {
         Print("AltDataFetch: GEX response for " + ticker + " was too short to be a chain - skipping.");
         return false;
        }
      int e0;
      string spotS = JsonField(body, "current_price", 0, e0);   // absent from option rows, so
      if(spotS == "")                                           // a plain scan finds the header one
         spotS = JsonField(body, "close", 0, e0);
      double spot = StringToDouble(spotS);
      if(spot <= 0.0)
        {
         Print("AltDataFetch: GEX " + ticker + " - no usable spot price in the response, skipping.");
         return false;
        }
      //--- per-expiry accumulation (54 expiries on SPX; the cap is generous)
      int    expCode[128];
      double expGex[128];
      int    nExp = 0;
      double gexCall = 0.0, gexPut = 0.0;
      double oiCall = 0.0, oiPut = 0.0;
      int    rows = 0;
      int    pos = 0;
      //--- Row layout is option, ..., open_interest, volume, delta, gamma - so one forward
      //--- sweep reads each row's three fields in the order they appear.
      while(true)
        {
         int e1;
         string sym = JsonField(body, "option", pos, e1);
         if(sym == "" || e1 < 0)
            break;
         int e2, e3;
         string oiS = JsonField(body, "open_interest", e1, e2);
         string gS  = JsonField(body, "gamma", (e2 > e1 ? e2 : e1), e3);
         pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
         double oi = StringToDouble(oiS);
         double gam = StringToDouble(gS);
         if(oi <= 0.0 || gam == 0.0)
            continue;                        // no exposure - skip the string work entirely
         //--- OCC symbol: <root><YYMMDD><C|P><strike*1000, 8 digits>, so the tail is fixed
         //--- width regardless of how long the root is (SPX, SPXW, IWM, ...).
         int len = StringLen(sym);
         if(len < 15)
            continue;
         string kind = StringSubstr(sym, len - 9, 1);
         int expiry = (int)StringToInteger(StringSubstr(sym, len - 15, 6));
         //--- dollar gamma per 1% move: gamma x OI x 100 (contract multiplier) x S^2 x 0.01
         double dollar = gam * oi * 100.0 * spot * spot * 0.01;
         if(kind == "C")
           {
            gexCall += dollar;
            oiCall  += oi;
           }
         else
           {
            gexPut -= dollar;               // dealers short puts: opposite sign convention
            oiPut  += oi;
            dollar = -dollar;
           }
         int slot = -1;
         for(int i = 0; i < nExp; i++)
            if(expCode[i] == expiry)
              {
               slot = i;
               break;
              }
         if(slot < 0 && nExp < 128)
           {
            slot = nExp++;
            expCode[slot] = expiry;
            expGex[slot]  = 0.0;
           }
         if(slot >= 0)
            expGex[slot] += dollar;
         rows++;
        }
      if(rows == 0)
        {
         Print("AltDataFetch: GEX " + ticker + " - parsed 0 contracts with exposure, skipping "
               "(the CDN response format may have changed).");
         return false;
        }
      //--- Partial-chain guard: a truncated 13 MB download parses cleanly and yields a
      //--- plausible-but-wrong GEX. A day-over-day contract-count collapse (>50%) is the
      //--- fingerprint of truncation, not of markets - OI does not halve overnight.
      int prevRows = LastGexRowCount(file);
      if(prevRows > 0 && rows < prevRows / 2)
        {
         PrintFormat("AltDataFetch: GEX %s - only %d contracts vs %d yesterday (<50%%): response "
                     "looks TRUNCATED, not recording. Will retry on the next timer pass.",
                     ticker, rows, prevRows);
         return false;
        }
      //--- three nearest expiries, by selection (nExp is tiny)
      double front[3] = {0.0, 0.0, 0.0};
      int    frontCode = 0, prevCode = 0;
      for(int k = 0; k < 3; k++)
        {
         int best = -1;
         for(int i = 0; i < nExp; i++)
           {
            if(expCode[i] <= prevCode)
               continue;
            if(best < 0 || expCode[i] < expCode[best])
               best = i;
           }
         if(best < 0)
            break;
         front[k] = expGex[best] / 1e9;
         prevCode = expCode[best];
         if(k == 0)
            frontCode = prevCode;
        }
      bool isNew = !FileIsExist(file, FILE_COMMON);
      int h = FileOpen(file, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltDataFetch: cannot write %s (%d) - GEX not recorded today.", file, GetLastError());
         return false;
        }
      if(isNew)
         FileWriteString(h, "date;recorded_utc;spot;gex_total;gex_call;gex_put;oi_call;oi_put;"
                            "gex_exp1;gex_exp2;gex_exp3;expiry1;n_expiry;n_rows\n");
      FileSeek(h, 0, SEEK_END);
      FileWriteString(h, StringFormat("%s;%s;%.4f;%.6f;%.6f;%.6f;%.0f;%.0f;%.6f;%.6f;%.6f;%d;%d;%d\n",
                                      TimeToString(today, TIME_DATE),
                                      TimeToString(TimeGMT(), TIME_DATE | TIME_MINUTES),
                                      spot, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9,
                                      oiCall, oiPut, front[0], front[1], front[2],
                                      frontCode, nExp, rows));
      FileClose(h);
      m_lastGexDate = today;
      //--- Progress matters here: the file is useless until it is long enough to screen, and
      //--- the only way to know how far along it is, is to say so.
      int have = 1;
      datetime probe = LastGexDate(file);
      if(probe > 0)
         have = CountGexRows(file);
      PrintFormat("AltDataFetch: GEX %s recorded - total %+.2f Bn/1%% (calls %+.2f, puts %+.2f), "
                  "%d contracts, %d expiries, spot %.2f. %d day(s) banked; this is a FORWARD "
                  "RECORDER - no free history exists, so it feeds no model until there is enough "
                  "to screen (~250 trading days).",
                  ticker, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9, rows, nExp, spot, have);
      return true;
     }

   //--- n_rows (field 14) of the last data line - the previous session's contract count
   int               LastGexRowCount(string file)
     {
      int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return 0;
      string last = "";
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
            last = line;
        }
      FileClose(h);
      if(last == "")
         return 0;
      string f[];
      if(StringSplit(last, ';', f) < 14)
         return 0;
      return (int)StringToInteger(f[13]);
     }

   int               CountGexRows(string file)
     {
      int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return 0;
      int n = 0;
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
            n++;
        }
      FileClose(h);
      return n;
     }

   //--- -------------------------------- feature math (mirrors export.py)
   //--- pandas rolling(W, min_periods=M).rank(pct=True) on the LAST element: rank of the current
   //--- value among the window's values (average rank for ties) divided by the window count.
   double            RollingPctRank(const double &v[], int i, int window, int minPeriods)
     {
      int lo = MathMax(0, i - window + 1);
      int n = i - lo + 1;
      if(n < minPeriods)
         return EMPTY_VALUE;
      double cur = v[i];
      int below = 0, ties = 0;
      for(int k = lo; k <= i; k++)
        {
         if(v[k] < cur)
            below++;
         else if(v[k] == cur)
            ties++;
        }
      double rank = below + (1.0 + ties) / 2.0;
      return rank / n;
     }

   //--- as-of index: largest row with published <= day, else -1
   int               AsOf(const SAltRawSeries &s, datetime day)
     {
      int lo = 0, hi = s.rows - 1, ans = -1;
      while(lo <= hi)
        {
         int mid = (lo + hi) / 2;
         if(s.published[mid] <= day)
           {
            ans = mid;
            lo = mid + 1;
           }
         else
            hi = mid - 1;
        }
      return ans;
     }

   //--- raw series kept resident between timer ticks (one chart = one symbol, so only the
   //--- sources the chart's catalog row names are ever populated)
   //--- m_ivol holds whichever instrument-specific implied-vol index THIS chart's catalog
   //--- row names (GVZ for gold, OVX for oil, VXN/VXD/RVX for the index symbols) - one
   //--- chart is one symbol, so a single slot serves regardless of which series it is.
   SAltRawSeries     m_vix, m_usd, m_cot, m_eia, m_ivol;
   //--- US macro block. UNRATE is the exception: it IS revised (seasonal refits, ~0.1-0.2pp), there
   //--- is no unrevised variant, and the EA cannot run the ALFRED first-print protocol - so a fresh
   //--- backfill carries small revisions into history.
   SAltRawSeries     m_dgs10, m_curve, m_bei, m_dff, m_ecb, m_cpi, m_unrate;

   //--- ------------------------------- symbol catalog (data, not code) An MT5 symbol cannot be
   //--- mapped to a CFTC contract by rule - the naming universes are unrelated and every broker
   //--- invents its own index tickers - so the catalog IS the mechanism: the chart symbol selects a
   //--- row at runtime and ONLY that row's sources are fetched.
   SAltSymbolSpec    m_specs[];
   string            m_userFrom[];   // broker symbol   (from symbol_map.cfg / the dialog)
   string            m_userTo[];     // canonical, or "NONE" = user declined alt data
   bool              m_userMapLoaded;

   void              AddSpec(string canonical, string aliases, string cotWhere,
                             double cotSign, string feats, string ivolSeries = "")
     {
      int n = ArraySize(m_specs);
      ArrayResize(m_specs, n + 1);
      m_specs[n].canonical  = canonical;
      m_specs[n].aliases    = aliases;
      m_specs[n].cotWhere   = cotWhere;
      m_specs[n].cotSign    = cotSign;
      m_specs[n].features   = feats;
      m_specs[n].ivolSeries = ivolSeries;
     }

   void              BuildCatalog(void)
     {
      //--- feature bundles (locals, deliberately prefixed so no macro elsewhere can collide) POLICY
      //--- (user directive 2026-08-16, twice): every feature the fetched sources can serve is wired
      //--- on every symbol - the research screens inform priors but do NOT gate the input list ("we
      //--- will leave the NN to do its job").
      string f_eia  = "eia_stk_idx1y;eia_stk_chg4;eia_util";
      string f_risk = "vix_chg5;vix;usd_chg5";                // equity-risk complex
      string f_cot  = "cot_idx_1y;cot_idx_3y;cot_chg_4w";     // COT positioning family
      //--- US macro block: 10y yield 20d change, curve slope, 5y breakeven 20d change, Fed-ECB
      //--- policy gap, CPI yoy, unemployment 12m change.
      string f_mac  = "mac_y10;mac_curve;mac_bei;mac_gap;mac_cpi;mac_unemp";
      //--- CFTC predicates. A predicate that matches nothing costs only an empty column.
      string w_es  = "market_and_exchange_names in('E-MINI S&P 500 - CHICAGO MERCANTILE EXCHANGE',"
                     "'E-MINI S&P 500 STOCK INDEX - CHICAGO MERCANTILE EXCHANGE')";
      string w_jpy = "market_and_exchange_names in('JAPANESE YEN - CHICAGO MERCANTILE EXCHANGE')";
      string w_eur = "market_and_exchange_names like 'EURO FX -%'";
      string w_gbp = "market_and_exchange_names like 'BRITISH POUND%'";
      string w_cad = "market_and_exchange_names like 'CANADIAN DOLLAR%'";
      string w_aud = "market_and_exchange_names like 'AUSTRALIAN DOLLAR%'";
      string w_nzd = "market_and_exchange_names like 'NZ DOLLAR%'";
      string w_chf = "market_and_exchange_names like 'SWISS FRANC%'";
      string w_btc = "market_and_exchange_names like 'BITCOIN%'";

      //--- ---------------- equity indices (VIX/USD complex; only SP500 is screened)
      //--- Screened findings preserved as priors, not gates: SP500 vix_chg5/vix/usd_chg5/
      //--- cot_spec_net and USDJPY COT family cleared family-wise + incremental; XAUUSD
      //--- gvz_chg5 (GVZ = gold's own IV) is the campaign's 2nd-strongest incremental
      //--- feature (MI|vol 0.0197 p=0.002, 4.6x vix_chg5 on gold); EURUSD/USDJPY vix_chg5
      //--- screened clean; the macro block screened null everywhere; EIA screened null on
      //--- WTI. All ship regardless - see the policy note above.
      AddSpec("SP500",  "SP500;SPX500;US500;USA500;USA500IDX;US_500;SPXUSD;S&P500;SP_500;"
                        "SPX;USA500IDXUSD;US500Cash", w_es, 1.0,
              f_risk + ";cot_spec_net;" + f_eia + ";" + f_mac);
      AddSpec("NAS100", "NAS100;NASDAQ;US100;USA100;USATEC;USTEC;NDX100;NQ100;TECH100;"
                        "USA100IDX;US_100;NDX;USATECHIDX", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXNCLS");
      AddSpec("US30",   "US30;DJ30;DOW30;USA30;WALLSTREET;WS30;DJIUSD;USA30IDX;"
                        "US_30;DOWJONES;DJI", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXDCLS");
      AddSpec("US2000", "US2000;RUSSELL2000;RUT;USA2000;RTY;US_2000", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "RVXCLS");
      AddSpec("DE40",   "DE40;GER40;DAX40;DAX;GER30;DE30;GERMANY40;DEU40;GRXEUR;"
                        "DE_40;GER_40", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("UK100",  "UK100;FTSE100;FTSE;GBR100;UKX;GB100;UK_100;BRXGBP", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("JP225",  "JP225;NIKKEI;NI225;JPN225;J225;JPXJPY;JP_225", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("EU50",   "EU50;STOXX50;ESTX50;EUSTX50;SX5E;EUR50;E50EUR", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("FR40",   "FR40;CAC40;FRA40;CAC;FRXEUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("AU200",  "AU200;ASX200;AUS200;SPI200;AU_200;AUXAUD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("HK50",   "HK50;HSI;HKG33;HK33;HANGSENG;HKXHKD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- FX majors (full complement: COT + risk complex + EIA + macro;
      //--- no per-instrument IV - EVZ was discontinued 2025-03 and must never be wired)
      AddSpec("EURUSD", "EURUSD;EUR_USD", w_eur,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDJPY", "USDJPY;USD_JPY", w_jpy, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("GBPUSD", "GBPUSD;GBP_USD", w_gbp,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDCAD", "USDCAD;USD_CAD", w_cad, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("AUDUSD", "AUDUSD;AUD_USD", w_aud,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("NZDUSD", "NZDUSD;NZD_USD", w_nzd,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDCHF", "USDCHF;USD_CHF", w_chf, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- metals (gold/silver COT is in the DISAGGREGATED CFTC dataset -
      //--- different id AND column names - still not wired; silver has no free IV index)
      AddSpec("XAUUSD", "XAUUSD;GOLD;XAU_USD;GOLDUSD", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "GVZCLS");
      AddSpec("XAGUSD", "XAGUSD;SILVER;XAG_USD;SILVERUSD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- energy
      AddSpec("XTIUSD", "XTIUSD;USOIL;WTI;CRUDE;USCRUDE;OIL;WTICOUSD;"
                        "OILUSD;LIGHTCMDUSD;CRUDEOIL", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
      AddSpec("XBRUSD", "XBRUSD;UKOIL;BRENT;UKOUSD;BCOUSD;BRENTCMDUSD", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
      AddSpec("NATGAS", "NATGAS;NGAS;XNGUSD;NATURALGAS", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- crypto
      AddSpec("BTCUSD", "BTCUSD;BITCOIN;XBTUSD;BTC_USD", w_btc, 1.0,
              f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
     }

   //--- ------------------------------------- user mapping persistence
   //--- symbol_map.cfg: one "BROKERSYMBOL=CANONICAL" per line, written by the mapping
   //--- dialog. "NONE" records a deliberate decline so the dialog never nags again.
   void              LoadUserMap(void)
     {
      if(m_userMapLoaded)
         return;
      m_userMapLoaded = true;
      int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return;
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         StringTrimLeft(line);
         StringTrimRight(line);
         if(line == "" || StringGetCharacter(line, 0) == '#')
            continue;
         string kv[];
         if(StringSplit(line, '=', kv) != 2)
            continue;
         int n = ArraySize(m_userFrom);
         ArrayResize(m_userFrom, n + 1);
         ArrayResize(m_userTo, n + 1);
         m_userFrom[n] = kv[0];
         m_userTo[n]   = kv[1];
        }
      FileClose(h);
      if(ArraySize(m_userFrom) > 0)
         PrintFormat("AltDataFetch: %d user symbol mapping(s) loaded from symbol_map.cfg.",
                     ArraySize(m_userFrom));
     }

   //--- exact, then prefix, over one row's alias list
   bool              AliasMatches(int spec, string symbol, bool prefixPass)
     {
      string a[];
      int n = StringSplit(m_specs[spec].aliases, ';', a);
      for(int i = 0; i < n; i++)
        {
         if(a[i] == "")
            continue;
         if(!prefixPass && symbol == a[i])
            return true;
         //--- require the alias to be a strict prefix AND at least 3 chars, so a short
         //--- ticker can never swallow an unrelated symbol
         if(prefixPass && StringLen(a[i]) >= 3 && StringFind(symbol, a[i]) == 0)
            return true;
        }
      return false;
     }

   int               FindSpec(string symbol)
     {
      LoadUserMap();
      //--- a user mapping wins over everything, including a catalog row for the same name
      for(int u = 0; u < ArraySize(m_userFrom); u++)
        {
         if(m_userFrom[u] != symbol)
            continue;
         if(m_userTo[u] == "NONE")
            return -2;                       // deliberately declined - do not ask again
         for(int i = 0; i < ArraySize(m_specs); i++)
            if(m_specs[i].canonical == m_userTo[u])
               return i;
         PrintFormat("AltDataFetch: symbol_map.cfg maps %s to unknown catalog entry '%s' - ignoring.",
                     symbol, m_userTo[u]);
        }
      for(int i = 0; i < ArraySize(m_specs); i++)
         if(AliasMatches(i, symbol, false))
            return i;
      for(int i = 0; i < ArraySize(m_specs); i++)
         if(AliasMatches(i, symbol, true))
            return i;
      return -1;                             // unknown - the dialog asks the user
     }

public:
                     CAltDataFetch(void) : m_fredKey(""), m_keyLoaded(false),
                                           m_eiaKey(""), m_eiaWarned(false),
                                           m_lastGexDate(0), m_gexDateLoaded(false),
                                           m_urlAlerted(false),
                                           m_userMapLoaded(false)
     {
      for(int i = 0; i < ArraySize(m_lastAttempt); i++)
         m_lastAttempt[i] = 0;
      BuildCatalog();
     }

   //--- ------------------------------------- symbol mapping interface
   bool              NeedsMapping(string symbol) { return FindSpec(symbol) == -1; }

   int               CatalogCount(void) { return ArraySize(m_specs); }
   string            CatalogName(int i)
     {
      return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i].canonical : "";
     }
   //--- one-line description for the dropdown: canonical + what it would contribute
   string            CatalogLabel(int i)
     {
      if(i < 0 || i >= ArraySize(m_specs))
         return "";
      string f[];
      int n = StringSplit(m_specs[i].features, ';', f);
      return m_specs[i].canonical + "  (" + IntegerToString(n) + " features)";
     }

   //--- Persist the user's answer and apply it immediately. canonical == "NONE" records a
   //--- deliberate decline. Appends to symbol_map.cfg, replacing any previous line for the
   //--- same broker symbol, so the answer survives restarts and a wiped feature folder.
   bool              SaveUserMapping(string symbol, string canonical)
     {
      LoadUserMap();
      bool replaced = false;
      for(int i = 0; i < ArraySize(m_userFrom); i++)
         if(m_userFrom[i] == symbol)
           {
            m_userTo[i] = canonical;
            replaced = true;
           }
      if(!replaced)
        {
         int n = ArraySize(m_userFrom);
         ArrayResize(m_userFrom, n + 1);
         ArrayResize(m_userTo, n + 1);
         m_userFrom[n] = symbol;
         m_userTo[n]   = canonical;
        }
      int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltDataFetch: cannot write symbol_map.cfg (%d) - the mapping applies to "
                     "this session only and will be asked again next attach.", GetLastError());
         return false;
        }
      FileWriteString(h, "# Warrior EA alt-data symbol map: BROKERSYMBOL=CANONICAL (NONE = no alt data)\n");
      for(int i = 0; i < ArraySize(m_userFrom); i++)
         FileWriteString(h, m_userFrom[i] + "=" + m_userTo[i] + "\n");
      FileClose(h);
      PrintFormat("AltDataFetch: %s mapped to %s (saved in symbol_map.cfg).", symbol, canonical);
      return true;
     }

   //--- Maintain the raw caches and the feature CSV for `symbol`. Returns true when the feature
   //--- file was rebuilt (caller should reload the panels).
   bool              Update(string symbol)
     {
      int si = FindSpec(symbol);
      if(si < 0)
         return false;   // unknown symbol or user declined - nothing to serve
      bool needVix = StringFind(m_specs[si].features, "vix") >= 0;
      bool needUsd = StringFind(m_specs[si].features, "usd_") >= 0;
      bool needEia = StringFind(m_specs[si].features, "eia_") >= 0;
      bool needCot = (m_specs[si].cotWhere != "");
      bool needIvol = (m_specs[si].ivolSeries != "" && StringFind(m_specs[si].features, "ivol") >= 0);
      bool needMac = StringFind(m_specs[si].features, "mac_") >= 0;
      bool changed = false;
      //--- plausibility bands: generous enough for any real print in the series' history
      //--- (VIX peaked 89 in 2008; utilization-style checks live in FeatureValue), tight
      //--- enough that transport garbage cannot pass
      if(needVix)
        {
         if(m_vix.rows == 0)
            LoadRaw("raw_VIXCLS.csv", m_vix, 1);
         if(UpdateFred("VIXCLS", m_vix, 0, 4, 1.0, 200.0))
            changed = true;
        }
      if(needUsd)
        {
         //--- staleDays 10: the H.10 dollar index posts with ~a week's lag (live fetch
         //--- 2026-08-16 was current only through 08-07), so a 4-day horizon would fire a
         //--- futile refetch attempt every hour in the gap between releases
         if(m_usd.rows == 0)
            LoadRaw("raw_DTWEXBGS.csv", m_usd, 1);
         if(UpdateFred("DTWEXBGS", m_usd, 1, 10, 50.0, 250.0))
            changed = true;
        }
      if(needCot)
        {
         //--- cache is named by CANONICAL, so two brokers' names for the same instrument
         //--- share one download instead of fetching the same contract twice
         string cache = "raw_COT_" + m_specs[si].canonical + ".csv";
         if(m_cot.rows == 0)
            LoadRaw(cache, m_cot, 3);
         if(UpdateCot(cache, m_specs[si].cotWhere, m_cot, 2))
            changed = true;
        }
      if(needEia)
        {
         if(m_eia.rows == 0)
            LoadRaw("raw_EIA_WPSR.csv", m_eia, 3);
         if(UpdateEia(m_eia, 3))
            changed = true;
        }
      if(needIvol)
        {
         if(m_ivol.rows == 0)
            LoadRaw("raw_" + m_specs[si].ivolSeries + ".csv", m_ivol, 1);
         if(UpdateFred(m_specs[si].ivolSeries, m_ivol, 4, 4, 1.0, 300.0))
            changed = true;
        }
      if(needMac)
        {
         if(m_dgs10.rows == 0)  LoadRaw("raw_DGS10.csv", m_dgs10, 1);
         if(m_curve.rows == 0)  LoadRaw("raw_T10Y2Y.csv", m_curve, 1);
         if(m_bei.rows == 0)    LoadRaw("raw_T5YIE.csv", m_bei, 1);
         if(m_dff.rows == 0)    LoadRaw("raw_DFF.csv", m_dff, 1);
         if(m_ecb.rows == 0)    LoadRaw("raw_ECBDFR.csv", m_ecb, 1);
         if(m_cpi.rows == 0)    LoadRaw("raw_CPIAUCNS.csv", m_cpi, 1);
         if(m_unrate.rows == 0) LoadRaw("raw_UNRATE.csv", m_unrate, 1);
         if(UpdateFred("DGS10", m_dgs10, 5, 4, -5.0, 30.0))      changed = true;
         if(UpdateFred("T10Y2Y", m_curve, 6, 4, -10.0, 10.0))    changed = true;
         if(UpdateFred("T5YIE", m_bei, 7, 4, -5.0, 15.0))        changed = true;
         if(UpdateFred("DFF", m_dff, 8, 4, -2.0, 30.0))          changed = true;
         if(UpdateFred("ECBDFR", m_ecb, 9, 4, -5.0, 30.0))       changed = true;
         if(UpdateFred("CPIAUCNS", m_cpi, 10, 32, 20.0, 1000.0)) changed = true;
         if(UpdateFred("UNRATE", m_unrate, 11, 32, 0.0, 35.0))   changed = true;
        }
      //--- Rebuild when stale/missing and ANY needed source has rows: a temporarily dead source
      //--- leaves its columns empty (header intact) and the panel 0-fills them - degradation must
      //--- never take the working features down with it (the silent-FRED incident took exactly that
      //--- shape under the old all-sources-required gate).
      UpdateGex(m_specs[si].canonical);
      bool anyReady = (needVix && m_vix.rows > 0) || (needUsd && m_usd.rows > 0)
                      || (needCot && m_cot.rows > 0) || (needEia && m_eia.rows > 0)
                      || (needIvol && m_ivol.rows > 0) || (needMac && m_dgs10.rows > 0);
      string outFile = AltDataFileSymbol(symbol) + "_D1.csv";   // sanitizer shared with the panel
      if((changed || !FileIsExist(ALTFETCH_DIR + outFile, FILE_COMMON)) && anyReady)
         return RebuildFeatures(si, outFile);
      return false;
     }

protected:
   //--- --------------------------------- catalog-driven feature writer Walks the daily grid and
   //--- emits exactly export.py's format; the transforms are the FIXED a-priori constants
   //--- documented there.
   bool              RebuildFeatures(int si, string outFile)
     {
      string feats[];
      int nf = StringSplit(m_specs[si].features, ';', feats);
      if(nf <= 0)
         return false;
      //--- weekly COT net-spec series with the catalog sign applied
      double cspec[];
      ArrayResize(cspec, m_cot.rows);
      for(int i = 0; i < m_cot.rows; i++)
         cspec[i] = (m_cot.v1[i] > 0) ? m_specs[si].cotSign * (m_cot.v2[i] - m_cot.v3[i]) / m_cot.v1[i] : 0.0;
      //--- EIA crude-stocks copy for the rolling percentile
      double estk[];
      ArrayResize(estk, m_eia.rows);
      for(int i = 0; i < m_eia.rows; i++)
         estk[i] = m_eia.v1[i];
      //--- temp + swap, same reasoning as SaveRaw: panels on other charts read this file
      string tmp = ALTFETCH_DIR + outFile + ".tmp";
      int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return false;
      string header = "date";
      for(int f = 0; f < nf; f++)
         header += ";" + feats[f];
      FileWriteString(h, header + "\n");
      datetime today = TimeCurrent();
      for(datetime day = ALTFETCH_GRID_START; day <= today; day += 86400)
        {
         string line = TimeToString(day, TIME_DATE);
         bool any = false;
         for(int f = 0; f < nf; f++)
           {
            string val = FeatureValue(feats[f], day, cspec, estk);
            if(val != "")
               any = true;
            line += ";" + val;
           }
         if(any)
            FileWriteString(h, line + "\n");
        }
      FileClose(h);
      if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + outFile, FILE_COMMON | FILE_REWRITE))
        {
         PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - feature file not updated.",
                     tmp, outFile, GetLastError());
         FileDelete(tmp, FILE_COMMON);
         return false;
        }
      PrintFormat("AltDataFetch: %s rebuilt (EA-maintained) - %d features: %s.",
                  outFile, nf, m_specs[si].features);
      return true;
     }

   //--- One feature value at one day; "" while its source history is not yet deep enough.
   //--- Each branch does its own as-of lookup (binary search, cheap at rebuild frequency),
   //--- so adding a source never widens a parameter list again.
   string            FeatureValue(string id, datetime day,
                                  const double &cspec[], const double &estk[])
     {
      if(id == "vix_chg5")
        {
         int i = AsOf(m_vix, day);
         return (i >= 5) ? DoubleToString((m_vix.v1[i] - m_vix.v1[i - 5]) / 10.0, 6) : "";
        }
      if(id == "vix")
        {
         int i = AsOf(m_vix, day);
         return (i >= 0) ? DoubleToString(m_vix.v1[i] / 100.0, 6) : "";
        }
      //--- the chart's OWN implied-vol index; same a-priori scales as vix/vix_chg5 so the
      //--- pairs are directly comparable to the first batch-norm layer
      if(id == "ivol_chg5")
        {
         int i = AsOf(m_ivol, day);
         return (i >= 5) ? DoubleToString((m_ivol.v1[i] - m_ivol.v1[i - 5]) / 10.0, 6) : "";
        }
      if(id == "ivol")
        {
         int i = AsOf(m_ivol, day);
         return (i >= 0) ? DoubleToString(m_ivol.v1[i] / 100.0, 6) : "";
        }
      if(id == "usd_chg5")
        {
         int i = AsOf(m_usd, day);
         return (i >= 5) ? DoubleToString(m_usd.v1[i] - m_usd.v1[i - 5], 6) : "";
        }
      if(id == "cot_spec_net")
        {
         int i = AsOf(m_cot, day);
         return (i >= 0 && m_cot.v1[i] > 0) ? DoubleToString(cspec[i], 6) : "";
        }
      if(id == "cot_idx_1y" || id == "cot_idx_3y")
        {
         int i = AsOf(m_cot, day);
         if(i < 0)
            return "";
         bool oneYear = (id == "cot_idx_1y");
         double r = RollingPctRank(cspec, i, oneYear ? 52 : 156, oneYear ? 26 : 78);
         return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
        }
      if(id == "cot_chg_4w")
        {
         int i = AsOf(m_cot, day);
         return (i >= 4) ? DoubleToString(cspec[i] - cspec[i - 4], 6) : "";
        }
      if(id == "eia_stk_idx1y")
        {
         int i = AsOf(m_eia, day);
         if(i < 0 || estk[i] <= 0)
            return "";
         double r = RollingPctRank(estk, i, 52, 26);
         return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
        }
      if(id == "eia_stk_chg4")
        {
         int i = AsOf(m_eia, day);
         return (i >= 4 && estk[i] > 0 && estk[i - 4] > 0)
                ? DoubleToString((estk[i] / estk[i - 4] - 1.0) * 10.0, 6) : "";
        }
      if(id == "eia_util")
        {
         int i = AsOf(m_eia, day);
         return (i >= 0 && m_eia.v3[i] > 0)
                ? DoubleToString((m_eia.v3[i] - 90.0) / 10.0, 6) : "";
        }
      //--- US macro block. Index arithmetic is in OBSERVATIONS (business days for the
      //--- daily series, months for CPI/UNRATE), matching the research screen exactly.
      if(id == "mac_y10")
        {
         int i = AsOf(m_dgs10, day);
         return (i >= 20) ? DoubleToString(m_dgs10.v1[i] - m_dgs10.v1[i - 20], 6) : "";
        }
      if(id == "mac_curve")
        {
         int i = AsOf(m_curve, day);
         return (i >= 0) ? DoubleToString(m_curve.v1[i], 6) : "";
        }
      if(id == "mac_bei")
        {
         int i = AsOf(m_bei, day);
         return (i >= 20) ? DoubleToString(m_bei.v1[i] - m_bei.v1[i - 20], 6) : "";
        }
      if(id == "mac_gap")
        {
         int i = AsOf(m_dff, day), j = AsOf(m_ecb, day);
         return (i >= 0 && j >= 0)
                ? DoubleToString((m_dff.v1[i] - m_ecb.v1[j]) / 10.0, 6) : "";
        }
      if(id == "mac_cpi")
        {
         int i = AsOf(m_cpi, day);
         return (i >= 12 && m_cpi.v1[i - 12] > 0)
                ? DoubleToString((m_cpi.v1[i] / m_cpi.v1[i - 12] - 1.0) * 10.0, 6) : "";
        }
      if(id == "mac_unemp")
        {
         int i = AsOf(m_unrate, day);
         return (i >= 12) ? DoubleToString((m_unrate.v1[i] - m_unrate.v1[i - 12]) / 10.0, 6) : "";
        }
      return "";
     }
  };