Warrior_EA/System/AltDataCatalog.mqh

//+------------------------------------------------------------------+
//|                                             AltDataCatalog.mqh   |
//|                                                     AnimateDread |
//| Symbol catalog + alias resolution + user-mapping persistence.    |
//| An MT5 symbol cannot be mapped to a CFTC contract by rule - the  |
//| naming universes are unrelated and every broker invents its own  |
//| index tickers - so this catalog IS the mechanism: the chart      |
//| symbol selects a row at runtime and ONLY that row's sources are  |
//| fetched. Knows nothing about HTTP or feature math.               |
//+------------------------------------------------------------------+
#ifndef ALTDATACATALOG_MQH
#define ALTDATACATALOG_MQH

//--- one symbol catalog row: which sources feed the instrument and which feature columns its
//--- {SYM}_D1.csv carries (column order = EA input order, APPEND-ONLY per the .cfg name pin)
struct SAltSymbolSpec
  {
   string            canonical; // catalog key - also names the shared COT raw cache
   string            aliases;   // ';'-separated broker names; matched EXACT first, then as a
                                // PREFIX so suffixed variants (US500.cash, XAUUSDm, EURUSD.r)
                                // resolve without being listed one by one
   string            cotWhere;  // SoQL predicate selecting the CFTC contract; "" = no COT leg
   double            cotSign;   // +1 futures quote matches the chart; -1 inverted (JPY/CAD futures are FX/USD)
   string            features;  // ';'-separated feature ids, in CSV column order
   string            ivolSeries;// FRED id of THIS instrument's own implied-vol index; "" = none
  };

class CAltDataCatalog
  {
private:
   SAltSymbolSpec    m_specs[];
   string            m_userFrom[];   // broker symbol   (from symbol_map.cfg / the dialog)
   string            m_userTo[];     // canonical, or "NONE" = user declined alt data
   bool              m_userMapLoaded;

   void              AddSpec(string canonical, string aliases, string cotWhere,
                             double cotSign, string feats, string ivolSeries = "")
     {
      int n = ArraySize(m_specs);
      ArrayResize(m_specs, n + 1);
      m_specs[n].canonical  = canonical;
      m_specs[n].aliases    = aliases;
      m_specs[n].cotWhere   = cotWhere;
      m_specs[n].cotSign    = cotSign;
      m_specs[n].features   = feats;
      m_specs[n].ivolSeries = ivolSeries;
     }

   void              BuildCatalog(void)
     {
      //--- feature bundles (locals, deliberately prefixed so no macro elsewhere can collide) POLICY
      //--- (user directive 2026-08-16, twice): every feature the fetched sources can serve is wired
      //--- on every symbol - the research screens inform priors but do NOT gate the input list ("we
      //--- will leave the NN to do its job").
      string f_eia  = "eia_stk_idx1y;eia_stk_chg4;eia_util";
      string f_risk = "vix_chg5;vix;usd_chg5";                // equity-risk complex
      string f_cot  = "cot_idx_1y;cot_idx_3y;cot_chg_4w";     // COT positioning family
      //--- US macro block: 10y yield 20d change, curve slope, 5y breakeven 20d change, Fed-ECB
      //--- policy gap, CPI yoy, unemployment 12m change.
      string f_mac  = "mac_y10;mac_curve;mac_bei;mac_gap;mac_cpi;mac_unemp";
      //--- CFTC predicates. A predicate that matches nothing costs only an empty column.
      string w_es  = "market_and_exchange_names in('E-MINI S&P 500 - CHICAGO MERCANTILE EXCHANGE',"
                     "'E-MINI S&P 500 STOCK INDEX - CHICAGO MERCANTILE EXCHANGE')";
      string w_jpy = "market_and_exchange_names in('JAPANESE YEN - CHICAGO MERCANTILE EXCHANGE')";
      string w_eur = "market_and_exchange_names like 'EURO FX -%'";
      string w_gbp = "market_and_exchange_names like 'BRITISH POUND%'";
      string w_cad = "market_and_exchange_names like 'CANADIAN DOLLAR%'";
      string w_aud = "market_and_exchange_names like 'AUSTRALIAN DOLLAR%'";
      string w_nzd = "market_and_exchange_names like 'NZ DOLLAR%'";
      string w_chf = "market_and_exchange_names like 'SWISS FRANC%'";
      string w_btc = "market_and_exchange_names like 'BITCOIN%'";

      //--- ---------------- equity indices (VIX/USD complex; only SP500 is screened)
      //--- Screened findings preserved as priors, not gates: SP500 vix_chg5/vix/usd_chg5/
      //--- cot_spec_net and USDJPY COT family cleared family-wise + incremental; XAUUSD
      //--- gvz_chg5 (GVZ = gold's own IV) is the campaign's 2nd-strongest incremental
      //--- feature (MI|vol 0.0197 p=0.002, 4.6x vix_chg5 on gold); EURUSD/USDJPY vix_chg5
      //--- screened clean; the macro block screened null everywhere; EIA screened null on
      //--- WTI. All ship regardless - see the policy note above.
      AddSpec("SP500",  "SP500;SPX500;US500;USA500;USA500IDX;US_500;SPXUSD;S&P500;SP_500;"
                        "SPX;USA500IDXUSD;US500Cash", w_es, 1.0,
              f_risk + ";cot_spec_net;" + f_eia + ";" + f_mac);
      AddSpec("NAS100", "NAS100;NASDAQ;US100;USA100;USATEC;USTEC;NDX100;NQ100;TECH100;"
                        "USA100IDX;US_100;NDX;USATECHIDX", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXNCLS");
      AddSpec("US30",   "US30;DJ30;DOW30;USA30;WALLSTREET;WS30;DJIUSD;USA30IDX;"
                        "US_30;DOWJONES;DJI", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXDCLS");
      AddSpec("US2000", "US2000;RUSSELL2000;RUT;USA2000;RTY;US_2000", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "RVXCLS");
      AddSpec("DE40",   "DE40;GER40;DAX40;DAX;GER30;DE30;GERMANY40;DEU40;GRXEUR;"
                        "DE_40;GER_40", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("UK100",  "UK100;FTSE100;FTSE;GBR100;UKX;GB100;UK_100;BRXGBP", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("JP225",  "JP225;NIKKEI;NI225;JPN225;J225;JPXJPY;JP_225", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("EU50",   "EU50;STOXX50;ESTX50;EUSTX50;SX5E;EUR50;E50EUR", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("FR40",   "FR40;CAC40;FRA40;CAC;FRXEUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("AU200",  "AU200;ASX200;AUS200;SPI200;AU_200;AUXAUD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("HK50",   "HK50;HSI;HKG33;HK33;HANGSENG;HKXHKD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- FX majors (full complement: COT + risk complex + EIA + macro;
      //--- no per-instrument IV - EVZ was discontinued 2025-03 and must never be wired)
      AddSpec("EURUSD", "EURUSD;EUR_USD", w_eur,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDJPY", "USDJPY;USD_JPY", w_jpy, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("GBPUSD", "GBPUSD;GBP_USD", w_gbp,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDCAD", "USDCAD;USD_CAD", w_cad, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("AUDUSD", "AUDUSD;AUD_USD", w_aud,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("NZDUSD", "NZDUSD;NZD_USD", w_nzd,  1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
      AddSpec("USDCHF", "USDCHF;USD_CHF", w_chf, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- metals (gold/silver COT is in the DISAGGREGATED CFTC dataset -
      //--- different id AND column names - still not wired; silver has no free IV index)
      AddSpec("XAUUSD", "XAUUSD;GOLD;XAU_USD;GOLDUSD", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "GVZCLS");
      AddSpec("XAGUSD", "XAGUSD;SILVER;XAG_USD;SILVERUSD", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- energy
      AddSpec("XTIUSD", "XTIUSD;USOIL;WTI;CRUDE;USCRUDE;OIL;WTICOUSD;"
                        "OILUSD;LIGHTCMDUSD;CRUDEOIL", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
      AddSpec("XBRUSD", "XBRUSD;UKOIL;BRENT;UKOUSD;BCOUSD;BRENTCMDUSD", "", 1.0,
              f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
      AddSpec("NATGAS", "NATGAS;NGAS;XNGUSD;NATURALGAS", "", 1.0,
              f_risk + ";" + f_eia + ";" + f_mac);

      //--- ---------------- crypto
      AddSpec("BTCUSD", "BTCUSD;BITCOIN;XBTUSD;BTC_USD", w_btc, 1.0,
              f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
     }

   //--- symbol_map.cfg: one "BROKERSYMBOL=CANONICAL" per line, written by the mapping
   //--- dialog. "NONE" records a deliberate decline so the dialog never nags again.
   void              LoadUserMap(void)
     {
      if(m_userMapLoaded)
         return;
      m_userMapLoaded = true;
      int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
         return;
      while(!FileIsEnding(h))
        {
         string line = FileReadString(h);
         StringTrimLeft(line);
         StringTrimRight(line);
         if(line == "" || StringGetCharacter(line, 0) == '#')
            continue;
         string kv[];
         if(StringSplit(line, '=', kv) != 2)
            continue;
         int n = ArraySize(m_userFrom);
         ArrayResize(m_userFrom, n + 1);
         ArrayResize(m_userTo, n + 1);
         m_userFrom[n] = kv[0];
         m_userTo[n]   = kv[1];
        }
      FileClose(h);
      if(ArraySize(m_userFrom) > 0)
         PrintFormat("AltDataFetch: %d user symbol mapping(s) loaded from symbol_map.cfg.",
                     ArraySize(m_userFrom));
     }

   //--- exact, then prefix, over one row's alias list
   bool              AliasMatches(int spec, string symbol, bool prefixPass)
     {
      string a[];
      int n = StringSplit(m_specs[spec].aliases, ';', a);
      for(int i = 0; i < n; i++)
        {
         if(a[i] == "")
            continue;
         if(!prefixPass && symbol == a[i])
            return true;
         //--- require the alias to be a strict prefix AND at least 3 chars, so a short
         //--- ticker can never swallow an unrelated symbol
         if(prefixPass && StringLen(a[i]) >= 3 && StringFind(symbol, a[i]) == 0)
            return true;
        }
      return false;
     }

public:
                     CAltDataCatalog(void) : m_userMapLoaded(false)
     {
      BuildCatalog();
     }

   int               FindSpec(string symbol)
     {
      LoadUserMap();
      //--- a user mapping wins over everything, including a catalog row for the same name
      for(int u = 0; u < ArraySize(m_userFrom); u++)
        {
         if(m_userFrom[u] != symbol)
            continue;
         if(m_userTo[u] == "NONE")
            return -2;                       // deliberately declined - do not ask again
         for(int i = 0; i < ArraySize(m_specs); i++)
            if(m_specs[i].canonical == m_userTo[u])
               return i;
         PrintFormat("AltDataFetch: symbol_map.cfg maps %s to unknown catalog entry '%s' - ignoring.",
                     symbol, m_userTo[u]);
        }
      for(int i = 0; i < ArraySize(m_specs); i++)
         if(AliasMatches(i, symbol, false))
            return i;
      for(int i = 0; i < ArraySize(m_specs); i++)
         if(AliasMatches(i, symbol, true))
            return i;
      return -1;                             // unknown - the dialog asks the user
     }

   //--- catalog row by index, by value - callers read it read-only (4 strings + 1 double, cheap copy)
   SAltSymbolSpec    Spec(int i)
     {
      SAltSymbolSpec empty;
      return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i] : empty;
     }

   bool              NeedsMapping(string symbol) { return FindSpec(symbol) == -1; }

   int               CatalogCount(void) { return ArraySize(m_specs); }
   string            CatalogName(int i)
     {
      return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i].canonical : "";
     }
   //--- one-line description for the dropdown: canonical + what it would contribute
   string            CatalogLabel(int i)
     {
      if(i < 0 || i >= ArraySize(m_specs))
         return "";
      string f[];
      int n = StringSplit(m_specs[i].features, ';', f);
      return m_specs[i].canonical + "  (" + IntegerToString(n) + " features)";
     }

   //--- Persist the user's answer and apply it immediately. canonical == "NONE" records a
   //--- deliberate decline. Appends to symbol_map.cfg, replacing any previous line for the
   //--- same broker symbol, so the answer survives restarts and a wiped feature folder.
   bool              SaveUserMapping(string symbol, string canonical)
     {
      LoadUserMap();
      bool replaced = false;
      for(int i = 0; i < ArraySize(m_userFrom); i++)
         if(m_userFrom[i] == symbol)
           {
            m_userTo[i] = canonical;
            replaced = true;
           }
      if(!replaced)
        {
         int n = ArraySize(m_userFrom);
         ArrayResize(m_userFrom, n + 1);
         ArrayResize(m_userTo, n + 1);
         m_userFrom[n] = symbol;
         m_userTo[n]   = canonical;
        }
      int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
      if(h == INVALID_HANDLE)
        {
         PrintFormat("AltDataFetch: cannot write symbol_map.cfg (%d) - the mapping applies to "
                     "this session only and will be asked again next attach.", GetLastError());
         return false;
        }
      FileWriteString(h, "# Warrior EA alt-data symbol map: BROKERSYMBOL=CANONICAL (NONE = no alt data)\n");
      for(int i = 0; i < ArraySize(m_userFrom); i++)
         FileWriteString(h, m_userFrom[i] + "=" + m_userTo[i] + "\n");
      FileClose(h);
      PrintFormat("AltDataFetch: %s mapped to %s (saved in symbol_map.cfg).", symbol, canonical);
      return true;
     }
  };
#endif