//+------------------------------------------------------------------+ //| AltDataCatalog.mqh | //| AnimateDread | //| Symbol catalog + alias resolution + user-mapping persistence. | //| An MT5 symbol cannot be mapped to a CFTC contract by rule - the | //| naming universes are unrelated and every broker invents its own | //| index tickers - so this catalog IS the mechanism: the chart | //| symbol selects a row at runtime and ONLY that row's sources are | //| fetched. Knows nothing about HTTP or feature math. | //+------------------------------------------------------------------+ #ifndef ALTDATACATALOG_MQH #define ALTDATACATALOG_MQH //--- one symbol catalog row: which sources feed the instrument and which feature columns its //--- {SYM}_D1.csv carries (column order = EA input order, APPEND-ONLY per the .cfg name pin) struct SAltSymbolSpec { string canonical; // catalog key - also names the shared COT raw cache string aliases; // ';'-separated broker names; matched EXACT first, then as a // PREFIX so suffixed variants (US500.cash, XAUUSDm, EURUSD.r) // resolve without being listed one by one string cotWhere; // SoQL predicate selecting the CFTC contract; "" = no COT leg double cotSign; // +1 futures quote matches the chart; -1 inverted (JPY/CAD futures are FX/USD) string features; // ';'-separated feature ids, in CSV column order string ivolSeries;// FRED id of THIS instrument's own implied-vol index; "" = none }; class CAltDataCatalog { private: SAltSymbolSpec m_specs[]; string m_userFrom[]; // broker symbol (from symbol_map.cfg / the dialog) string m_userTo[]; // canonical, or "NONE" = user declined alt data bool m_userMapLoaded; void AddSpec(string canonical, string aliases, string cotWhere, double cotSign, string feats, string ivolSeries = "") { int n = ArraySize(m_specs); ArrayResize(m_specs, n + 1); m_specs[n].canonical = canonical; m_specs[n].aliases = aliases; m_specs[n].cotWhere = cotWhere; m_specs[n].cotSign = cotSign; m_specs[n].features = feats; m_specs[n].ivolSeries = ivolSeries; } void BuildCatalog(void) { //--- feature bundles (locals, deliberately prefixed so no macro elsewhere can collide) POLICY //--- (user directive 2026-08-16, twice): every feature the fetched sources can serve is wired //--- on every symbol - the research screens inform priors but do NOT gate the input list ("we //--- will leave the NN to do its job"). string f_eia = "eia_stk_idx1y;eia_stk_chg4;eia_util"; string f_risk = "vix_chg5;vix;usd_chg5"; // equity-risk complex string f_cot = "cot_idx_1y;cot_idx_3y;cot_chg_4w"; // COT positioning family //--- US macro block: 10y yield 20d change, curve slope, 5y breakeven 20d change, Fed-ECB //--- policy gap, CPI yoy, unemployment 12m change. string f_mac = "mac_y10;mac_curve;mac_bei;mac_gap;mac_cpi;mac_unemp"; //--- CFTC predicates. A predicate that matches nothing costs only an empty column. string w_es = "market_and_exchange_names in('E-MINI S&P 500 - CHICAGO MERCANTILE EXCHANGE'," "'E-MINI S&P 500 STOCK INDEX - CHICAGO MERCANTILE EXCHANGE')"; string w_jpy = "market_and_exchange_names in('JAPANESE YEN - CHICAGO MERCANTILE EXCHANGE')"; string w_eur = "market_and_exchange_names like 'EURO FX -%'"; string w_gbp = "market_and_exchange_names like 'BRITISH POUND%'"; string w_cad = "market_and_exchange_names like 'CANADIAN DOLLAR%'"; string w_aud = "market_and_exchange_names like 'AUSTRALIAN DOLLAR%'"; string w_nzd = "market_and_exchange_names like 'NZ DOLLAR%'"; string w_chf = "market_and_exchange_names like 'SWISS FRANC%'"; string w_btc = "market_and_exchange_names like 'BITCOIN%'"; //--- ---------------- equity indices (VIX/USD complex; only SP500 is screened) //--- Screened findings preserved as priors, not gates: SP500 vix_chg5/vix/usd_chg5/ //--- cot_spec_net and USDJPY COT family cleared family-wise + incremental; XAUUSD //--- gvz_chg5 (GVZ = gold's own IV) is the campaign's 2nd-strongest incremental //--- feature (MI|vol 0.0197 p=0.002, 4.6x vix_chg5 on gold); EURUSD/USDJPY vix_chg5 //--- screened clean; the macro block screened null everywhere; EIA screened null on //--- WTI. All ship regardless - see the policy note above. AddSpec("SP500", "SP500;SPX500;US500;USA500;USA500IDX;US_500;SPXUSD;S&P500;SP_500;" "SPX;USA500IDXUSD;US500Cash", w_es, 1.0, f_risk + ";cot_spec_net;" + f_eia + ";" + f_mac); AddSpec("NAS100", "NAS100;NASDAQ;US100;USA100;USATEC;USTEC;NDX100;NQ100;TECH100;" "USA100IDX;US_100;NDX;USATECHIDX", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXNCLS"); AddSpec("US30", "US30;DJ30;DOW30;USA30;WALLSTREET;WS30;DJIUSD;USA30IDX;" "US_30;DOWJONES;DJI", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXDCLS"); AddSpec("US2000", "US2000;RUSSELL2000;RUT;USA2000;RTY;US_2000", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "RVXCLS"); AddSpec("DE40", "DE40;GER40;DAX40;DAX;GER30;DE30;GERMANY40;DEU40;GRXEUR;" "DE_40;GER_40", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("UK100", "UK100;FTSE100;FTSE;GBR100;UKX;GB100;UK_100;BRXGBP", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("JP225", "JP225;NIKKEI;NI225;JPN225;J225;JPXJPY;JP_225", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("EU50", "EU50;STOXX50;ESTX50;EUSTX50;SX5E;EUR50;E50EUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("FR40", "FR40;CAC40;FRA40;CAC;FRXEUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("AU200", "AU200;ASX200;AUS200;SPI200;AU_200;AUXAUD", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); AddSpec("HK50", "HK50;HSI;HKG33;HK33;HANGSENG;HKXHKD", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); //--- ---------------- FX majors (full complement: COT + risk complex + EIA + macro; //--- no per-instrument IV - EVZ was discontinued 2025-03 and must never be wired) AddSpec("EURUSD", "EURUSD;EUR_USD", w_eur, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("USDJPY", "USDJPY;USD_JPY", w_jpy, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("GBPUSD", "GBPUSD;GBP_USD", w_gbp, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("USDCAD", "USDCAD;USD_CAD", w_cad, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("AUDUSD", "AUDUSD;AUD_USD", w_aud, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("NZDUSD", "NZDUSD;NZD_USD", w_nzd, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); AddSpec("USDCHF", "USDCHF;USD_CHF", w_chf, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); //--- ---------------- metals (gold/silver COT is in the DISAGGREGATED CFTC dataset - //--- different id AND column names - still not wired; silver has no free IV index) AddSpec("XAUUSD", "XAUUSD;GOLD;XAU_USD;GOLDUSD", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "GVZCLS"); AddSpec("XAGUSD", "XAGUSD;SILVER;XAG_USD;SILVERUSD", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); //--- ---------------- energy AddSpec("XTIUSD", "XTIUSD;USOIL;WTI;CRUDE;USCRUDE;OIL;WTICOUSD;" "OILUSD;LIGHTCMDUSD;CRUDEOIL", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS"); AddSpec("XBRUSD", "XBRUSD;UKOIL;BRENT;UKOUSD;BCOUSD;BRENTCMDUSD", "", 1.0, f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS"); AddSpec("NATGAS", "NATGAS;NGAS;XNGUSD;NATURALGAS", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac); //--- ---------------- crypto AddSpec("BTCUSD", "BTCUSD;BITCOIN;XBTUSD;BTC_USD", w_btc, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac); } //--- symbol_map.cfg: one "BROKERSYMBOL=CANONICAL" per line, written by the mapping //--- dialog. "NONE" records a deliberate decline so the dialog never nags again. void LoadUserMap(void) { if(m_userMapLoaded) return; m_userMapLoaded = true; int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) return; while(!FileIsEnding(h)) { string line = FileReadString(h); StringTrimLeft(line); StringTrimRight(line); if(line == "" || StringGetCharacter(line, 0) == '#') continue; string kv[]; if(StringSplit(line, '=', kv) != 2) continue; int n = ArraySize(m_userFrom); ArrayResize(m_userFrom, n + 1); ArrayResize(m_userTo, n + 1); m_userFrom[n] = kv[0]; m_userTo[n] = kv[1]; } FileClose(h); if(ArraySize(m_userFrom) > 0) PrintFormat("AltDataFetch: %d user symbol mapping(s) loaded from symbol_map.cfg.", ArraySize(m_userFrom)); } //--- exact, then prefix, over one row's alias list bool AliasMatches(int spec, string symbol, bool prefixPass) { string a[]; int n = StringSplit(m_specs[spec].aliases, ';', a); for(int i = 0; i < n; i++) { if(a[i] == "") continue; if(!prefixPass && symbol == a[i]) return true; //--- require the alias to be a strict prefix AND at least 3 chars, so a short //--- ticker can never swallow an unrelated symbol if(prefixPass && StringLen(a[i]) >= 3 && StringFind(symbol, a[i]) == 0) return true; } return false; } public: CAltDataCatalog(void) : m_userMapLoaded(false) { BuildCatalog(); } int FindSpec(string symbol) { LoadUserMap(); //--- a user mapping wins over everything, including a catalog row for the same name for(int u = 0; u < ArraySize(m_userFrom); u++) { if(m_userFrom[u] != symbol) continue; if(m_userTo[u] == "NONE") return -2; // deliberately declined - do not ask again for(int i = 0; i < ArraySize(m_specs); i++) if(m_specs[i].canonical == m_userTo[u]) return i; PrintFormat("AltDataFetch: symbol_map.cfg maps %s to unknown catalog entry '%s' - ignoring.", symbol, m_userTo[u]); } for(int i = 0; i < ArraySize(m_specs); i++) if(AliasMatches(i, symbol, false)) return i; for(int i = 0; i < ArraySize(m_specs); i++) if(AliasMatches(i, symbol, true)) return i; return -1; // unknown - the dialog asks the user } //--- catalog row by index, by value - callers read it read-only (4 strings + 1 double, cheap copy) SAltSymbolSpec Spec(int i) { SAltSymbolSpec empty; return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i] : empty; } bool NeedsMapping(string symbol) { return FindSpec(symbol) == -1; } int CatalogCount(void) { return ArraySize(m_specs); } string CatalogName(int i) { return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i].canonical : ""; } //--- one-line description for the dropdown: canonical + what it would contribute string CatalogLabel(int i) { if(i < 0 || i >= ArraySize(m_specs)) return ""; string f[]; int n = StringSplit(m_specs[i].features, ';', f); return m_specs[i].canonical + " (" + IntegerToString(n) + " features)"; } //--- Persist the user's answer and apply it immediately. canonical == "NONE" records a //--- deliberate decline. Appends to symbol_map.cfg, replacing any previous line for the //--- same broker symbol, so the answer survives restarts and a wiped feature folder. bool SaveUserMapping(string symbol, string canonical) { LoadUserMap(); bool replaced = false; for(int i = 0; i < ArraySize(m_userFrom); i++) if(m_userFrom[i] == symbol) { m_userTo[i] = canonical; replaced = true; } if(!replaced) { int n = ArraySize(m_userFrom); ArrayResize(m_userFrom, n + 1); ArrayResize(m_userTo, n + 1); m_userFrom[n] = symbol; m_userTo[n] = canonical; } int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) { PrintFormat("AltDataFetch: cannot write symbol_map.cfg (%d) - the mapping applies to " "this session only and will be asked again next attach.", GetLastError()); return false; } FileWriteString(h, "# Warrior EA alt-data symbol map: BROKERSYMBOL=CANONICAL (NONE = no alt data)\n"); for(int i = 0; i < ArraySize(m_userFrom); i++) FileWriteString(h, m_userFrom[i] + "=" + m_userTo[i] + "\n"); FileClose(h); PrintFormat("AltDataFetch: %s mapped to %s (saved in symbol_map.cfg).", symbol, canonical); return true; } }; #endif