//+------------------------------------------------------------------+ //| AltData.mqh | //| AnimateDread | //+------------------------------------------------------------------+ //| Externally collected, publication-stamped feature panel. | //| | //| research/altdata (Python) collects sources that are NOT derivable | //| from any chart series - CFTC COT positioning, the VIX complex, | //| macro series - stamps every row with the date it became PUBLICLY | //| KNOWABLE, as-of joins them onto calendar days, and exports | //| Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv: | //| | //| date;vix_chg5;vix;usd_chg5;cot_spec_net | //| 2010.07.24;0.194000;0.239800;... | //| | //| Only features that survived BOTH the family-wise MI bar AND the | //| incremental (conditional-on-trailing-range) test are exported, so | //| the per-symbol sets differ - see research/altdata/DESIGN.md and | //| the screen results (commits 4a56d8d, 6d50e63). | //| | //| LOOKAHEAD CONTRACT: a row dated D contains only values published | //| by D 00:00 UTC (the collectors' stamps carry their own >=1-day | //| conservative buffers on top - COT: Tuesday report stamped to the | //| Saturday after its Friday release). Features(t) returns the last | //| row dated <= t, so a bar can never read a value the live run | //| would not have had. The same files serve training, the tester and | //| live - history is reproducible by construction. | //| | //| DEGRADATION CONTRACT (mirrors CCrossAssetPanel): a missing file, | //| a gap, a pinned column the file no longer carries - all 0-fill. | //| The block is additive context; it must never reject a bar or | //| block a trade. The EA runs without Python at all times: these are | //| plain files, refreshed externally, stale-tolerated here. | //| | //| PINNING (mirrors m_crossAssetPairsPinned): the trained model's | //| .cfg records the NAME LIST its inputs were built from. On resume | //| the pin is applied BEFORE the width sum, so a file that gained | //| columns neither shifts existing slots nor changes the width - | //| pinned columns are looked up by NAME in the current file. | //+------------------------------------------------------------------+ //--- 2026-08-16: raised 16 -> 32. The wire-everything policy put FX symbols at 15 //--- features; at 16 the NEXT added column would have been silently truncated by the //--- MathMin below - exactly the class of quiet degradation this module must never do. #define ALTDATA_MAX_FEATURES 32 #define ALTDATA_MAX_PIN_CHARS 1024 //--- Symbol -> safe file-name token, shared by every consumer that builds a path from a //--- broker symbol (feature files, GEX recorder, TunedPeriods). Broker names can carry //--- path-hostile characters ("EUR/USD", "#AAPL" variants, exotic suffixes); writing //--- through them either fails or lands in an unintended subfolder. Deterministic and //--- collision-tolerant: two symbols differing only in a hostile character map together, //--- which is acceptable (they would be the same instrument at any sane broker). string AltDataFileSymbol(string symbol) { string out = symbol; StringReplace(out, "/", "-"); StringReplace(out, "\\", "-"); StringReplace(out, ":", "-"); StringReplace(out, "*", "-"); StringReplace(out, "?", "-"); StringReplace(out, "\"", "-"); StringReplace(out, "<", "-"); StringReplace(out, ">", "-"); StringReplace(out, "|", "-"); return out; } class CAltDataPanel { protected: datetime m_rowTime[]; // row date (server-naive 00:00), ascending double m_vals[]; // rows x m_fileCount, EMPTY_VALUE = blank cell int m_rows; int m_fileCount; // columns in the loaded file //--- Per-file-column MEDIAN over the covered range, used to fill bars the file cannot answer for. //--- WHY NOT ZERO, which is what this did until 2026-08-17. Half these columns are LEVELS - vix, //--- ivol, mac_y10, mac_cpi, mac_unemp, eia_util - and for a level, 0 is not "no information", it //--- is an impossible reading far outside the series' real range (VIX does not visit zero). The //--- 0-fill therefore did not degrade the block, it injected a spike at an extreme value. //--- //--- And the spike is not randomly placed: every alt file starts 2010-01-01 while the charts run //--- much deeper (USDJPY H4 reaches ~1994, roughly HALF its history), so "alt block is all zeros" //--- is exactly the predicate "this bar is older than 2010". The IS/OOS split is chronological, so //--- that predicate is ~half of IS and none of OOS: a feature the model can learn in-sample that //--- is guaranteed to be useless out-of-sample, plus a bimodal input for the first BatchNorm to //--- normalise over. Not a lookahead leak - a distribution corruption, which is quieter. //--- //--- A CONSTANT cannot leak, whatever its source: it takes the same value on every pre-coverage //--- bar, so it carries no information about which of those bars won. That is what makes filling //--- with a median computed over the covered (later) range legitimate here even though the median //--- is "future" relative to the bars it fills - it is one number, not a per-bar signal. The //--- median rather than the mean because these series are skewed and a mean would sit off-centre. double m_median[]; string m_fileNames[]; // file column names, file order string m_pinnedNames[]; // pinned name list ("" = none): the OUTPUT contract int m_pinnedCount; int m_map[]; // output slot -> file column (-1 = 0-fill) string m_loadedFile; string m_loadedSymbol; // what Load() was CALLED with - reload uses these ENUM_TIMEFRAMES m_loadedPeriod; // verbatim, never re-derived from the file path datetime m_lastLoadTry; bool m_loggedMissing; void BuildMap(void) { int outCount = FeatureCount(); ArrayResize(m_map, outCount); for(int j = 0; j < outCount; j++) { string want = (m_pinnedCount > 0) ? m_pinnedNames[j] : m_fileNames[j]; m_map[j] = -1; for(int c = 0; c < m_fileCount; c++) if(m_fileNames[c] == want) { m_map[j] = c; break; } if(m_map[j] < 0 && m_pinnedCount > 0) PrintFormat("AltData: pinned feature '%s' not present in %s - that input 0-fills " "until the export carries it again.", want, m_loadedFile); } } public: CAltDataPanel(void) : m_rows(0), m_fileCount(0), m_pinnedCount(0), m_loadedFile(""), m_loadedSymbol(""), m_loadedPeriod(PERIOD_CURRENT), m_lastLoadTry(0), m_loggedMissing(false) {} bool IsLoaded(void) const { return m_rows > 0; } datetime LastDate(void) const { return m_rows > 0 ? m_rowTime[m_rows - 1] : 0; } //--- The width contract: pinned list is the authority once set; otherwise the file decides. int FeatureCount(void) const { return m_pinnedCount > 0 ? m_pinnedCount : m_fileCount; } string NamesCsv(void) const { string csv = ""; int n = FeatureCount(); for(int j = 0; j < n; j++) csv += (j > 0 ? "," : "") + (m_pinnedCount > 0 ? m_pinnedNames[j] : m_fileNames[j]); return csv; } void SetPinnedNames(string csv) { m_pinnedCount = (csv == "") ? 0 : StringSplit(csv, ',', m_pinnedNames); if(m_pinnedCount > ALTDATA_MAX_FEATURES) { PrintFormat("AltData: pinned list of %d exceeds the %d-feature cap - truncating. " "This should never happen; check the .cfg.", m_pinnedCount, ALTDATA_MAX_FEATURES); m_pinnedCount = ALTDATA_MAX_FEATURES; } if(m_fileCount > 0) BuildMap(); } //--- Parse Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv, FALLING BACK to the //--- {SYMBOL}_D1.csv the fetcher maintains. The daily file is timeframe-agnostic by //--- construction: rows are as-of daily values joined by published <= bar open, and that //--- join is exactly what Features() does per bar - an H4 bar simply reads the same //--- daily row its calendar day maps to. (The 2026-08-16 H4 attach trained with ZERO //--- alt features because this fallback did not exist and only _D1 files are written.) //--- A per-TF file, if some future exporter writes one, still takes precedence. //--- Missing BOTH is a NORMAL state (symbol with no catalog row, collector not yet run): //--- the panel just reports 0 features / keeps its pinned width and 0-fills. bool Load(string symbol, ENUM_TIMEFRAMES period) { m_lastLoadTry = TimeCurrent(); string tf = StringSubstr(EnumToString(period), 7); // PERIOD_D1 -> D1 string fileSym = AltDataFileSymbol(symbol); string name = "Warrior_EA\\AltData\\" + fileSym + "_" + tf + ".csv"; if(!FileIsExist(name, FILE_COMMON) && tf != "D1") { string daily = "Warrior_EA\\AltData\\" + fileSym + "_D1.csv"; if(FileIsExist(daily, FILE_COMMON)) { PrintFormat("AltData: no %s - using the daily file %s (as-of join makes daily " "alt rows valid on any timeframe).", name, daily); name = daily; } } if(!FileIsExist(name, FILE_COMMON)) { if(!m_loggedMissing) { m_loggedMissing = true; PrintFormat("AltData: no %s - external block %s.", name, m_pinnedCount > 0 ? "0-fills its pinned width" : "disabled (0 features)"); } return false; } //--- share flags unconditionally: the exporter may rewrite the file while charts hold it open //--- (and the tester agent holds files, see the FileCopy share-flag bug). int h = FileOpen(name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) { PrintFormat("AltData: %s exists but FileOpen failed (%d).", name, GetLastError()); return false; } string header = FileReadString(h); string cols[]; int nCols = StringSplit(header, ';', cols); if(nCols < 2 || cols[0] != "date") { PrintFormat("AltData: %s header unrecognised ('%s') - ignoring file.", name, header); FileClose(h); return false; } int fileCount = MathMin(nCols - 1, ALTDATA_MAX_FEATURES); ArrayResize(m_fileNames, fileCount); for(int c = 0; c < fileCount; c++) m_fileNames[c] = cols[c + 1]; int cap = 8192, rows = 0; ArrayResize(m_rowTime, cap); ArrayResize(m_vals, cap * fileCount); datetime prev = 0; while(!FileIsEnding(h)) { string line = FileReadString(h); if(StringLen(line) < 11) continue; string f[]; if(StringSplit(line, ';', f) < fileCount + 1) continue; datetime t = StringToTime(f[0]); if(t <= 0 || t <= prev) // strictly ascending or the binary search breaks continue; if(rows >= cap) { cap = cap * 3 / 2; ArrayResize(m_rowTime, cap); ArrayResize(m_vals, cap * fileCount); } m_rowTime[rows] = t; for(int c = 0; c < fileCount; c++) { string cell = f[c + 1]; m_vals[rows * fileCount + c] = (StringLen(cell) == 0) ? EMPTY_VALUE : StringToDouble(cell); } prev = t; rows++; } FileClose(h); if(rows == 0) { PrintFormat("AltData: %s parsed to 0 rows - ignoring file.", name); return false; } m_rows = rows; m_fileCount = fileCount; m_loadedFile = name; m_loadedSymbol = symbol; m_loadedPeriod = period; ArrayResize(m_rowTime, rows); ArrayResize(m_vals, rows * fileCount); //--- Column medians over the covered range, ignoring blank cells - see m_median. Cheap: a few //--- thousand rows x ~15 columns, once per file load. ArrayResize(m_median, fileCount); ArrayInitialize(m_median, 0.0); int blanks = 0; for(int c = 0; c < fileCount; c++) { double col[]; ArrayResize(col, rows); int m = 0; for(int rr = 0; rr < rows; rr++) { double v = m_vals[rr * fileCount + c]; if(v != EMPTY_VALUE) col[m++] = v; else blanks++; } if(m <= 0) continue; ArrayResize(col, m); ArraySort(col); m_median[c] = (m % 2 == 1) ? col[m / 2] : 0.5 * (col[m / 2 - 1] + col[m / 2]); } if(blanks > 0) PrintFormat("AltData: %s has %d blank cells across %d rows x %d columns - these read as the" " column MEDIAN, not as 0 (see m_median). A level series reported as 0 is an" " impossible value, not a missing one.", name, blanks, rows, fileCount); BuildMap(); PrintFormat("AltData: %s - %d features [%s], %d daily rows %s -> %s%s.", name, FeatureCount(), NamesCsv(), rows, TimeToString(m_rowTime[0], TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_pinnedCount > 0 ? " (pinned name list applied)" : ""); return true; } //--- As-of read: the last row dated <= t, mapped through the pin. Always fills FeatureCount() //--- slots; anything unavailable is 0.0 (the neutral the sanitize loop would produce anyway). void Features(datetime t, double &out[]) { int n = FeatureCount(); ArrayResize(out, n); ArrayInitialize(out, 0.0); if(m_rows == 0 || n == 0) return; //--- BEFORE THE FILE'S FIRST ROW: median-fill, do not zero-fill. See m_median for why a zero here //--- was a spike at an impossible value that coincided exactly with the chronological IS/OOS //--- split. Still never rejects the bar - the block stays additive context, as designed. if(t < m_rowTime[0]) { for(int j = 0; j < n; j++) { int c = m_map[j]; if(c >= 0 && c < ArraySize(m_median)) out[j] = m_median[c]; } return; } int lo = 0, hi = m_rows - 1; while(lo < hi) { int mid = (lo + hi + 1) / 2; if(m_rowTime[mid] <= t) lo = mid; else hi = mid - 1; } int base = lo * m_fileCount; for(int j = 0; j < n; j++) { int c = m_map[j]; if(c < 0) continue; double v = m_vals[base + c]; //--- A BLANK CELL is a gap in the collector's series, not a reading of zero - measured //--- 2026-08-17: eia_stk_idx1y alone has 181 blanks in 6,073 rows. Same argument as the //--- pre-coverage fill above, so the same constant. out[j] = (v != EMPTY_VALUE) ? v : ((c < ArraySize(m_median)) ? m_median[c] : 0.0); } } //--- Live freshness: the exporter rewrites the file on its own schedule; reload when the chart //--- has moved past the loaded data, at most once an hour. In the tester the newest bar is //--- historical and always <= LastDate(), so this never fires there - by design. void EnsureFresh(datetime newestBar) { if(m_rows == 0 || newestBar <= LastDate() + 86400) return; if(TimeCurrent() - m_lastLoadTry < 3600) return; string sym = m_loadedFile; // reload the exact file we came from if(sym == "") return; PrintFormat("AltData: newest bar %s is beyond loaded data (%s) - reloading %s.", TimeToString(newestBar, TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_loadedFile); //--- Re-parse in place; a failed reload keeps the old (stale but valid) panel. m_lastLoadTry = TimeCurrent(); int h = FileOpen(m_loadedFile, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) return; FileClose(h); //--- Cheap existence probe passed; reload with the EXACT (symbol, period) Load() was //--- called with. The previous version re-derived them by splitting the file path on //--- its first '_' - which mis-parsed every symbol containing an underscore (OANDA's //--- EUR_USD, US_500 are in the alias catalog) and only knew three timeframes. Never //--- reconstruct what you already know. if(m_loadedSymbol != "") Load(m_loadedSymbol, m_loadedPeriod); } }; #undef ALTDATA_MAX_FEATURES