//+------------------------------------------------------------------+ //| AltData.mqh | //| AnimateDread | //| research/altdata (Python) collects sources that are NOT | //| derivable from any chart series - CFTC COT positioning, the VIX | //| complex, macro series - stamps every row with the date it became | //| PUBLICLY KNOWABLE, as-of joins them onto calendar days, and | //| exports Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv: | //+------------------------------------------------------------------+ #include //--- 2026-08-16: raised 16 -> 32. The wire-everything policy put FX symbols at 15 //--- features; at 16 the NEXT added column would have been silently truncated by the //--- MathMin below - exactly the class of quiet degradation this module must never do. #define ALTDATA_MAX_FEATURES 32 #define ALTDATA_MAX_PIN_CHARS 1024 //--- Symbol -> safe file-name token, shared by every consumer that builds a path from a broker //--- symbol (feature files, GEX recorder, TunedPeriods). string AltDataFileSymbol(string symbol) { string out = symbol; StringReplace(out, "/", "-"); StringReplace(out, "\\", "-"); StringReplace(out, ":", "-"); StringReplace(out, "*", "-"); StringReplace(out, "?", "-"); StringReplace(out, "\"", "-"); StringReplace(out, "<", "-"); StringReplace(out, ">", "-"); StringReplace(out, "|", "-"); return out; } class CAltDataPanel { protected: datetime m_rowTime[]; // row date (server-naive 00:00), ascending double m_vals[]; // rows x m_fileCount, EMPTY_VALUE = blank cell int m_rows; int m_fileCount; // columns in the loaded file //--- Per-file-column MEDIAN over the covered range, used to fill bars the file cannot answer //--- for. WHY NOT ZERO, which is what this did until 2026-08-17. double m_median[]; string m_fileNames[]; // file column names, file order string m_pinnedNames[]; // pinned name list ("" = none): the OUTPUT contract int m_pinnedCount; int m_map[]; // output slot -> file column (-1 = 0-fill) string m_loadedFile; string m_loadedSymbol; // what Load() was CALLED with - reload uses these ENUM_TIMEFRAMES m_loadedPeriod; // verbatim, never re-derived from the file path datetime m_lastLoadTry; bool m_loggedMissing; void BuildMap(void) { int outCount = FeatureCount(); ArrayResize(m_map, outCount); for(int j = 0; j < outCount; j++) { string want = (m_pinnedCount > 0) ? m_pinnedNames[j] : m_fileNames[j]; m_map[j] = -1; for(int c = 0; c < m_fileCount; c++) if(m_fileNames[c] == want) { m_map[j] = c; break; } if(m_map[j] < 0 && m_pinnedCount > 0) PrintFormat("AltData: pinned feature '%s' not present in %s - that input 0-fills " "until the export carries it again.", want, m_loadedFile); } } public: CAltDataPanel(void) : m_rows(0), m_fileCount(0), m_pinnedCount(0), m_loadedFile(""), m_loadedSymbol(""), m_loadedPeriod(PERIOD_CURRENT), m_lastLoadTry(0), m_loggedMissing(false) {} datetime LastDate(void) const { return m_rows > 0 ? m_rowTime[m_rows - 1] : 0; } //--- The width contract: pinned list is the authority once set; otherwise the file decides. int FeatureCount(void) const { return m_pinnedCount > 0 ? m_pinnedCount : m_fileCount; } string NamesCsv(void) const { string csv = ""; int n = FeatureCount(); for(int j = 0; j < n; j++) csv += (j > 0 ? "," : "") + (m_pinnedCount > 0 ? m_pinnedNames[j] : m_fileNames[j]); return csv; } void SetPinnedNames(string csv) { m_pinnedCount = (csv == "") ? 0 : StringSplit(csv, ',', m_pinnedNames); if(m_pinnedCount > ALTDATA_MAX_FEATURES) { PrintFormat("AltData: pinned list of %d exceeds the %d-feature cap - truncating. " "This should never happen; check the .cfg.", m_pinnedCount, ALTDATA_MAX_FEATURES); m_pinnedCount = ALTDATA_MAX_FEATURES; } if(m_fileCount > 0) BuildMap(); } //--- Parse Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv, FALLING BACK to the //--- {SYMBOL}_D1.csv the fetcher maintains. bool Load(string symbol, ENUM_TIMEFRAMES period) { m_lastLoadTry = TimeCurrent(); string tf = StringSubstr(EnumToString(period), 7); // PERIOD_D1 -> D1 string fileSym = AltDataFileSymbol(symbol); string name = "Warrior_EA\\AltData\\" + fileSym + "_" + tf + ".csv"; if(!FileIsExist(name, FILE_COMMON) && tf != "D1") { string daily = "Warrior_EA\\AltData\\" + fileSym + "_D1.csv"; if(FileIsExist(daily, FILE_COMMON)) { PrintFormat("AltData: no %s - using the daily file %s (as-of join makes daily " "alt rows valid on any timeframe).", name, daily); name = daily; } } if(!FileIsExist(name, FILE_COMMON)) { if(!m_loggedMissing) { m_loggedMissing = true; PrintFormat("AltData: no %s - external block %s.", name, m_pinnedCount > 0 ? "0-fills its pinned width" : "disabled (0 features)"); } return false; } //--- share flags unconditionally: the exporter may rewrite the file while charts hold it open //--- (and the tester agent holds files, see the FileCopy share-flag bug). int h = FileOpen(name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) { PrintFormat("AltData: %s exists but FileOpen failed (%d).", name, GetLastError()); return false; } string header = FileReadString(h); string cols[]; int nCols = StringSplit(header, ';', cols); if(nCols < 2 || cols[0] != "date") { PrintFormat("AltData: %s header unrecognised ('%s') - ignoring file.", name, header); FileClose(h); return false; } int fileCount = MathMin(nCols - 1, ALTDATA_MAX_FEATURES); ArrayResize(m_fileNames, fileCount); for(int c = 0; c < fileCount; c++) m_fileNames[c] = cols[c + 1]; int cap = 8192, rows = 0; ArrayResize(m_rowTime, cap); ArrayResize(m_vals, cap * fileCount); datetime prev = 0; while(!FileIsEnding(h)) { string line = FileReadString(h); if(StringLen(line) < 11) continue; string f[]; if(StringSplit(line, ';', f) < fileCount + 1) continue; datetime t = StringToTime(f[0]); if(t <= 0 || t <= prev) // strictly ascending or the binary search breaks continue; if(rows >= cap) { cap = cap * 3 / 2; ArrayResize(m_rowTime, cap); ArrayResize(m_vals, cap * fileCount); } m_rowTime[rows] = t; for(int c = 0; c < fileCount; c++) { string cell = f[c + 1]; m_vals[rows * fileCount + c] = (StringLen(cell) == 0) ? EMPTY_VALUE : StringToDouble(cell); } prev = t; rows++; } FileClose(h); if(rows == 0) { PrintFormat("AltData: %s parsed to 0 rows - ignoring file.", name); return false; } m_rows = rows; m_fileCount = fileCount; m_loadedFile = name; m_loadedSymbol = symbol; m_loadedPeriod = period; ArrayResize(m_rowTime, rows); ArrayResize(m_vals, rows * fileCount); //--- Column medians over the covered range, ignoring blank cells - see m_median. Cheap: a few //--- thousand rows x ~15 columns, once per file load. ArrayResize(m_median, fileCount); ArrayInitialize(m_median, 0.0); int blanks = 0; for(int c = 0; c < fileCount; c++) { double col[]; ArrayResize(col, rows); int m = 0; for(int rr = 0; rr < rows; rr++) { double v = m_vals[rr * fileCount + c]; if(v != EMPTY_VALUE) col[m++] = v; else blanks++; } if(m <= 0) continue; ArrayResize(col, m); m_median[c] = MathMedian(col); } if(blanks > 0) PrintFormat("AltData: %s has %d blank cells across %d rows x %d columns - these read as the" " column MEDIAN, not as 0 (see m_median). A level series reported as 0 is an" " impossible value, not a missing one.", name, blanks, rows, fileCount); BuildMap(); PrintFormat("AltData: %s - %d features [%s], %d daily rows %s -> %s%s.", name, FeatureCount(), NamesCsv(), rows, TimeToString(m_rowTime[0], TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_pinnedCount > 0 ? " (pinned name list applied)" : ""); return true; } //--- As-of read: the last row dated <= t, mapped through the pin. Always fills FeatureCount() //--- slots; anything unavailable is 0.0 (the neutral the sanitize loop would produce anyway). void Features(datetime t, double &out[]) { int n = FeatureCount(); ArrayResize(out, n); ArrayInitialize(out, 0.0); if(m_rows == 0 || n == 0) return; //--- BEFORE THE FILE'S FIRST ROW: median-fill, do not zero-fill. See m_median for why a zero here //--- was a spike at an impossible value that coincided exactly with the chronological IS/OOS //--- split. Still never rejects the bar - the block stays additive context, as designed. if(t < m_rowTime[0]) { for(int j = 0; j < n; j++) { int c = m_map[j]; if(c >= 0 && c < ArraySize(m_median)) out[j] = m_median[c]; } return; } //--- ROW DATES ARE NAIVE, BAR TIMES ARE BROKER TIME. A row dated D means "public from D 00:00 //--- UTC", so the first bar of each broker day reads it the broker's offset (2-3h) early. //--- Shorten a stamp and this becomes a leak: the buffer is the guarantee, not this //--- comparison. int lo = 0, hi = m_rows - 1; while(lo < hi) { int mid = (lo + hi + 1) / 2; if(m_rowTime[mid] <= t) lo = mid; else hi = mid - 1; } int base = lo * m_fileCount; for(int j = 0; j < n; j++) { int c = m_map[j]; if(c < 0) continue; double v = m_vals[base + c]; //--- A BLANK CELL is a gap in the collector's series, not a reading of zero - measured //--- 2026-08-17: eia_stk_idx1y alone has 181 blanks in 6,073 rows. Same argument as the //--- pre-coverage fill above, so the same constant. out[j] = (v != EMPTY_VALUE) ? v : ((c < ArraySize(m_median)) ? m_median[c] : 0.0); } } //--- Live freshness: the exporter rewrites the file on its own schedule; reload when the chart //--- has moved past the loaded data, at most once an hour. In the tester the newest bar is //--- historical and always <= LastDate(), so this never fires there - by design. void EnsureFresh(datetime newestBar) { if(m_rows == 0 || newestBar <= LastDate() + 86400) return; if(TimeCurrent() - m_lastLoadTry < 3600) return; string sym = m_loadedFile; // reload the exact file we came from if(sym == "") return; PrintFormat("AltData: newest bar %s is beyond loaded data (%s) - reloading %s.", TimeToString(newestBar, TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_loadedFile); //--- Re-parse in place; a failed reload keeps the old (stale but valid) panel. m_lastLoadTry = TimeCurrent(); int h = FileOpen(m_loadedFile, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON); if(h == INVALID_HANDLE) return; FileClose(h); //--- Cheap existence probe passed; reload with the EXACT (symbol, period) Load() was called //--- with. Never reconstruct what you already know. if(m_loadedSymbol != "") Load(m_loadedSymbol, m_loadedPeriod); } }; #undef ALTDATA_MAX_FEATURES