mnbvc188199/Warrior_EA
Watch
1
0
Fork
You've already forked Warrior_EA
 1
forked from animatedread/Warrior_EA
Code Pull requests Activity
Warrior_EA/System/AltData.mqh
AnimateDread b91c7b1f7a refactor(comments): box headers to stdlib length 
The //| box blocks were excluded from 0b06f8e and 5efdb48 and were what
remained: 160 of them ran to 10+ lines, the longest to 88. Compressed to their
leading topic sentences - 5 lines for a function header, 8 for a file header -
keeping the box format and the standard MQL5 name/author lines verbatim.

Verified at the BYTE level this time, across every in-scope file: the list of
non-comment lines is byte-identical to HEAD and braces balance. The first check
compared a locale-decoded 'git show' against a UTF-8 read and flagged 25 files
that had not changed at all - every BOM and every non-ASCII line mismatched.

47,696 -> 40,665 lines in scope; comment share 38% -> 26%.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-22 00:30:14 -04:00

308 lines
13 KiB
MQL5
Raw Permalink Blame History

//+------------------------------------------------------------------+
//| AltData.mqh |
//| AnimateDread |
//| research/altdata (Python) collects sources that are NOT |
//| derivable from any chart series - CFTC COT positioning, the VIX |
//| complex, macro series - stamps every row with the date it became |
//| PUBLICLY KNOWABLE, as-of joins them onto calendar days, and |
//| exports Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv: |
//+------------------------------------------------------------------+
#include <Math\Stat\Math.mqh>
//--- 2026-08-16: raised 16 -> 32. The wire-everything policy put FX symbols at 15
//--- features; at 16 the NEXT added column would have been silently truncated by the
//--- MathMin below - exactly the class of quiet degradation this module must never do.
#define ALTDATA_MAX_FEATURES 32
#define ALTDATA_MAX_PIN_CHARS 1024
//--- Symbol -> safe file-name token, shared by every consumer that builds a path from a broker
//--- symbol (feature files, GEX recorder, TunedPeriods).
string AltDataFileSymbol(string symbol)
{
string out = symbol;
StringReplace(out, "/", "-");
StringReplace(out, "\\", "-");
StringReplace(out, ":", "-");
StringReplace(out, "*", "-");
StringReplace(out, "?", "-");
StringReplace(out, "\"", "-");
StringReplace(out, "<", "-");
StringReplace(out, ">", "-");
StringReplace(out, "|", "-");
return out;
}
class CAltDataPanel
{
protected:
datetime m_rowTime[]; // row date (server-naive 00:00), ascending
double m_vals[]; // rows x m_fileCount, EMPTY_VALUE = blank cell
int m_rows;
int m_fileCount; // columns in the loaded file
//--- Per-file-column MEDIAN over the covered range, used to fill bars the file cannot answer
//--- for. WHY NOT ZERO, which is what this did until 2026-08-17.
double m_median[];
string m_fileNames[]; // file column names, file order
string m_pinnedNames[]; // pinned name list ("" = none): the OUTPUT contract
int m_pinnedCount;
int m_map[]; // output slot -> file column (-1 = 0-fill)
string m_loadedFile;
string m_loadedSymbol; // what Load() was CALLED with - reload uses these
ENUM_TIMEFRAMES m_loadedPeriod; // verbatim, never re-derived from the file path
datetime m_lastLoadTry;
bool m_loggedMissing;
void BuildMap(void)
{
int outCount = FeatureCount();
ArrayResize(m_map, outCount);
for(int j = 0; j < outCount; j++)
{
string want = (m_pinnedCount > 0) ? m_pinnedNames[j] : m_fileNames[j];
m_map[j] = -1;
for(int c = 0; c < m_fileCount; c++)
if(m_fileNames[c] == want)
{
m_map[j] = c;
break;
}
if(m_map[j] < 0 && m_pinnedCount > 0)
PrintFormat("AltData: pinned feature '%s' not present in %s - that input 0-fills "
"until the export carries it again.", want, m_loadedFile);
}
}
public:
CAltDataPanel(void) : m_rows(0), m_fileCount(0), m_pinnedCount(0),
m_loadedFile(""), m_loadedSymbol(""), m_loadedPeriod(PERIOD_CURRENT),
m_lastLoadTry(0), m_loggedMissing(false) {}
datetime LastDate(void) const { return m_rows > 0 ? m_rowTime[m_rows - 1] : 0; }
//--- The width contract: pinned list is the authority once set; otherwise the file decides.
int FeatureCount(void) const { return m_pinnedCount > 0 ? m_pinnedCount : m_fileCount; }
string NamesCsv(void) const
{
string csv = "";
int n = FeatureCount();
for(int j = 0; j < n; j++)
csv += (j > 0 ? "," : "") + (m_pinnedCount > 0 ? m_pinnedNames[j] : m_fileNames[j]);
return csv;
}
void SetPinnedNames(string csv)
{
m_pinnedCount = (csv == "") ? 0 : StringSplit(csv, ',', m_pinnedNames);
if(m_pinnedCount > ALTDATA_MAX_FEATURES)
{
PrintFormat("AltData: pinned list of %d exceeds the %d-feature cap - truncating. "
"This should never happen; check the .cfg.", m_pinnedCount, ALTDATA_MAX_FEATURES);
m_pinnedCount = ALTDATA_MAX_FEATURES;
}
if(m_fileCount > 0)
BuildMap();
}
//--- Parse Common\Files\Warrior_EA\AltData\{SYMBOL}_{TF}.csv, FALLING BACK to the
//--- {SYMBOL}_D1.csv the fetcher maintains.
bool Load(string symbol, ENUM_TIMEFRAMES period)
{
m_lastLoadTry = TimeCurrent();
string tf = StringSubstr(EnumToString(period), 7); // PERIOD_D1 -> D1
string fileSym = AltDataFileSymbol(symbol);
string name = "Warrior_EA\\AltData\\" + fileSym + "_" + tf + ".csv";
if(!FileIsExist(name, FILE_COMMON) && tf != "D1")
{
string daily = "Warrior_EA\\AltData\\" + fileSym + "_D1.csv";
if(FileIsExist(daily, FILE_COMMON))
{
PrintFormat("AltData: no %s - using the daily file %s (as-of join makes daily "
"alt rows valid on any timeframe).", name, daily);
name = daily;
}
}
if(!FileIsExist(name, FILE_COMMON))
{
if(!m_loggedMissing)
{
m_loggedMissing = true;
PrintFormat("AltData: no %s - external block %s.", name,
m_pinnedCount > 0 ? "0-fills its pinned width" : "disabled (0 features)");
}
return false;
}
//--- share flags unconditionally: the exporter may rewrite the file while charts hold it open
//--- (and the tester agent holds files, see the FileCopy share-flag bug).
int h = FileOpen(name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
if(h == INVALID_HANDLE)
{
PrintFormat("AltData: %s exists but FileOpen failed (%d).", name, GetLastError());
return false;
}
string header = FileReadString(h);
string cols[];
int nCols = StringSplit(header, ';', cols);
if(nCols < 2 || cols[0] != "date")
{
PrintFormat("AltData: %s header unrecognised ('%s') - ignoring file.", name, header);
FileClose(h);
return false;
}
int fileCount = MathMin(nCols - 1, ALTDATA_MAX_FEATURES);
ArrayResize(m_fileNames, fileCount);
for(int c = 0; c < fileCount; c++)
m_fileNames[c] = cols[c + 1];
int cap = 8192, rows = 0;
ArrayResize(m_rowTime, cap);
ArrayResize(m_vals, cap * fileCount);
datetime prev = 0;
while(!FileIsEnding(h))
{
string line = FileReadString(h);
if(StringLen(line) < 11)
continue;
string f[];
if(StringSplit(line, ';', f) < fileCount + 1)
continue;
datetime t = StringToTime(f[0]);
if(t <= 0 || t <= prev) // strictly ascending or the binary search breaks
continue;
if(rows >= cap)
{
cap = cap * 3 / 2;
ArrayResize(m_rowTime, cap);
ArrayResize(m_vals, cap * fileCount);
}
m_rowTime[rows] = t;
for(int c = 0; c < fileCount; c++)
{
string cell = f[c + 1];
m_vals[rows * fileCount + c] = (StringLen(cell) == 0) ? EMPTY_VALUE : StringToDouble(cell);
}
prev = t;
rows++;
}
FileClose(h);
if(rows == 0)
{
PrintFormat("AltData: %s parsed to 0 rows - ignoring file.", name);
return false;
}
m_rows = rows;
m_fileCount = fileCount;
m_loadedFile = name;
m_loadedSymbol = symbol;
m_loadedPeriod = period;
ArrayResize(m_rowTime, rows);
ArrayResize(m_vals, rows * fileCount);
//--- Column medians over the covered range, ignoring blank cells - see m_median. Cheap: a few
//--- thousand rows x ~15 columns, once per file load.
ArrayResize(m_median, fileCount);
ArrayInitialize(m_median, 0.0);
int blanks = 0;
for(int c = 0; c < fileCount; c++)
{
double col[];
ArrayResize(col, rows);
int m = 0;
for(int rr = 0; rr < rows; rr++)
{
double v = m_vals[rr * fileCount + c];
if(v != EMPTY_VALUE)
col[m++] = v;
else
blanks++;
}
if(m <= 0)
continue;
ArrayResize(col, m);
m_median[c] = MathMedian(col);
}
if(blanks > 0)
PrintFormat("AltData: %s has %d blank cells across %d rows x %d columns - these read as the"
" column MEDIAN, not as 0 (see m_median). A level series reported as 0 is an"
" impossible value, not a missing one.", name, blanks, rows, fileCount);
BuildMap();
PrintFormat("AltData: %s - %d features [%s], %d daily rows %s -> %s%s.",
name, FeatureCount(), NamesCsv(), rows,
TimeToString(m_rowTime[0], TIME_DATE), TimeToString(LastDate(), TIME_DATE),
m_pinnedCount > 0 ? " (pinned name list applied)" : "");
return true;
}
//--- As-of read: the last row dated <= t, mapped through the pin. Always fills FeatureCount()
//--- slots; anything unavailable is 0.0 (the neutral the sanitize loop would produce anyway).
void Features(datetime t, double &out[])
{
int n = FeatureCount();
ArrayResize(out, n);
ArrayInitialize(out, 0.0);
if(m_rows == 0 || n == 0)
return;
//--- BEFORE THE FILE'S FIRST ROW: median-fill, do not zero-fill. See m_median for why a zero here
//--- was a spike at an impossible value that coincided exactly with the chronological IS/OOS
//--- split. Still never rejects the bar - the block stays additive context, as designed.
if(t < m_rowTime[0])
{
for(int j = 0; j < n; j++)
{
int c = m_map[j];
if(c >= 0 && c < ArraySize(m_median))
out[j] = m_median[c];
}
return;
}
//--- ROW DATES ARE NAIVE, BAR TIMES ARE BROKER TIME. A row dated D means "public from D 00:00
//--- UTC", so the first bar of each broker day reads it the broker's offset (2-3h) early.
//--- Shorten a stamp and this becomes a leak: the buffer is the guarantee, not this
//--- comparison.
int lo = 0, hi = m_rows - 1;
while(lo < hi)
{
int mid = (lo + hi + 1) / 2;
if(m_rowTime[mid] <= t)
lo = mid;
else
hi = mid - 1;
}
int base = lo * m_fileCount;
for(int j = 0; j < n; j++)
{
int c = m_map[j];
if(c < 0)
continue;
double v = m_vals[base + c];
//--- A BLANK CELL is a gap in the collector's series, not a reading of zero - measured
//--- 2026-08-17: eia_stk_idx1y alone has 181 blanks in 6,073 rows. Same argument as the
//--- pre-coverage fill above, so the same constant.
out[j] = (v != EMPTY_VALUE) ? v
: ((c < ArraySize(m_median)) ? m_median[c] : 0.0);
}
}
//--- Live freshness: the exporter rewrites the file on its own schedule; reload when the chart
//--- has moved past the loaded data, at most once an hour. In the tester the newest bar is
//--- historical and always <= LastDate(), so this never fires there - by design.
void EnsureFresh(datetime newestBar)
{
if(m_rows == 0 || newestBar <= LastDate() + 86400)
return;
if(TimeCurrent() - m_lastLoadTry < 3600)
return;
string sym = m_loadedFile; // reload the exact file we came from
if(sym == "")
return;
PrintFormat("AltData: newest bar %s is beyond loaded data (%s) - reloading %s.",
TimeToString(newestBar, TIME_DATE), TimeToString(LastDate(), TIME_DATE), m_loadedFile);
//--- Re-parse in place; a failed reload keeps the old (stale but valid) panel.
m_lastLoadTry = TimeCurrent();
int h = FileOpen(m_loadedFile, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
if(h == INVALID_HANDLE)
return;
FileClose(h);
//--- Cheap existence probe passed; reload with the EXACT (symbol, period) Load() was called
//--- with. Never reconstruct what you already know.
if(m_loadedSymbol != "")
Load(m_loadedSymbol, m_loadedPeriod);
}
};
#undef ALTDATA_MAX_FEATURES
View git blame Copy permalink