forked from mnbvc188199/Warrior_EA
ad80e0bb57
Two things, one of which was a compile error.
1. ExitPolicy() was declared in the protected block but is pushed in from
Warrior_EA.mq5:770. Moved to public beside the other EA-facing setters.
2. Chart objects surviving OnDeinit. The 4,500 ms teardown budget is measured
from the STOP REQUEST, not from OnDeinit's first line, and OnDeinit cannot
begin until whatever is in flight returns - so a scan still running after
_StopFlag is raised does not delay the cleanup, it SPENDS it, and the purge
never gets its turn.
New CExpertSignalAIBase::ShutdownRequested() = IsStopped() || m_shutdownInProgress.
Deliberately NOT m_trainingStopRequested: that latches, and a latched flag
would permanently disable scans that must run again on the next Start.
Guarded, longest first:
- TuneIndicatorsByFilter - per candidate, restoring the OPERATOR's settings
on the way out (best[] is mutated in place; the tuner otherwise keeps the
last trial's parameters, which nothing chose).
- ReportBarrierGeometryScan - per pairing, breaking to ONE restore point so
m_barrierScanLiveLabels can never be left true (that makes ComputeLabelForBar
read the last candidate's multiples as the configured geometry).
- ReportFeatureLabelInformation / ReportExcursionInformation / lag profile -
nulls ABANDON rather than truncate: fewer draws is not a smaller null, it
is a wrong one, and p shifts toward significance. m_dirEvidence staying
false is the safe direction.
- SimulateExitPolicyOutcomes - zeroes its accumulators so the divergence line
is dropped instead of latching a partial expectancy as the run's only report.
- ReportGeometryExpectancyScan - per ladder rung.
- HttpGet - one choke point for up to a dozen blocking WebRequests per
first-pass Update(). An in-flight request cannot be cancelled; refusing to
start another is the whole remedy.
- PollTraining, OnChartEventHandler's study event, TuneIndicatorsAndTrain -
entry points, so a queued event cannot open an era during teardown.
TuneIndicatorsAndTrain's guard is the first statement, ahead of the
m_tuneFilterDone / g_ensembleChartTuneDone latches.
- OnTick / OnTimer / OnChartEvent.
Training's own bar loops already honoured this (pass 1 per bar, passes 2/2.5/3
yield on a 120 ms budget); the warm-up scans did not, and they are the longest
uninterruptible stretches the EA has.
StopTraining() is unchanged: the operator's Stop still finalises synchronously.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1532 lines
71 KiB
MQL5
1532 lines
71 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| AltDataFetch.mqh |
|
|
//| AnimateDread |
|
|
//+------------------------------------------------------------------+
|
|
//| Self-sufficient alt-data maintenance: the EA downloads its own |
|
|
//| missing history at attach time and keeps appending forward while |
|
|
//| deployed, so online learning never depends on an external Python |
|
|
//| process. research/altdata remains the RESEARCH side (screening, |
|
|
//| new-source adjudication); this module maintains the exact same |
|
|
//| files in production. |
|
|
//| |
|
|
//| The fetch layer is TABLE-DRIVEN (symbol catalog below): the |
|
|
//| attached chart's symbol selects a catalog row, and only that |
|
|
//| row's sources are downloaded. Adding a symbol = one catalog row. |
|
|
//| |
|
|
//| Sources: |
|
|
//| CFTC Socrata API (no key): publicreporting.cftc.gov |
|
|
//| gpe5-46if = Traders in Financial Futures, futures only |
|
|
//| FRED API (free key): api.stlouisfed.org |
|
|
//| VIXCLS, DTWEXBGS - unrevised market series, so current |
|
|
//| values ARE first prints (the vintage question is settled |
|
|
//| research-side; revised series must not be added here without |
|
|
//| an ALFRED story). |
|
|
//| EIA v2 API (free key): api.eia.gov |
|
|
//| Weekly Petroleum Status Report - crude stocks ex-SPR, field |
|
|
//| production, refinery utilization. |
|
|
//| |
|
|
//| PUBLICATION STAMPS (identical rules to research/altdata): |
|
|
//| COT: report is as-of Tuesday, released Friday 15:30 ET -> |
|
|
//| published = the Saturday 00:00 after the report date. |
|
|
//| FRED daily: published = observation date + 1 day. |
|
|
//| EIA weekly: covers week ending Friday, released Wednesday |
|
|
//| 10:30 ET -> published = observed + 6 days (Thursday 00:00 |
|
|
//| UTC, holiday-safe) - mirrors research/altdata/eia.py. |
|
|
//| A feature row dated D is built only from values published <= D, |
|
|
//| so the backfilled history equals what a live run would have seen. |
|
|
//| |
|
|
//| REQUIREMENTS: Tools > Options > Expert Advisors > "Allow |
|
|
//| WebRequest for listed URL" must contain: |
|
|
//| https://publicreporting.cftc.gov |
|
|
//| https://api.stlouisfed.org |
|
|
//| https://api.eia.gov |
|
|
//| FRED key: FredApiKey input (default carries the owner's key). |
|
|
//| A keys.txt line "fred=<key>" in Common\Files\Warrior_EA\AltData\ |
|
|
//| is the fallback when the input is blanked. |
|
|
//| WebRequest is unavailable in the tester by platform |
|
|
//| rule - tester runs read whatever the cache files already hold, |
|
|
//| which is exactly the reproducibility contract. |
|
|
//+------------------------------------------------------------------+
|
|
#include "AltData.mqh"
|
|
|
|
#define ALTFETCH_DIR "Warrior_EA\\AltData\\"
|
|
#define ALTFETCH_TIMEOUT_MS 15000
|
|
//--- the SPX option chain is ~13 MB; the small feeds' 15 s would time out on a slow link
|
|
#define ALTFETCH_GEX_TIMEOUT_MS 90000
|
|
#define ALTFETCH_RETRY_SECONDS 3600
|
|
#define ALTFETCH_GRID_START D'2010.01.01'
|
|
|
|
//--- one raw series: weekly COT positioning, a daily FRED series, or the EIA WPSR block
|
|
struct SAltRawSeries
|
|
{
|
|
datetime date[]; // observation date, ascending
|
|
datetime published[];
|
|
double v1[]; // FRED: value | COT: open interest | EIA: crude stocks ex-SPR
|
|
double v2[]; // COT: spec long | EIA: field production
|
|
double v3[]; // COT: spec short | EIA: refinery utilization %
|
|
int rows;
|
|
};
|
|
|
|
//--- one symbol catalog row: which sources feed the instrument and which feature columns its
|
|
//--- {SYM}_D1.csv carries (column order = EA input order, APPEND-ONLY per the .cfg name pin)
|
|
struct SAltSymbolSpec
|
|
{
|
|
string canonical; // catalog key - also names the shared COT raw cache
|
|
string aliases; // ';'-separated broker names; matched EXACT first, then as a
|
|
// PREFIX so suffixed variants (US500.cash, XAUUSDm, EURUSD.r)
|
|
// resolve without being listed one by one
|
|
string cotWhere; // SoQL predicate selecting the CFTC contract; "" = no COT leg
|
|
double cotSign; // +1 futures quote matches the chart; -1 inverted (JPY/CAD futures are FX/USD)
|
|
string features; // ';'-separated feature ids, in CSV column order
|
|
string ivolSeries;// FRED id of THIS instrument's own implied-vol index; "" = none
|
|
};
|
|
|
|
class CAltDataFetch
|
|
{
|
|
protected:
|
|
string m_fredKey;
|
|
bool m_keyLoaded;
|
|
string m_eiaKey;
|
|
bool m_eiaWarned;
|
|
datetime m_lastGexDate; // last UTC day whose GEX row is already on disk
|
|
bool m_gexDateLoaded;
|
|
string m_blockedHost[]; // 4014 backoff, PER HOST - one missing whitelist entry
|
|
datetime m_blockedUntil[]; // must not silence the hosts that are whitelisted
|
|
bool m_urlAlerted; // full four-line reference printed once per session
|
|
datetime m_lastAttempt[16]; // per-source retry throttle: 0=VIX 1=USD 2=COT 3=EIA 4=IVOL
|
|
// 5=DGS10 6=T10Y2Y 7=T5YIE 8=DFF 9=ECBDFR 10=CPI 11=UNRATE
|
|
|
|
//--- ---------------------------------------------------------- utils
|
|
//--- The FredApiKey input is the key's home - it travels with the EA and survives the
|
|
//--- Common\Files wipe that starts every fresh test (the 2026-08-16 incident: keys.txt
|
|
//--- was wiped with the folder, FredKey() returned "" with NO log line, COT updated but
|
|
//--- FRED never ran and the SP500 rebuild - gated on all three raws - never fired).
|
|
//--- keys.txt is only the fallback for a blanked input, re-read on every hourly-throttled
|
|
//--- attempt until a key is found; m_keyLoaded means "warned once", never fail silently.
|
|
string FredKey(void)
|
|
{
|
|
if(FredApiKey != "")
|
|
return FredApiKey;
|
|
if(m_fredKey != "")
|
|
return m_fredKey;
|
|
int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
if(!m_keyLoaded)
|
|
Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt "
|
|
"does not exist in Common\\Files - FRED features (VIX, USD index) cannot update, "
|
|
"so the SP500/XAUUSD feature files will not build. Set the FredApiKey input "
|
|
"(free key: fred.stlouisfed.org) or create keys.txt with one line: fred=<key>. "
|
|
"The file is re-checked hourly; no restart needed.");
|
|
m_keyLoaded = true;
|
|
return "";
|
|
}
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringFind(line, "fred=") == 0)
|
|
m_fredKey = StringSubstr(line, 5);
|
|
}
|
|
FileClose(h);
|
|
if(m_fredKey == "" && !m_keyLoaded)
|
|
Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt has "
|
|
"no 'fred=<key>' line - FRED-based features will not update until one is provided "
|
|
"(free key: fred.stlouisfed.org).");
|
|
m_keyLoaded = true;
|
|
return m_fredKey;
|
|
}
|
|
|
|
//--- same contract as FredKey: input first, keys.txt "eia=" line as fallback, loud once
|
|
string EiaKey(void)
|
|
{
|
|
if(EiaApiKey != "")
|
|
return EiaApiKey;
|
|
if(m_eiaKey != "")
|
|
return m_eiaKey;
|
|
int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h != INVALID_HANDLE)
|
|
{
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringFind(line, "eia=") == 0)
|
|
m_eiaKey = StringSubstr(line, 4);
|
|
}
|
|
FileClose(h);
|
|
}
|
|
if(m_eiaKey == "" && !m_eiaWarned)
|
|
{
|
|
m_eiaWarned = true;
|
|
Print("AltDataFetch: the EiaApiKey input is blank and no 'eia=<key>' line exists in " +
|
|
ALTFETCH_DIR + "keys.txt - the EIA feature columns will stay empty (panel 0-fills; "
|
|
"free key: eia.gov/opendata). Re-checked hourly; no restart needed.");
|
|
}
|
|
return m_eiaKey;
|
|
}
|
|
|
|
//--- "https://host/..." -> "https://host" (what the MT5 whitelist dialog wants)
|
|
string HostOf(string url)
|
|
{
|
|
int p = StringFind(url, "://");
|
|
int q = StringFind(url, "/", p < 0 ? 0 : p + 3);
|
|
return (q < 0) ? url : StringSubstr(url, 0, q);
|
|
}
|
|
|
|
//--- URL safe for the JOURNAL: api_key values masked. The 2026-08-16 EIA failure
|
|
//--- printed the first 80 chars of the URL - which included most of the key.
|
|
//--- Every error path that echoes a URL must go through this.
|
|
string MaskUrl(string url)
|
|
{
|
|
int p = StringFind(url, "api_key=");
|
|
if(p < 0)
|
|
return StringSubstr(url, 0, 100);
|
|
int e = StringFind(url, "&", p);
|
|
string masked = StringSubstr(url, 0, p + 8) + "***";
|
|
if(e > 0)
|
|
masked += StringSubstr(url, e);
|
|
return StringSubstr(masked, 0, 100);
|
|
}
|
|
|
|
int BlockedIndex(string host)
|
|
{
|
|
for(int i = 0; i < ArraySize(m_blockedHost); i++)
|
|
if(m_blockedHost[i] == host)
|
|
return i;
|
|
return -1;
|
|
}
|
|
|
|
bool HttpGet(string url, string &body, int timeoutMs = ALTFETCH_TIMEOUT_MS)
|
|
{
|
|
//--- PER-HOST backoff. 4014 is a per-URL verdict: one missing whitelist entry must
|
|
//--- not silence the hosts that ARE whitelisted (the 2026-08-16 incident: the user
|
|
//--- whitelisted the original hosts, a later build added new ones, and the old
|
|
//--- GLOBAL backoff suppressed every working source for an hour per miss while the
|
|
//--- alert never named the offender).
|
|
//--- THE ONE CHOKE POINT every fetch passes through, and the reason it is guarded here rather than
|
|
//--- at each caller: a first-pass Update() issues up to a dozen BLOCKING WebRequests back to back,
|
|
//--- each up to ALTFETCH_TIMEOUT_MS. MetaTrader's ~4,500 ms teardown budget is measured from the
|
|
//--- stop REQUEST and OnDeinit cannot begin until this returns, so a stop landing inside that chain
|
|
//--- spends the entire cleanup window on downloads for a program that is being unloaded - and the
|
|
//--- chart keeps its panel, status label and arrows because the purge never runs. An in-flight
|
|
//--- request cannot be cancelled; refusing to START another one is the whole of the remedy.
|
|
if(IsStopped())
|
|
return false;
|
|
string host = HostOf(url);
|
|
int bi = BlockedIndex(host);
|
|
if(bi >= 0 && TimeCurrent() < m_blockedUntil[bi])
|
|
return false;
|
|
char data[], result[];
|
|
string rh;
|
|
ResetLastError();
|
|
//--- identify ourselves: some API gateways reject requests with an empty User-Agent
|
|
//--- at the transport level (candidate cause of the EIA 1003 on first live fetch)
|
|
int status = WebRequest("GET", url, "User-Agent: WarriorEA/1.0\r\n", timeoutMs, data, result, rh);
|
|
if(status == -1)
|
|
{
|
|
int err = GetLastError();
|
|
if(err == 4014)
|
|
{
|
|
//--- Backoff, not a permanent latch: the whitelist takes effect for a RUNNING EA
|
|
//--- the moment the user saves the options dialog, so retrying hourly picks the
|
|
//--- fix up without a re-attach.
|
|
bool firstTimeThisHost = (bi < 0);
|
|
if(bi < 0)
|
|
{
|
|
bi = ArraySize(m_blockedHost);
|
|
ArrayResize(m_blockedHost, bi + 1);
|
|
ArrayResize(m_blockedUntil, bi + 1);
|
|
m_blockedHost[bi] = host;
|
|
}
|
|
m_blockedUntil[bi] = TimeCurrent() + 3600;
|
|
if(firstTimeThisHost)
|
|
{
|
|
//--- ALWAYS name the offender - the whole point of this message is telling
|
|
//--- the user which entry is missing, not that "something" is
|
|
Alert("Warrior EA: web request to ", host, " is blocked (4014) - this exact ",
|
|
"address is missing from the WebRequest whitelist (or the checkbox is ",
|
|
"off). Copy-paste line and steps are in the Experts log.");
|
|
Print("AltDataFetch: ==================================================================");
|
|
Print("AltDataFetch: BLOCKED host (add this exact line as its own whitelist entry):");
|
|
Print(host);
|
|
Print("AltDataFetch: MT5 menu: Tools > Options > Expert Advisors ->");
|
|
Print("AltDataFetch: tick 'Allow WebRequest for listed URL', add the line, click OK.");
|
|
Print("AltDataFetch: Takes effect immediately - the EA retries within the hour.");
|
|
if(!m_urlAlerted)
|
|
{
|
|
m_urlAlerted = true;
|
|
Print("AltDataFetch: For reference, the FULL set this EA uses (each its own entry):");
|
|
Print("https://publicreporting.cftc.gov");
|
|
Print("https://api.stlouisfed.org");
|
|
Print("https://api.eia.gov");
|
|
Print("https://cdn.cboe.com");
|
|
Print("AltDataFetch: Until fixed it runs normally on cached alt-data files - features");
|
|
Print("AltDataFetch: stay valid but stop updating past their last downloaded date.");
|
|
}
|
|
Print("AltDataFetch: ==================================================================");
|
|
}
|
|
else
|
|
Print("AltDataFetch: still blocked (4014): " + host + " - retrying in an hour.");
|
|
}
|
|
else
|
|
PrintFormat("AltDataFetch: WebRequest error %d for %s", err, MaskUrl(url));
|
|
return false;
|
|
}
|
|
if(status != 200)
|
|
{
|
|
PrintFormat("AltDataFetch: HTTP %d for %s%s", status, MaskUrl(url),
|
|
status >= 1000 ? " (1xxx = MT5 transport-layer failure, not a server "
|
|
"response - retried on the normal hourly schedule)" : "");
|
|
return false;
|
|
}
|
|
body = CharArrayToString(result, 0, WHOLE_ARRAY, CP_UTF8);
|
|
return true;
|
|
}
|
|
|
|
//--- minimal targeted JSON scan: value of "key":"..." (or numeric) at/after position `from`.
|
|
//--- Sufficient for Socrata/FRED responses, which are flat arrays of flat objects.
|
|
string JsonField(const string &body, string key, int from, int &valueEnd)
|
|
{
|
|
valueEnd = -1;
|
|
string tag = "\"" + key + "\":";
|
|
int p = StringFind(body, tag, from);
|
|
if(p < 0)
|
|
return "";
|
|
p += StringLen(tag);
|
|
if(StringGetCharacter(body, p) == '"')
|
|
{
|
|
int q = StringFind(body, "\"", p + 1);
|
|
if(q < 0)
|
|
return "";
|
|
valueEnd = q + 1;
|
|
return StringSubstr(body, p + 1, q - p - 1);
|
|
}
|
|
int e = p;
|
|
while(e < StringLen(body))
|
|
{
|
|
ushort c = StringGetCharacter(body, e);
|
|
if((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')
|
|
e++;
|
|
else
|
|
break;
|
|
}
|
|
valueEnd = e;
|
|
return StringSubstr(body, p, e - p);
|
|
}
|
|
|
|
datetime SaturdayAfter(datetime reportDate)
|
|
{
|
|
MqlDateTime st;
|
|
TimeToStruct(reportDate, st);
|
|
int days = (6 - st.day_of_week + 7) % 7; // MQL5: 0=Sunday .. 6=Saturday
|
|
if(days == 0)
|
|
days = 7;
|
|
return reportDate + days * 86400;
|
|
}
|
|
|
|
string UrlEncodePart(string s)
|
|
{
|
|
string out = "";
|
|
for(int i = 0; i < StringLen(s); i++)
|
|
{
|
|
ushort c = StringGetCharacter(s, i);
|
|
//--- '%' MUST come first and MUST be escaped: SoQL like-predicates use it as the
|
|
//--- wildcard ('EURO FX -%'), and an unescaped one turns the rest of the query into
|
|
//--- a malformed percent-escape.
|
|
if(c == '%') out += "%25";
|
|
else if(c == ' ') out += "%20";
|
|
else if(c == '\'') out += "%27";
|
|
else if(c == '&') out += "%26";
|
|
else if(c == '>') out += "%3E";
|
|
else if(c == '=') out += "%3D";
|
|
else if(c == '+') out += "%2B";
|
|
else if(c == '#') out += "%23";
|
|
else out += ShortToString(c);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
//--- ------------------------------------------ raw cache load/save
|
|
bool LoadRaw(string name, SAltRawSeries &s, int cols)
|
|
{
|
|
s.rows = 0;
|
|
int h = FileOpen(ALTFETCH_DIR + name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
FileReadString(h); // header
|
|
int cap = 2048;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
string f[];
|
|
if(StringSplit(line, ';', f) < cols + 2)
|
|
continue;
|
|
if(s.rows >= cap)
|
|
{
|
|
cap = cap * 3 / 2;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
}
|
|
s.date[s.rows] = StringToTime(f[0]);
|
|
s.published[s.rows] = StringToTime(f[1]);
|
|
s.v1[s.rows] = StringToDouble(f[2]);
|
|
s.v2[s.rows] = cols > 1 ? StringToDouble(f[3]) : 0.0;
|
|
s.v3[s.rows] = cols > 2 ? StringToDouble(f[4]) : 0.0;
|
|
s.rows++;
|
|
}
|
|
FileClose(h);
|
|
return s.rows > 0;
|
|
}
|
|
|
|
//--- Whole-file rewrites go through a temp + FileMove swap: multiple charts share these
|
|
//--- caches (raw_VIXCLS serves every symbol; two timeframes of one symbol share all of
|
|
//--- them), and a reader hitting a truncate-then-write mid-flight parses a torn file.
|
|
//--- The temp name carries the destination name so parallel writers of DIFFERENT files
|
|
//--- never collide; parallel writers of the SAME file write identical content, and the
|
|
//--- last FileMove wins - which is fine, they agree.
|
|
bool SaveRaw(string name, SAltRawSeries &s, int cols, string header)
|
|
{
|
|
string tmp = ALTFETCH_DIR + name + ".tmp";
|
|
int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d)", tmp, GetLastError());
|
|
return false;
|
|
}
|
|
FileWriteString(h, header + "\n");
|
|
for(int i = 0; i < s.rows; i++)
|
|
{
|
|
string line = TimeToString(s.date[i], TIME_DATE) + ";" + TimeToString(s.published[i], TIME_DATE) + ";"
|
|
+ DoubleToString(s.v1[i], 6);
|
|
if(cols > 1)
|
|
line += ";" + DoubleToString(s.v2[i], 6);
|
|
if(cols > 2)
|
|
line += ";" + DoubleToString(s.v3[i], 6);
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
FileClose(h);
|
|
if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + name, FILE_COMMON | FILE_REWRITE))
|
|
{
|
|
PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - cache not updated this pass.",
|
|
tmp, name, GetLastError());
|
|
FileDelete(tmp, FILE_COMMON);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
//--- ------------------------------------------------- FRED source
|
|
//--- appends observations after the cached tail; full history when the cache is empty
|
|
//--- vmin/vmax: per-series plausibility band. A value outside it is a data ERROR
|
|
//--- (transport corruption, format drift, StringToDouble on garbage returning 0.0),
|
|
//--- and one absurd value poisons every change/percentile feature computed across it -
|
|
//--- the BatchNorm NaN-latch incident came from exactly one huge-but-finite input.
|
|
//--- Rejected rows are counted and reported, never silently dropped.
|
|
bool UpdateFred(string seriesId, SAltRawSeries &s, int throttleSlot,
|
|
int staleDays = 4, double vmin = -1e18, double vmax = 1e18)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- daily series: current until ~2 business days behind (weekends/holidays).
|
|
//--- monthly series pass staleDays ~32 so a 30-day-old latest print does not
|
|
//--- trigger a pointless fetch attempt every hour for weeks.
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < staleDays * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string key = FredKey();
|
|
if(key == "")
|
|
return false;
|
|
string url = "https://api.stlouisfed.org/fred/series/observations?series_id=" + seriesId +
|
|
"&api_key=" + key + "&file_type=json&limit=100000";
|
|
//--- ALWAYS bound the start. An unbounded backfill of a century-old series downloads
|
|
//--- data the 2010 feature grid can never use, and the pre-1970 tail is actively
|
|
//--- hostile: those dates are outside datetime range and 1913-era CPI levels sat
|
|
//--- below the plausibility band, producing 157 scary-but-meaningless "REJECTED"
|
|
//--- lines on the first live fetch. 2005 leaves 5y of lookback margin ahead of the
|
|
//--- grid for the longest window (3y COT percentile analog).
|
|
string startDate = (s.rows > 0)
|
|
? TimeToString(s.date[s.rows - 1] + 86400, TIME_DATE)
|
|
: "2005.01.01";
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; FRED wants ISO dashes
|
|
url += "&observation_start=" + startDate;
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0, rejected = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = JsonField(body, "date", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2;
|
|
string v = JsonField(body, "value", e1, e2);
|
|
pos = (e2 > e1 ? e2 : e1);
|
|
if(v == "" || v == ".")
|
|
continue;
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || (s.rows > 0 && dt <= s.date[s.rows - 1]))
|
|
continue;
|
|
double val = StringToDouble(v);
|
|
if(val < vmin || val > vmax || !MathIsValidNumber(val))
|
|
{
|
|
rejected++;
|
|
continue;
|
|
}
|
|
int n = s.rows;
|
|
ArrayResize(s.date, n + 1);
|
|
ArrayResize(s.published, n + 1);
|
|
ArrayResize(s.v1, n + 1);
|
|
ArrayResize(s.v2, n + 1);
|
|
ArrayResize(s.v3, n + 1);
|
|
s.date[n] = dt;
|
|
s.published[n] = dt + 86400; // unrevised daily: knowable next day
|
|
s.v1[n] = val;
|
|
s.rows = n + 1;
|
|
added++;
|
|
}
|
|
if(rejected > 0)
|
|
PrintFormat("AltDataFetch: %s REJECTED %d value(s) outside the plausibility band "
|
|
"[%g, %g] - transport corruption or format drift; investigate if repeated.",
|
|
seriesId, rejected, vmin, vmax);
|
|
if(added > 0)
|
|
{
|
|
SaveRaw("raw_" + seriesId + ".csv", s, 1, "date;published;value");
|
|
PrintFormat("AltDataFetch: %s +%d rows (through %s)", seriesId, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: %s returned a response with ZERO usable observations and "
|
|
"the cache is empty - series id wrong or API format changed. Features "
|
|
"depending on it will stay empty (panel 0-fills).", seriesId);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- -------------------------------------------------- COT source
|
|
bool UpdateCot(string cacheName, string cotWhere, SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly: a new Tuesday report exists once the last one is >7d old; allow 3 extra
|
|
//--- days before nagging the API (Friday release + weekend)
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string where = cotWhere;
|
|
if(s.rows > 0)
|
|
{
|
|
string lastDate = TimeToString(s.date[s.rows - 1], TIME_DATE);
|
|
StringReplace(lastDate, ".", "-"); // ISO for the SoQL timestamp comparison
|
|
where += " AND report_date_as_yyyy_mm_dd > '" + lastDate + "'";
|
|
}
|
|
string url = "https://publicreporting.cftc.gov/resource/gpe5-46if.json?%24select=" +
|
|
"report_date_as_yyyy_mm_dd,open_interest_all,lev_money_positions_long,lev_money_positions_short" +
|
|
"&%24where=" + UrlEncodePart(where) +
|
|
"&%24order=report_date_as_yyyy_mm_dd&%24limit=50000";
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = JsonField(body, "report_date_as_yyyy_mm_dd", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2, e3, e4;
|
|
string oi = JsonField(body, "open_interest_all", e1, e2);
|
|
string ll = JsonField(body, "lev_money_positions_long", e2, e3);
|
|
string ls = JsonField(body, "lev_money_positions_short", e3, e4);
|
|
pos = (e4 > e1 ? e4 : e1);
|
|
datetime dt = StringToTime(StringSubstr(d, 0, 10));
|
|
if(dt <= 0 || oi == "" || ll == "" || ls == "")
|
|
continue;
|
|
if(s.rows > 0 && dt == s.date[s.rows - 1])
|
|
{
|
|
//--- same report date from a second contract-name variant: keep the larger-OI row
|
|
if(StringToDouble(oi) > s.v1[s.rows - 1])
|
|
{
|
|
s.v1[s.rows - 1] = StringToDouble(oi);
|
|
s.v2[s.rows - 1] = StringToDouble(ll);
|
|
s.v3[s.rows - 1] = StringToDouble(ls);
|
|
}
|
|
continue;
|
|
}
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
int n = s.rows;
|
|
ArrayResize(s.date, n + 1);
|
|
ArrayResize(s.published, n + 1);
|
|
ArrayResize(s.v1, n + 1);
|
|
ArrayResize(s.v2, n + 1);
|
|
ArrayResize(s.v3, n + 1);
|
|
s.date[n] = dt;
|
|
s.published[n] = SaturdayAfter(dt);
|
|
s.v1[n] = StringToDouble(oi);
|
|
s.v2[n] = StringToDouble(ll);
|
|
s.v3[n] = StringToDouble(ls);
|
|
s.rows = n + 1;
|
|
added++;
|
|
}
|
|
if(added > 0)
|
|
{
|
|
SaveRaw(cacheName, s, 3, "date;published;oi;lev_long;lev_short");
|
|
PrintFormat("AltDataFetch: %s +%d weekly reports (through %s)", cacheName, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: COT query for %s returned ZERO reports on an empty cache - "
|
|
"the contract-name predicate probably matches nothing at the endpoint "
|
|
"(the like-predicates for non-SP500/JPY contracts are unverified; see the "
|
|
"catalog). COT features stay empty (panel 0-fills) until it is corrected.",
|
|
cacheName);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ---------------------------------------------------- EIA source
|
|
//--- Weekly Petroleum Status Report via the EIA v2 API, three series in one call
|
|
//--- (2010->now x 3 ~ 2,600 rows, under the API's 5000-row page - revisit if series
|
|
//--- are added). Rows arrive sorted by period; same-period rows fill their column by
|
|
//--- series id. Publication stamp: observed + 6 days = Thursday 00:00 UTC after the
|
|
//--- Wednesday release (holiday-safe), identical to research/altdata/eia.py.
|
|
bool UpdateEia(SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly cadence, same nag guard as COT
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string key = EiaKey();
|
|
if(key == "")
|
|
return false;
|
|
string startDate = TimeToString(s.rows > 0 ? s.date[s.rows - 1] + 86400 : ALTFETCH_GRID_START, TIME_DATE);
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; the API wants ISO dashes
|
|
string url = "https://api.eia.gov/v2/petroleum/sum/sndw/data/?api_key=" + key +
|
|
"&frequency=weekly&data%5B0%5D=value" +
|
|
"&facets%5Bseries%5D%5B0%5D=WCESTUS1" + // crude stocks ex-SPR
|
|
"&facets%5Bseries%5D%5B1%5D=WCRFPUS2" + // field production
|
|
"&facets%5Bseries%5D%5B2%5D=WPULEUS3" + // refinery utilization %
|
|
"&sort%5B0%5D%5Bcolumn%5D=period&sort%5B0%5D%5Bdirection%5D=asc" +
|
|
"&length=5000&start=" + startDate;
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1, e2, e3;
|
|
string d = JsonField(body, "period", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
string sid = JsonField(body, "series", e1, e2);
|
|
string v = JsonField(body, "value", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || sid == "" || v == "")
|
|
continue;
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
if(s.rows == 0 || dt > s.date[s.rows - 1])
|
|
{
|
|
int n = s.rows;
|
|
ArrayResize(s.date, n + 1);
|
|
ArrayResize(s.published, n + 1);
|
|
ArrayResize(s.v1, n + 1);
|
|
ArrayResize(s.v2, n + 1);
|
|
ArrayResize(s.v3, n + 1);
|
|
s.date[n] = dt;
|
|
s.published[n] = dt + 6 * 86400;
|
|
s.v1[n] = 0.0;
|
|
s.v2[n] = 0.0;
|
|
s.v3[n] = 0.0;
|
|
s.rows = n + 1;
|
|
added++;
|
|
}
|
|
int r = s.rows - 1;
|
|
if(sid == "WCESTUS1") s.v1[r] = StringToDouble(v);
|
|
else if(sid == "WCRFPUS2") s.v2[r] = StringToDouble(v);
|
|
else if(sid == "WPULEUS3") s.v3[r] = StringToDouble(v);
|
|
}
|
|
if(added > 0)
|
|
{
|
|
SaveRaw("raw_EIA_WPSR.csv", s, 3, "date;published;stocks;production;utilization");
|
|
PrintFormat("AltDataFetch: raw_EIA_WPSR.csv +%d weekly reports (through %s)", added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ------------------------------------ GEX forward recorder
|
|
//--- WHY THIS IS A RECORDER AND NOT A FEATURE: option open interest is a SNAPSHOT
|
|
//--- source. Nobody publishes free history, so unlike COT/FRED/EIA there is nothing to
|
|
//--- backfill - the series only accrues from the day recording starts, which is exactly
|
|
//--- why this ships before the redeploy rather than after. It writes one row per trading
|
|
//--- day and feeds NOTHING: wiring a feature whose value is missing across ~100% of the
|
|
//--- training sample would waste input width and hand batch-norm a constant. Revisit
|
|
//--- when the file holds a year or so, then screen it like every other candidate.
|
|
//---
|
|
//--- THE THESIS (why GEX is worth a year of waiting): dealer gamma is a RANGE mechanism,
|
|
//--- and range is this project's proven channel. Net long dealer gamma means hedging
|
|
//--- sells rallies and buys dips - realised range compresses; net short gamma amplifies
|
|
//--- both ways - range expands. That is a directional-agnostic volatility statement,
|
|
//--- which is the shape of thing the excursion head can actually use.
|
|
//---
|
|
//--- Source: CBOE's free delayed-quotes CDN, no key. It publishes per-contract gamma
|
|
//--- and open interest directly, so no option pricing model is needed here (and no
|
|
//--- model risk enters the recorded data). Validated 2026-08-16 against the live SPX
|
|
//--- chain: 29,362 rows, 20,993 with nonzero gamma, 54 expiries, total +90.7 Bn/1%.
|
|
string CboeTicker(string canonical)
|
|
{
|
|
if(canonical == "SP500") return "_SPX"; // index options - where dealer positioning concentrates
|
|
if(canonical == "NAS100") return "_NDX";
|
|
if(canonical == "US30") return "_DJX";
|
|
if(canonical == "US2000") return "IWM";
|
|
if(canonical == "XAUUSD") return "GLD";
|
|
if(canonical == "XAGUSD") return "SLV";
|
|
if(canonical == "XTIUSD") return "USO";
|
|
if(canonical == "BTCUSD") return "IBIT";
|
|
return ""; // no listed-options proxy worth recording
|
|
}
|
|
|
|
//--- last date already in the file, or 0
|
|
datetime LastGexDate(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 1)
|
|
return 0;
|
|
return StringToTime(f[0]);
|
|
}
|
|
|
|
bool UpdateGex(string canonical)
|
|
{
|
|
string ticker = CboeTicker(canonical);
|
|
if(ticker == "")
|
|
return false;
|
|
//--- Once per UTC day, after the US cash close (21:00 UTC covers both DST regimes),
|
|
//--- and never on a weekend - the chain would just repeat Friday's settled numbers.
|
|
MqlDateTime g;
|
|
TimeToStruct(TimeGMT(), g);
|
|
if(g.day_of_week == 0 || g.day_of_week == 6)
|
|
return false;
|
|
if(g.hour < 21)
|
|
return false;
|
|
string file = ALTFETCH_DIR + "gex_" + canonical + ".csv";
|
|
datetime today = StringToTime(StringFormat("%04d.%02d.%02d", g.year, g.mon, g.day));
|
|
if(!m_gexDateLoaded)
|
|
{
|
|
m_lastGexDate = LastGexDate(file);
|
|
m_gexDateLoaded = true;
|
|
}
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- Re-read from DISK before spending the download: two charts of the same symbol
|
|
//--- (e.g. D1 + H4) each run their own fetcher, and a per-instance cached date would
|
|
//--- let the second chart append a duplicate row for the day the first just recorded.
|
|
m_lastGexDate = LastGexDate(file);
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- the SPX chain is ~13 MB: this needs a far longer timeout than the small feeds
|
|
string body;
|
|
if(!HttpGet("https://cdn.cboe.com/api/global/delayed_quotes/options/" + ticker + ".json",
|
|
body, ALTFETCH_GEX_TIMEOUT_MS))
|
|
return false;
|
|
if(StringLen(body) < 1000)
|
|
{
|
|
Print("AltDataFetch: GEX response for " + ticker + " was too short to be a chain - skipping.");
|
|
return false;
|
|
}
|
|
int e0;
|
|
string spotS = JsonField(body, "current_price", 0, e0); // absent from option rows, so
|
|
if(spotS == "") // a plain scan finds the header one
|
|
spotS = JsonField(body, "close", 0, e0);
|
|
double spot = StringToDouble(spotS);
|
|
if(spot <= 0.0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - no usable spot price in the response, skipping.");
|
|
return false;
|
|
}
|
|
//--- per-expiry accumulation (54 expiries on SPX; the cap is generous)
|
|
int expCode[128];
|
|
double expGex[128];
|
|
int nExp = 0;
|
|
double gexCall = 0.0, gexPut = 0.0;
|
|
double oiCall = 0.0, oiPut = 0.0;
|
|
int rows = 0;
|
|
int pos = 0;
|
|
//--- Row layout is option, ..., open_interest, volume, delta, gamma - so one forward
|
|
//--- sweep reads each row's three fields in the order they appear.
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string sym = JsonField(body, "option", pos, e1);
|
|
if(sym == "" || e1 < 0)
|
|
break;
|
|
int e2, e3;
|
|
string oiS = JsonField(body, "open_interest", e1, e2);
|
|
string gS = JsonField(body, "gamma", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
double oi = StringToDouble(oiS);
|
|
double gam = StringToDouble(gS);
|
|
if(oi <= 0.0 || gam == 0.0)
|
|
continue; // no exposure - skip the string work entirely
|
|
//--- OCC symbol: <root><YYMMDD><C|P><strike*1000, 8 digits>, so the tail is fixed
|
|
//--- width regardless of how long the root is (SPX, SPXW, IWM, ...).
|
|
int len = StringLen(sym);
|
|
if(len < 15)
|
|
continue;
|
|
string kind = StringSubstr(sym, len - 9, 1);
|
|
int expiry = (int)StringToInteger(StringSubstr(sym, len - 15, 6));
|
|
//--- dollar gamma per 1% move: gamma x OI x 100 (contract multiplier) x S^2 x 0.01
|
|
double dollar = gam * oi * 100.0 * spot * spot * 0.01;
|
|
if(kind == "C")
|
|
{
|
|
gexCall += dollar;
|
|
oiCall += oi;
|
|
}
|
|
else
|
|
{
|
|
gexPut -= dollar; // dealers short puts: opposite sign convention
|
|
oiPut += oi;
|
|
dollar = -dollar;
|
|
}
|
|
int slot = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
if(expCode[i] == expiry)
|
|
{
|
|
slot = i;
|
|
break;
|
|
}
|
|
if(slot < 0 && nExp < 128)
|
|
{
|
|
slot = nExp++;
|
|
expCode[slot] = expiry;
|
|
expGex[slot] = 0.0;
|
|
}
|
|
if(slot >= 0)
|
|
expGex[slot] += dollar;
|
|
rows++;
|
|
}
|
|
if(rows == 0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - parsed 0 contracts with exposure, skipping "
|
|
"(the CDN response format may have changed).");
|
|
return false;
|
|
}
|
|
//--- Partial-chain guard: a truncated 13 MB download parses cleanly and yields a
|
|
//--- plausible-but-wrong GEX. A day-over-day contract-count collapse (>50%) is the
|
|
//--- fingerprint of truncation, not of markets - OI does not halve overnight.
|
|
int prevRows = LastGexRowCount(file);
|
|
if(prevRows > 0 && rows < prevRows / 2)
|
|
{
|
|
PrintFormat("AltDataFetch: GEX %s - only %d contracts vs %d yesterday (<50%%): response "
|
|
"looks TRUNCATED, not recording. Will retry on the next timer pass.",
|
|
ticker, rows, prevRows);
|
|
return false;
|
|
}
|
|
//--- three nearest expiries, by selection (nExp is tiny)
|
|
double front[3] = {0.0, 0.0, 0.0};
|
|
int frontCode = 0, prevCode = 0;
|
|
for(int k = 0; k < 3; k++)
|
|
{
|
|
int best = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
{
|
|
if(expCode[i] <= prevCode)
|
|
continue;
|
|
if(best < 0 || expCode[i] < expCode[best])
|
|
best = i;
|
|
}
|
|
if(best < 0)
|
|
break;
|
|
front[k] = expGex[best] / 1e9;
|
|
prevCode = expCode[best];
|
|
if(k == 0)
|
|
frontCode = prevCode;
|
|
}
|
|
bool isNew = !FileIsExist(file, FILE_COMMON);
|
|
int h = FileOpen(file, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d) - GEX not recorded today.", file, GetLastError());
|
|
return false;
|
|
}
|
|
if(isNew)
|
|
FileWriteString(h, "date;recorded_utc;spot;gex_total;gex_call;gex_put;oi_call;oi_put;"
|
|
"gex_exp1;gex_exp2;gex_exp3;expiry1;n_expiry;n_rows\n");
|
|
FileSeek(h, 0, SEEK_END);
|
|
FileWriteString(h, StringFormat("%s;%s;%.4f;%.6f;%.6f;%.6f;%.0f;%.0f;%.6f;%.6f;%.6f;%d;%d;%d\n",
|
|
TimeToString(today, TIME_DATE),
|
|
TimeToString(TimeGMT(), TIME_DATE | TIME_MINUTES),
|
|
spot, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9,
|
|
oiCall, oiPut, front[0], front[1], front[2],
|
|
frontCode, nExp, rows));
|
|
FileClose(h);
|
|
m_lastGexDate = today;
|
|
//--- Progress matters here: the file is useless until it is long enough to screen, and
|
|
//--- the only way to know how far along it is, is to say so.
|
|
int have = 1;
|
|
datetime probe = LastGexDate(file);
|
|
if(probe > 0)
|
|
have = CountGexRows(file);
|
|
PrintFormat("AltDataFetch: GEX %s recorded - total %+.2f Bn/1%% (calls %+.2f, puts %+.2f), "
|
|
"%d contracts, %d expiries, spot %.2f. %d day(s) banked; this is a FORWARD "
|
|
"RECORDER - no free history exists, so it feeds no model until there is enough "
|
|
"to screen (~250 trading days).",
|
|
ticker, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9, rows, nExp, spot, have);
|
|
return true;
|
|
}
|
|
|
|
//--- n_rows (field 14) of the last data line - the previous session's contract count
|
|
int LastGexRowCount(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 14)
|
|
return 0;
|
|
return (int)StringToInteger(f[13]);
|
|
}
|
|
|
|
int CountGexRows(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
int n = 0;
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
n++;
|
|
}
|
|
FileClose(h);
|
|
return n;
|
|
}
|
|
|
|
//--- -------------------------------- feature math (mirrors export.py)
|
|
//--- pandas rolling(W, min_periods=M).rank(pct=True) on the LAST element: rank of the current
|
|
//--- value among the window's values (average rank for ties) divided by the window count.
|
|
double RollingPctRank(const double &v[], int i, int window, int minPeriods)
|
|
{
|
|
int lo = MathMax(0, i - window + 1);
|
|
int n = i - lo + 1;
|
|
if(n < minPeriods)
|
|
return EMPTY_VALUE;
|
|
double cur = v[i];
|
|
int below = 0, ties = 0;
|
|
for(int k = lo; k <= i; k++)
|
|
{
|
|
if(v[k] < cur)
|
|
below++;
|
|
else if(v[k] == cur)
|
|
ties++;
|
|
}
|
|
double rank = below + (1.0 + ties) / 2.0;
|
|
return rank / n;
|
|
}
|
|
|
|
//--- as-of index: largest row with published <= day, else -1
|
|
int AsOf(const SAltRawSeries &s, datetime day)
|
|
{
|
|
int lo = 0, hi = s.rows - 1, ans = -1;
|
|
while(lo <= hi)
|
|
{
|
|
int mid = (lo + hi) / 2;
|
|
if(s.published[mid] <= day)
|
|
{
|
|
ans = mid;
|
|
lo = mid + 1;
|
|
}
|
|
else
|
|
hi = mid - 1;
|
|
}
|
|
return ans;
|
|
}
|
|
|
|
//--- raw series kept resident between timer ticks (one chart = one symbol, so only the
|
|
//--- sources the chart's catalog row names are ever populated)
|
|
//--- m_ivol holds whichever instrument-specific implied-vol index THIS chart's catalog
|
|
//--- row names (GVZ for gold, OVX for oil, VXN/VXD/RVX for the index symbols) - one
|
|
//--- chart is one symbol, so a single slot serves regardless of which series it is.
|
|
SAltRawSeries m_vix, m_usd, m_cot, m_eia, m_ivol;
|
|
//--- US macro block. VINTAGE CAVEAT, deliberate and documented: DGS10/T10Y2Y/T5YIE/DFF/
|
|
//--- ECBDFR are unrevised market/policy series (current value IS the first print), and
|
|
//--- CPI is fetched as CPIAUCNS - the NSA index, which is essentially never revised -
|
|
//--- precisely so the plain-FRED backfill stays vintage-clean. UNRATE is the exception:
|
|
//--- it IS revised (seasonal refits, ~0.1-0.2pp), there is no unrevised variant, and the
|
|
//--- EA cannot run the ALFRED first-print protocol - so a fresh backfill carries small
|
|
//--- revisions into history. Accepted as an exploratory-feature caveat; the research
|
|
//--- screen used true first prints and measured null either way.
|
|
SAltRawSeries m_dgs10, m_curve, m_bei, m_dff, m_ecb, m_cpi, m_unrate;
|
|
|
|
//--- ------------------------------- symbol catalog (data, not code)
|
|
//--- An MT5 symbol cannot be mapped to a CFTC contract by rule - the naming universes are
|
|
//--- unrelated and every broker invents its own index tickers - so the catalog IS the
|
|
//--- mechanism: the chart symbol selects a row at runtime and ONLY that row's sources are
|
|
//--- fetched. Alias lists cover the naming used by the brokers this EA targets (The5ers,
|
|
//--- FTMO, AvaTrade) plus the other common conventions (Dukascopy USA500IDX, OANDA
|
|
//--- SPX500, IC Markets/Pepperstone US500, ...); prefix matching absorbs the per-broker
|
|
//--- suffix zoo (.cash, .r, .pro, m, +, _SB). Anything unrecognised raises the mapping
|
|
//--- dialog (Panel\AltDataMapDialog.mqh) and the answer persists in symbol_map.cfg.
|
|
//---
|
|
//--- FEATURE SETS - what is certified vs what is exploratory:
|
|
//--- SP500 / USDJPY / XAUUSD carry the SCREENED survivors (family-wise + incremental
|
|
//--- gates, research/altdata/DESIGN.md). Everything else is EXPLORATORY: the same
|
|
//--- feature families extended by analogy to instruments that were never screened,
|
|
//--- plus the EIA petroleum block on every row by user directive 2026-08-16 ("the NN
|
|
//--- might find patterns in it for both oil and regular symbols") - EIA screened NULL
|
|
//--- on WTI's short 7y sample. Exploratory inputs are not an edge claim: the deploy
|
|
//--- gate decides whether a model trained on them ever trades.
|
|
SAltSymbolSpec m_specs[];
|
|
string m_userFrom[]; // broker symbol (from symbol_map.cfg / the dialog)
|
|
string m_userTo[]; // canonical, or "NONE" = user declined alt data
|
|
bool m_userMapLoaded;
|
|
|
|
void AddSpec(string canonical, string aliases, string cotWhere,
|
|
double cotSign, string feats, string ivolSeries = "")
|
|
{
|
|
int n = ArraySize(m_specs);
|
|
ArrayResize(m_specs, n + 1);
|
|
m_specs[n].canonical = canonical;
|
|
m_specs[n].aliases = aliases;
|
|
m_specs[n].cotWhere = cotWhere;
|
|
m_specs[n].cotSign = cotSign;
|
|
m_specs[n].features = feats;
|
|
m_specs[n].ivolSeries = ivolSeries;
|
|
}
|
|
|
|
void BuildCatalog(void)
|
|
{
|
|
//--- feature bundles (locals, deliberately prefixed so no macro elsewhere can collide)
|
|
//--- POLICY (user directive 2026-08-16, twice): every feature the fetched sources can
|
|
//--- serve is wired on every symbol - the research screens inform priors but do NOT
|
|
//--- gate the input list ("we will leave the NN to do its job"). The deploy gate
|
|
//--- remains the arbiter of whether anything trained on these ever trades.
|
|
string f_eia = "eia_stk_idx1y;eia_stk_chg4;eia_util";
|
|
string f_risk = "vix_chg5;vix;usd_chg5"; // equity-risk complex
|
|
string f_cot = "cot_idx_1y;cot_idx_3y;cot_chg_4w"; // COT positioning family
|
|
//--- US macro block: 10y yield 20d change, curve slope, 5y breakeven 20d change,
|
|
//--- Fed-ECB policy gap, CPI yoy, unemployment 12m change. Screened NULL vs forward
|
|
//--- 5-day range on all four research symbols (screen_macro 2026-08-16) - wired
|
|
//--- anyway per the policy above; the screen result stands as the honest prior.
|
|
string f_mac = "mac_y10;mac_curve;mac_bei;mac_gap;mac_cpi;mac_unemp";
|
|
//--- CFTC predicates. The two verified in(...) clauses are the ones already proven
|
|
//--- against the endpoint; the rest use like 'PREFIX%' mirroring the research-side
|
|
//--- regexes in research/altdata/cot.py, which is safer than guessing an exact
|
|
//--- exchange suffix. A predicate that matches nothing costs only an empty column.
|
|
string w_es = "market_and_exchange_names in('E-MINI S&P 500 - CHICAGO MERCANTILE EXCHANGE',"
|
|
"'E-MINI S&P 500 STOCK INDEX - CHICAGO MERCANTILE EXCHANGE')";
|
|
string w_jpy = "market_and_exchange_names in('JAPANESE YEN - CHICAGO MERCANTILE EXCHANGE')";
|
|
string w_eur = "market_and_exchange_names like 'EURO FX -%'";
|
|
string w_gbp = "market_and_exchange_names like 'BRITISH POUND%'";
|
|
string w_cad = "market_and_exchange_names like 'CANADIAN DOLLAR%'";
|
|
string w_aud = "market_and_exchange_names like 'AUSTRALIAN DOLLAR%'";
|
|
string w_nzd = "market_and_exchange_names like 'NZ DOLLAR%'";
|
|
string w_chf = "market_and_exchange_names like 'SWISS FRANC%'";
|
|
string w_btc = "market_and_exchange_names like 'BITCOIN%'";
|
|
|
|
//--- ---------------- equity indices (VIX/USD complex; only SP500 is screened)
|
|
//--- Screened findings preserved as priors, not gates: SP500 vix_chg5/vix/usd_chg5/
|
|
//--- cot_spec_net and USDJPY COT family cleared family-wise + incremental; XAUUSD
|
|
//--- gvz_chg5 (GVZ = gold's own IV) is the campaign's 2nd-strongest incremental
|
|
//--- feature (MI|vol 0.0197 p=0.002, 4.6x vix_chg5 on gold); EURUSD/USDJPY vix_chg5
|
|
//--- screened clean; the macro block screened null everywhere; EIA screened null on
|
|
//--- WTI. All ship regardless - see the policy note above.
|
|
AddSpec("SP500", "SP500;SPX500;US500;USA500;USA500IDX;US_500;SPXUSD;S&P500;SP_500;"
|
|
"SPX;USA500IDXUSD;US500Cash", w_es, 1.0,
|
|
f_risk + ";cot_spec_net;" + f_eia + ";" + f_mac);
|
|
AddSpec("NAS100", "NAS100;NASDAQ;US100;USA100;USATEC;USTEC;NDX100;NQ100;TECH100;"
|
|
"USA100IDX;US_100;NDX;USATECHIDX", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXNCLS");
|
|
AddSpec("US30", "US30;DJ30;DOW30;USA30;WALLSTREET;WS30;DJIUSD;USA30IDX;"
|
|
"US_30;DOWJONES;DJI", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXDCLS");
|
|
AddSpec("US2000", "US2000;RUSSELL2000;RUT;USA2000;RTY;US_2000", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "RVXCLS");
|
|
AddSpec("DE40", "DE40;GER40;DAX40;DAX;GER30;DE30;GERMANY40;DEU40;GRXEUR;"
|
|
"DE_40;GER_40", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("UK100", "UK100;FTSE100;FTSE;GBR100;UKX;GB100;UK_100;BRXGBP", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("JP225", "JP225;NIKKEI;NI225;JPN225;J225;JPXJPY;JP_225", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("EU50", "EU50;STOXX50;ESTX50;EUSTX50;SX5E;EUR50;E50EUR", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("FR40", "FR40;CAC40;FRA40;CAC;FRXEUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("AU200", "AU200;ASX200;AUS200;SPI200;AU_200;AUXAUD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("HK50", "HK50;HSI;HKG33;HK33;HANGSENG;HKXHKD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- FX majors (full complement: COT + risk complex + EIA + macro;
|
|
//--- no per-instrument IV - EVZ was discontinued 2025-03 and must never be wired)
|
|
AddSpec("EURUSD", "EURUSD;EUR_USD", w_eur, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDJPY", "USDJPY;USD_JPY", w_jpy, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("GBPUSD", "GBPUSD;GBP_USD", w_gbp, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDCAD", "USDCAD;USD_CAD", w_cad, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("AUDUSD", "AUDUSD;AUD_USD", w_aud, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("NZDUSD", "NZDUSD;NZD_USD", w_nzd, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDCHF", "USDCHF;USD_CHF", w_chf, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- metals (gold/silver COT is in the DISAGGREGATED CFTC dataset -
|
|
//--- different id AND column names - still not wired; silver has no free IV index)
|
|
AddSpec("XAUUSD", "XAUUSD;GOLD;XAU_USD;GOLDUSD", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "GVZCLS");
|
|
AddSpec("XAGUSD", "XAGUSD;SILVER;XAG_USD;SILVERUSD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- energy
|
|
AddSpec("XTIUSD", "XTIUSD;USOIL;WTI;CRUDE;USCRUDE;OIL;WTICOUSD;"
|
|
"OILUSD;LIGHTCMDUSD;CRUDEOIL", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
|
|
AddSpec("XBRUSD", "XBRUSD;UKOIL;BRENT;UKOUSD;BCOUSD;BRENTCMDUSD", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
|
|
AddSpec("NATGAS", "NATGAS;NGAS;XNGUSD;NATURALGAS", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- crypto
|
|
AddSpec("BTCUSD", "BTCUSD;BITCOIN;XBTUSD;BTC_USD", w_btc, 1.0,
|
|
f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
}
|
|
|
|
//--- ------------------------------------- user mapping persistence
|
|
//--- symbol_map.cfg: one "BROKERSYMBOL=CANONICAL" per line, written by the mapping
|
|
//--- dialog. "NONE" records a deliberate decline so the dialog never nags again.
|
|
void LoadUserMap(void)
|
|
{
|
|
if(m_userMapLoaded)
|
|
return;
|
|
m_userMapLoaded = true;
|
|
int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return;
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
StringTrimLeft(line);
|
|
StringTrimRight(line);
|
|
if(line == "" || StringGetCharacter(line, 0) == '#')
|
|
continue;
|
|
string kv[];
|
|
if(StringSplit(line, '=', kv) != 2)
|
|
continue;
|
|
int n = ArraySize(m_userFrom);
|
|
ArrayResize(m_userFrom, n + 1);
|
|
ArrayResize(m_userTo, n + 1);
|
|
m_userFrom[n] = kv[0];
|
|
m_userTo[n] = kv[1];
|
|
}
|
|
FileClose(h);
|
|
if(ArraySize(m_userFrom) > 0)
|
|
PrintFormat("AltDataFetch: %d user symbol mapping(s) loaded from symbol_map.cfg.",
|
|
ArraySize(m_userFrom));
|
|
}
|
|
|
|
//--- exact, then prefix, over one row's alias list
|
|
bool AliasMatches(int spec, string symbol, bool prefixPass)
|
|
{
|
|
string a[];
|
|
int n = StringSplit(m_specs[spec].aliases, ';', a);
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
if(a[i] == "")
|
|
continue;
|
|
if(!prefixPass && symbol == a[i])
|
|
return true;
|
|
//--- require the alias to be a strict prefix AND at least 3 chars, so a short
|
|
//--- ticker can never swallow an unrelated symbol
|
|
if(prefixPass && StringLen(a[i]) >= 3 && StringFind(symbol, a[i]) == 0)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
int FindSpec(string symbol)
|
|
{
|
|
LoadUserMap();
|
|
//--- a user mapping wins over everything, including a catalog row for the same name
|
|
for(int u = 0; u < ArraySize(m_userFrom); u++)
|
|
{
|
|
if(m_userFrom[u] != symbol)
|
|
continue;
|
|
if(m_userTo[u] == "NONE")
|
|
return -2; // deliberately declined - do not ask again
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(m_specs[i].canonical == m_userTo[u])
|
|
return i;
|
|
PrintFormat("AltDataFetch: symbol_map.cfg maps %s to unknown catalog entry '%s' - ignoring.",
|
|
symbol, m_userTo[u]);
|
|
}
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(AliasMatches(i, symbol, false))
|
|
return i;
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(AliasMatches(i, symbol, true))
|
|
return i;
|
|
return -1; // unknown - the dialog asks the user
|
|
}
|
|
|
|
public:
|
|
CAltDataFetch(void) : m_fredKey(""), m_keyLoaded(false),
|
|
m_eiaKey(""), m_eiaWarned(false),
|
|
m_lastGexDate(0), m_gexDateLoaded(false),
|
|
m_urlAlerted(false),
|
|
m_userMapLoaded(false)
|
|
{
|
|
for(int i = 0; i < ArraySize(m_lastAttempt); i++)
|
|
m_lastAttempt[i] = 0;
|
|
BuildCatalog();
|
|
}
|
|
|
|
//--- ------------------------------------- symbol mapping interface
|
|
//--- -1 = unknown symbol (raise the dialog), -2 = user declined, >=0 = mapped
|
|
int SpecIndex(string symbol) { return FindSpec(symbol); }
|
|
bool NeedsMapping(string symbol) { return FindSpec(symbol) == -1; }
|
|
|
|
int CatalogCount(void) { return ArraySize(m_specs); }
|
|
string CatalogName(int i)
|
|
{
|
|
return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i].canonical : "";
|
|
}
|
|
//--- one-line description for the dropdown: canonical + what it would contribute
|
|
string CatalogLabel(int i)
|
|
{
|
|
if(i < 0 || i >= ArraySize(m_specs))
|
|
return "";
|
|
string f[];
|
|
int n = StringSplit(m_specs[i].features, ';', f);
|
|
return m_specs[i].canonical + " (" + IntegerToString(n) + " features)";
|
|
}
|
|
|
|
//--- Persist the user's answer and apply it immediately. canonical == "NONE" records a
|
|
//--- deliberate decline. Appends to symbol_map.cfg, replacing any previous line for the
|
|
//--- same broker symbol, so the answer survives restarts and a wiped feature folder.
|
|
bool SaveUserMapping(string symbol, string canonical)
|
|
{
|
|
LoadUserMap();
|
|
bool replaced = false;
|
|
for(int i = 0; i < ArraySize(m_userFrom); i++)
|
|
if(m_userFrom[i] == symbol)
|
|
{
|
|
m_userTo[i] = canonical;
|
|
replaced = true;
|
|
}
|
|
if(!replaced)
|
|
{
|
|
int n = ArraySize(m_userFrom);
|
|
ArrayResize(m_userFrom, n + 1);
|
|
ArrayResize(m_userTo, n + 1);
|
|
m_userFrom[n] = symbol;
|
|
m_userTo[n] = canonical;
|
|
}
|
|
int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write symbol_map.cfg (%d) - the mapping applies to "
|
|
"this session only and will be asked again next attach.", GetLastError());
|
|
return false;
|
|
}
|
|
FileWriteString(h, "# Warrior EA alt-data symbol map: BROKERSYMBOL=CANONICAL (NONE = no alt data)\n");
|
|
for(int i = 0; i < ArraySize(m_userFrom); i++)
|
|
FileWriteString(h, m_userFrom[i] + "=" + m_userTo[i] + "\n");
|
|
FileClose(h);
|
|
PrintFormat("AltDataFetch: %s mapped to %s (saved in symbol_map.cfg).", symbol, canonical);
|
|
return true;
|
|
}
|
|
|
|
//--- Maintain the raw caches and the feature CSV for `symbol`. Returns true when the
|
|
//--- feature file was rebuilt (caller should reload the panels). Cheap when current:
|
|
//--- staleness checks are in-memory date compares; network only when genuinely stale.
|
|
//--- The output file is named by the CHART symbol (the panel loads by _Symbol), so a
|
|
//--- broker-suffixed chart gets its own correctly-named file from its root's catalog row.
|
|
bool Update(string symbol)
|
|
{
|
|
int si = FindSpec(symbol);
|
|
if(si < 0)
|
|
return false; // unknown symbol or user declined - nothing to serve
|
|
bool needVix = StringFind(m_specs[si].features, "vix") >= 0;
|
|
bool needUsd = StringFind(m_specs[si].features, "usd_") >= 0;
|
|
bool needEia = StringFind(m_specs[si].features, "eia_") >= 0;
|
|
bool needCot = (m_specs[si].cotWhere != "");
|
|
bool needIvol = (m_specs[si].ivolSeries != "" && StringFind(m_specs[si].features, "ivol") >= 0);
|
|
bool needMac = StringFind(m_specs[si].features, "mac_") >= 0;
|
|
bool changed = false;
|
|
//--- plausibility bands: generous enough for any real print in the series' history
|
|
//--- (VIX peaked 89 in 2008; utilization-style checks live in FeatureValue), tight
|
|
//--- enough that transport garbage cannot pass
|
|
if(needVix)
|
|
{
|
|
if(m_vix.rows == 0)
|
|
LoadRaw("raw_VIXCLS.csv", m_vix, 1);
|
|
if(UpdateFred("VIXCLS", m_vix, 0, 4, 1.0, 200.0))
|
|
changed = true;
|
|
}
|
|
if(needUsd)
|
|
{
|
|
//--- staleDays 10: the H.10 dollar index posts with ~a week's lag (live fetch
|
|
//--- 2026-08-16 was current only through 08-07), so a 4-day horizon would fire a
|
|
//--- futile refetch attempt every hour in the gap between releases
|
|
if(m_usd.rows == 0)
|
|
LoadRaw("raw_DTWEXBGS.csv", m_usd, 1);
|
|
if(UpdateFred("DTWEXBGS", m_usd, 1, 10, 50.0, 250.0))
|
|
changed = true;
|
|
}
|
|
if(needCot)
|
|
{
|
|
//--- cache is named by CANONICAL, so two brokers' names for the same instrument
|
|
//--- share one download instead of fetching the same contract twice
|
|
string cache = "raw_COT_" + m_specs[si].canonical + ".csv";
|
|
if(m_cot.rows == 0)
|
|
LoadRaw(cache, m_cot, 3);
|
|
if(UpdateCot(cache, m_specs[si].cotWhere, m_cot, 2))
|
|
changed = true;
|
|
}
|
|
if(needEia)
|
|
{
|
|
if(m_eia.rows == 0)
|
|
LoadRaw("raw_EIA_WPSR.csv", m_eia, 3);
|
|
if(UpdateEia(m_eia, 3))
|
|
changed = true;
|
|
}
|
|
if(needIvol)
|
|
{
|
|
if(m_ivol.rows == 0)
|
|
LoadRaw("raw_" + m_specs[si].ivolSeries + ".csv", m_ivol, 1);
|
|
if(UpdateFred(m_specs[si].ivolSeries, m_ivol, 4, 4, 1.0, 300.0))
|
|
changed = true;
|
|
}
|
|
if(needMac)
|
|
{
|
|
if(m_dgs10.rows == 0) LoadRaw("raw_DGS10.csv", m_dgs10, 1);
|
|
if(m_curve.rows == 0) LoadRaw("raw_T10Y2Y.csv", m_curve, 1);
|
|
if(m_bei.rows == 0) LoadRaw("raw_T5YIE.csv", m_bei, 1);
|
|
if(m_dff.rows == 0) LoadRaw("raw_DFF.csv", m_dff, 1);
|
|
if(m_ecb.rows == 0) LoadRaw("raw_ECBDFR.csv", m_ecb, 1);
|
|
if(m_cpi.rows == 0) LoadRaw("raw_CPIAUCNS.csv", m_cpi, 1);
|
|
if(m_unrate.rows == 0) LoadRaw("raw_UNRATE.csv", m_unrate, 1);
|
|
if(UpdateFred("DGS10", m_dgs10, 5, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("T10Y2Y", m_curve, 6, 4, -10.0, 10.0)) changed = true;
|
|
if(UpdateFred("T5YIE", m_bei, 7, 4, -5.0, 15.0)) changed = true;
|
|
if(UpdateFred("DFF", m_dff, 8, 4, -2.0, 30.0)) changed = true;
|
|
if(UpdateFred("ECBDFR", m_ecb, 9, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("CPIAUCNS", m_cpi, 10, 32, 20.0, 1000.0)) changed = true;
|
|
if(UpdateFred("UNRATE", m_unrate, 11, 32, 0.0, 35.0)) changed = true;
|
|
}
|
|
//--- Rebuild when stale/missing and ANY needed source has rows: a temporarily dead
|
|
//--- source leaves its columns empty (header intact) and the panel 0-fills them -
|
|
//--- degradation must never take the working features down with it (the silent-FRED
|
|
//--- incident took exactly that shape under the old all-sources-required gate).
|
|
//--- GEX is a forward RECORDER, deliberately outside the feature pipeline: it appends
|
|
//--- its own daily file and never touches `changed`, because it feeds no model yet.
|
|
UpdateGex(m_specs[si].canonical);
|
|
bool anyReady = (needVix && m_vix.rows > 0) || (needUsd && m_usd.rows > 0)
|
|
|| (needCot && m_cot.rows > 0) || (needEia && m_eia.rows > 0)
|
|
|| (needIvol && m_ivol.rows > 0) || (needMac && m_dgs10.rows > 0);
|
|
string outFile = AltDataFileSymbol(symbol) + "_D1.csv"; // sanitizer shared with the panel
|
|
if((changed || !FileIsExist(ALTFETCH_DIR + outFile, FILE_COMMON)) && anyReady)
|
|
return RebuildFeatures(si, outFile);
|
|
return false;
|
|
}
|
|
|
|
protected:
|
|
//--- --------------------------------- catalog-driven feature writer
|
|
//--- Walks the daily grid and emits exactly export.py's format; the transforms are the
|
|
//--- FIXED a-priori constants documented there. Any feature change must change
|
|
//--- research/altdata/export.py and FeatureValue() together, APPEND-ONLY - the .cfg
|
|
//--- name-list pin protects existing models either way.
|
|
bool RebuildFeatures(int si, string outFile)
|
|
{
|
|
string feats[];
|
|
int nf = StringSplit(m_specs[si].features, ';', feats);
|
|
if(nf <= 0)
|
|
return false;
|
|
//--- weekly COT net-spec series with the catalog sign applied
|
|
double cspec[];
|
|
ArrayResize(cspec, m_cot.rows);
|
|
for(int i = 0; i < m_cot.rows; i++)
|
|
cspec[i] = (m_cot.v1[i] > 0) ? m_specs[si].cotSign * (m_cot.v2[i] - m_cot.v3[i]) / m_cot.v1[i] : 0.0;
|
|
//--- EIA crude-stocks copy for the rolling percentile
|
|
double estk[];
|
|
ArrayResize(estk, m_eia.rows);
|
|
for(int i = 0; i < m_eia.rows; i++)
|
|
estk[i] = m_eia.v1[i];
|
|
//--- temp + swap, same reasoning as SaveRaw: panels on other charts read this file
|
|
string tmp = ALTFETCH_DIR + outFile + ".tmp";
|
|
int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
string header = "date";
|
|
for(int f = 0; f < nf; f++)
|
|
header += ";" + feats[f];
|
|
FileWriteString(h, header + "\n");
|
|
datetime today = TimeCurrent();
|
|
for(datetime day = ALTFETCH_GRID_START; day <= today; day += 86400)
|
|
{
|
|
string line = TimeToString(day, TIME_DATE);
|
|
bool any = false;
|
|
for(int f = 0; f < nf; f++)
|
|
{
|
|
string val = FeatureValue(feats[f], day, cspec, estk);
|
|
if(val != "")
|
|
any = true;
|
|
line += ";" + val;
|
|
}
|
|
if(any)
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
FileClose(h);
|
|
if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + outFile, FILE_COMMON | FILE_REWRITE))
|
|
{
|
|
PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - feature file not updated.",
|
|
tmp, outFile, GetLastError());
|
|
FileDelete(tmp, FILE_COMMON);
|
|
return false;
|
|
}
|
|
PrintFormat("AltDataFetch: %s rebuilt (EA-maintained) - %d features: %s.",
|
|
outFile, nf, m_specs[si].features);
|
|
return true;
|
|
}
|
|
|
|
//--- One feature value at one day; "" while its source history is not yet deep enough.
|
|
//--- Each branch does its own as-of lookup (binary search, cheap at rebuild frequency),
|
|
//--- so adding a source never widens a parameter list again.
|
|
string FeatureValue(string id, datetime day,
|
|
const double &cspec[], const double &estk[])
|
|
{
|
|
if(id == "vix_chg5")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 5) ? DoubleToString((m_vix.v1[i] - m_vix.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "vix")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 0) ? DoubleToString(m_vix.v1[i] / 100.0, 6) : "";
|
|
}
|
|
//--- the chart's OWN implied-vol index; same a-priori scales as vix/vix_chg5 so the
|
|
//--- pairs are directly comparable to the first batch-norm layer
|
|
if(id == "ivol_chg5")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 5) ? DoubleToString((m_ivol.v1[i] - m_ivol.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "ivol")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 0) ? DoubleToString(m_ivol.v1[i] / 100.0, 6) : "";
|
|
}
|
|
if(id == "usd_chg5")
|
|
{
|
|
int i = AsOf(m_usd, day);
|
|
return (i >= 5) ? DoubleToString(m_usd.v1[i] - m_usd.v1[i - 5], 6) : "";
|
|
}
|
|
if(id == "cot_spec_net")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 0 && m_cot.v1[i] > 0) ? DoubleToString(cspec[i], 6) : "";
|
|
}
|
|
if(id == "cot_idx_1y" || id == "cot_idx_3y")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
if(i < 0)
|
|
return "";
|
|
bool oneYear = (id == "cot_idx_1y");
|
|
double r = RollingPctRank(cspec, i, oneYear ? 52 : 156, oneYear ? 26 : 78);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "cot_chg_4w")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 4) ? DoubleToString(cspec[i] - cspec[i - 4], 6) : "";
|
|
}
|
|
if(id == "eia_stk_idx1y")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
if(i < 0 || estk[i] <= 0)
|
|
return "";
|
|
double r = RollingPctRank(estk, i, 52, 26);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "eia_stk_chg4")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 4 && estk[i] > 0 && estk[i - 4] > 0)
|
|
? DoubleToString((estk[i] / estk[i - 4] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "eia_util")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 0 && m_eia.v3[i] > 0)
|
|
? DoubleToString((m_eia.v3[i] - 90.0) / 10.0, 6) : "";
|
|
}
|
|
//--- US macro block. Index arithmetic is in OBSERVATIONS (business days for the
|
|
//--- daily series, months for CPI/UNRATE), matching the research screen exactly.
|
|
if(id == "mac_y10")
|
|
{
|
|
int i = AsOf(m_dgs10, day);
|
|
return (i >= 20) ? DoubleToString(m_dgs10.v1[i] - m_dgs10.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_curve")
|
|
{
|
|
int i = AsOf(m_curve, day);
|
|
return (i >= 0) ? DoubleToString(m_curve.v1[i], 6) : "";
|
|
}
|
|
if(id == "mac_bei")
|
|
{
|
|
int i = AsOf(m_bei, day);
|
|
return (i >= 20) ? DoubleToString(m_bei.v1[i] - m_bei.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_gap")
|
|
{
|
|
int i = AsOf(m_dff, day), j = AsOf(m_ecb, day);
|
|
return (i >= 0 && j >= 0)
|
|
? DoubleToString((m_dff.v1[i] - m_ecb.v1[j]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_cpi")
|
|
{
|
|
int i = AsOf(m_cpi, day);
|
|
return (i >= 12 && m_cpi.v1[i - 12] > 0)
|
|
? DoubleToString((m_cpi.v1[i] / m_cpi.v1[i - 12] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_unemp")
|
|
{
|
|
int i = AsOf(m_unrate, day);
|
|
return (i >= 12) ? DoubleToString((m_unrate.v1[i] - m_unrate.v1[i - 12]) / 10.0, 6) : "";
|
|
}
|
|
return "";
|
|
}
|
|
};
|