forked from animatedread/Warrior_EA
04d70fb748
Two literal duplications the scan found, both of the kind where a divergence is silent: - Training.mqh stashed the era-loop resume context at FOUR yield points, seven identical assignments each (pass 1 differing only in i-1). A field missed at one of them resumes the next chunk against a different era than the one that yielded, and nothing reports it until the numbers drift. Now StashEraResume(). - AltDataFetch grew its five parallel arrays inline in three places. They are one record split across five buffers, so a resize missed on any one reads out of range on the NEXT append, not at the site of the mistake. Now AltSeriesAppend(), which returns the new index and zero-fills; callers set only the columns their source has. Braces balance across every in-scope file. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1415 lines
62 KiB
MQL5
1415 lines
62 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| AltDataFetch.mqh |
|
|
//| AnimateDread |
|
|
//| missing history at attach time and keeps appending forward while |
|
|
//| deployed, so online learning never depends on an external Python |
|
|
//| process. research/altdata remains the RESEARCH side (screening, |
|
|
//| new-source adjudication); this module maintains the exact same |
|
|
//| files in production. |
|
|
//+------------------------------------------------------------------+
|
|
#include "AltData.mqh"
|
|
|
|
#define ALTFETCH_DIR "Warrior_EA\\AltData\\"
|
|
#define ALTFETCH_TIMEOUT_MS 15000
|
|
//--- the SPX option chain is ~13 MB; the small feeds' 15 s would time out on a slow link
|
|
#define ALTFETCH_GEX_TIMEOUT_MS 90000
|
|
#define ALTFETCH_RETRY_SECONDS 3600
|
|
#define ALTFETCH_GRID_START D'2010.01.01'
|
|
|
|
//--- one raw series: weekly COT positioning, a daily FRED series, or the EIA WPSR block
|
|
struct SAltRawSeries
|
|
{
|
|
datetime date[]; // observation date, ascending
|
|
datetime published[];
|
|
double v1[]; // FRED: value | COT: open interest | EIA: crude stocks ex-SPR
|
|
double v2[]; // COT: spec long | EIA: field production
|
|
double v3[]; // COT: spec short | EIA: refinery utilization %
|
|
int rows;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Append one empty row and return its index. |
|
|
//| |
|
|
//| The five arrays are ONE record split across five buffers, so a |
|
|
//| resize missed on any of them reads out of range on the next |
|
|
//| append rather than failing here. Callers fill the values they |
|
|
//| have; the rest stay 0. |
|
|
//+------------------------------------------------------------------+
|
|
int AltSeriesAppend(SAltRawSeries &s, const datetime date, const datetime published)
|
|
{
|
|
int n = s.rows;
|
|
ArrayResize(s.date, n + 1);
|
|
ArrayResize(s.published, n + 1);
|
|
ArrayResize(s.v1, n + 1);
|
|
ArrayResize(s.v2, n + 1);
|
|
ArrayResize(s.v3, n + 1);
|
|
s.date[n] = date;
|
|
s.published[n] = published;
|
|
s.v1[n] = 0.0;
|
|
s.v2[n] = 0.0;
|
|
s.v3[n] = 0.0;
|
|
s.rows = n + 1;
|
|
return n;
|
|
}
|
|
//--- one symbol catalog row: which sources feed the instrument and which feature columns its
|
|
//--- {SYM}_D1.csv carries (column order = EA input order, APPEND-ONLY per the .cfg name pin)
|
|
struct SAltSymbolSpec
|
|
{
|
|
string canonical; // catalog key - also names the shared COT raw cache
|
|
string aliases; // ';'-separated broker names; matched EXACT first, then as a
|
|
// PREFIX so suffixed variants (US500.cash, XAUUSDm, EURUSD.r)
|
|
// resolve without being listed one by one
|
|
string cotWhere; // SoQL predicate selecting the CFTC contract; "" = no COT leg
|
|
double cotSign; // +1 futures quote matches the chart; -1 inverted (JPY/CAD futures are FX/USD)
|
|
string features; // ';'-separated feature ids, in CSV column order
|
|
string ivolSeries;// FRED id of THIS instrument's own implied-vol index; "" = none
|
|
};
|
|
|
|
class CAltDataFetch
|
|
{
|
|
protected:
|
|
string m_fredKey;
|
|
bool m_keyLoaded;
|
|
string m_eiaKey;
|
|
bool m_eiaWarned;
|
|
datetime m_lastGexDate; // last UTC day whose GEX row is already on disk
|
|
bool m_gexDateLoaded;
|
|
string m_blockedHost[]; // 4014 backoff, PER HOST - one missing whitelist entry
|
|
datetime m_blockedUntil[]; // must not silence the hosts that are whitelisted
|
|
bool m_urlAlerted; // full four-line reference printed once per session
|
|
datetime m_lastAttempt[16]; // per-source retry throttle: 0=VIX 1=USD 2=COT 3=EIA 4=IVOL
|
|
// 5=DGS10 6=T10Y2Y 7=T5YIE 8=DFF 9=ECBDFR 10=CPI 11=UNRATE
|
|
|
|
//--- ---------------------------------------------------------- utils
|
|
//--- The FredApiKey input is the key's home - it travels with the EA and survives the
|
|
//--- Common\Files wipe that starts every fresh test (the 2026-08-16 incident: keys.txt
|
|
//--- was wiped with the folder, FredKey() returned "" with NO log line, COT updated but
|
|
//--- FRED never ran and the SP500 rebuild - gated on all three raws - never fired).
|
|
//--- keys.txt is only the fallback for a blanked input, re-read on every hourly-throttled
|
|
//--- attempt until a key is found; m_keyLoaded means "warned once", never fail silently.
|
|
string FredKey(void)
|
|
{
|
|
if(FredApiKey != "")
|
|
return FredApiKey;
|
|
if(m_fredKey != "")
|
|
return m_fredKey;
|
|
int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
if(!m_keyLoaded)
|
|
Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt "
|
|
"does not exist in Common\\Files - FRED features (VIX, USD index) cannot update, "
|
|
"so the SP500/XAUUSD feature files will not build. Set the FredApiKey input "
|
|
"(free key: fred.stlouisfed.org) or create keys.txt with one line: fred=<key>. "
|
|
"The file is re-checked hourly; no restart needed.");
|
|
m_keyLoaded = true;
|
|
return "";
|
|
}
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringFind(line, "fred=") == 0)
|
|
m_fredKey = StringSubstr(line, 5);
|
|
}
|
|
FileClose(h);
|
|
if(m_fredKey == "" && !m_keyLoaded)
|
|
Print("AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt has "
|
|
"no 'fred=<key>' line - FRED-based features will not update until one is provided "
|
|
"(free key: fred.stlouisfed.org).");
|
|
m_keyLoaded = true;
|
|
return m_fredKey;
|
|
}
|
|
|
|
//--- same contract as FredKey: input first, keys.txt "eia=" line as fallback, loud once
|
|
string EiaKey(void)
|
|
{
|
|
if(EiaApiKey != "")
|
|
return EiaApiKey;
|
|
if(m_eiaKey != "")
|
|
return m_eiaKey;
|
|
int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h != INVALID_HANDLE)
|
|
{
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringFind(line, "eia=") == 0)
|
|
m_eiaKey = StringSubstr(line, 4);
|
|
}
|
|
FileClose(h);
|
|
}
|
|
if(m_eiaKey == "" && !m_eiaWarned)
|
|
{
|
|
m_eiaWarned = true;
|
|
Print("AltDataFetch: the EiaApiKey input is blank and no 'eia=<key>' line exists in " +
|
|
ALTFETCH_DIR + "keys.txt - the EIA feature columns will stay empty (panel 0-fills; "
|
|
"free key: eia.gov/opendata). Re-checked hourly; no restart needed.");
|
|
}
|
|
return m_eiaKey;
|
|
}
|
|
|
|
//--- "https://host/..." -> "https://host" (what the MT5 whitelist dialog wants)
|
|
string HostOf(string url)
|
|
{
|
|
int p = StringFind(url, "://");
|
|
int q = StringFind(url, "/", p < 0 ? 0 : p + 3);
|
|
return (q < 0) ? url : StringSubstr(url, 0, q);
|
|
}
|
|
|
|
//--- URL safe for the JOURNAL: api_key values masked. The 2026-08-16 EIA failure
|
|
//--- printed the first 80 chars of the URL - which included most of the key.
|
|
//--- Every error path that echoes a URL must go through this.
|
|
string MaskUrl(string url)
|
|
{
|
|
int p = StringFind(url, "api_key=");
|
|
if(p < 0)
|
|
return StringSubstr(url, 0, 100);
|
|
int e = StringFind(url, "&", p);
|
|
string masked = StringSubstr(url, 0, p + 8) + "***";
|
|
if(e > 0)
|
|
masked += StringSubstr(url, e);
|
|
return StringSubstr(masked, 0, 100);
|
|
}
|
|
|
|
int BlockedIndex(string host)
|
|
{
|
|
for(int i = 0; i < ArraySize(m_blockedHost); i++)
|
|
if(m_blockedHost[i] == host)
|
|
return i;
|
|
return -1;
|
|
}
|
|
|
|
bool HttpGet(string url, string &body, int timeoutMs = ALTFETCH_TIMEOUT_MS)
|
|
{
|
|
//--- PER-HOST backoff. An in-flight request cannot be cancelled; refusing to START another
|
|
//--- one is the whole of the remedy.
|
|
if(IsStopped())
|
|
return false;
|
|
string host = HostOf(url);
|
|
int bi = BlockedIndex(host);
|
|
if(bi >= 0 && TimeCurrent() < m_blockedUntil[bi])
|
|
return false;
|
|
char data[], result[];
|
|
string rh;
|
|
ResetLastError();
|
|
//--- identify ourselves: some API gateways reject requests with an empty User-Agent
|
|
//--- at the transport level (candidate cause of the EIA 1003 on first live fetch)
|
|
int status = WebRequest("GET", url, "User-Agent: WarriorEA/1.0\r\n", timeoutMs, data, result, rh);
|
|
if(status == -1)
|
|
{
|
|
int err = GetLastError();
|
|
if(err == 4014)
|
|
{
|
|
//--- Backoff, not a permanent latch: the whitelist takes effect for a RUNNING EA
|
|
//--- the moment the user saves the options dialog, so retrying hourly picks the
|
|
//--- fix up without a re-attach.
|
|
bool firstTimeThisHost = (bi < 0);
|
|
if(bi < 0)
|
|
{
|
|
bi = ArraySize(m_blockedHost);
|
|
ArrayResize(m_blockedHost, bi + 1);
|
|
ArrayResize(m_blockedUntil, bi + 1);
|
|
m_blockedHost[bi] = host;
|
|
}
|
|
m_blockedUntil[bi] = TimeCurrent() + 3600;
|
|
if(firstTimeThisHost)
|
|
{
|
|
//--- ALWAYS name the offender - the whole point of this message is telling
|
|
//--- the user which entry is missing, not that "something" is
|
|
Alert("Warrior EA: web request to ", host, " is blocked (4014) - this exact ",
|
|
"address is missing from the WebRequest whitelist (or the checkbox is ",
|
|
"off). Copy-paste line and steps are in the Experts log.");
|
|
Print("AltDataFetch: ==================================================================");
|
|
Print("AltDataFetch: BLOCKED host (add this exact line as its own whitelist entry):");
|
|
Print(host);
|
|
Print("AltDataFetch: MT5 menu: Tools > Options > Expert Advisors ->");
|
|
Print("AltDataFetch: tick 'Allow WebRequest for listed URL', add the line, click OK.");
|
|
Print("AltDataFetch: Takes effect immediately - the EA retries within the hour.");
|
|
if(!m_urlAlerted)
|
|
{
|
|
m_urlAlerted = true;
|
|
Print("AltDataFetch: For reference, the FULL set this EA uses (each its own entry):");
|
|
Print("https://publicreporting.cftc.gov");
|
|
Print("https://api.stlouisfed.org");
|
|
Print("https://api.eia.gov");
|
|
Print("https://cdn.cboe.com");
|
|
Print("AltDataFetch: Until fixed it runs normally on cached alt-data files - features");
|
|
Print("AltDataFetch: stay valid but stop updating past their last downloaded date.");
|
|
}
|
|
Print("AltDataFetch: ==================================================================");
|
|
}
|
|
else
|
|
Print("AltDataFetch: still blocked (4014): " + host + " - retrying in an hour.");
|
|
}
|
|
else
|
|
PrintFormat("AltDataFetch: WebRequest error %d for %s", err, MaskUrl(url));
|
|
return false;
|
|
}
|
|
if(status != 200)
|
|
{
|
|
PrintFormat("AltDataFetch: HTTP %d for %s%s", status, MaskUrl(url),
|
|
status >= 1000 ? " (1xxx = MT5 transport-layer failure, not a server "
|
|
"response - retried on the normal hourly schedule)" : "");
|
|
return false;
|
|
}
|
|
body = CharArrayToString(result, 0, WHOLE_ARRAY, CP_UTF8);
|
|
return true;
|
|
}
|
|
|
|
//--- minimal targeted JSON scan: value of "key":"..." (or numeric) at/after position `from`.
|
|
//--- Sufficient for Socrata/FRED responses, which are flat arrays of flat objects.
|
|
string JsonField(const string &body, string key, int from, int &valueEnd)
|
|
{
|
|
valueEnd = -1;
|
|
string tag = "\"" + key + "\":";
|
|
int p = StringFind(body, tag, from);
|
|
if(p < 0)
|
|
return "";
|
|
p += StringLen(tag);
|
|
if(StringGetCharacter(body, p) == '"')
|
|
{
|
|
int q = StringFind(body, "\"", p + 1);
|
|
if(q < 0)
|
|
return "";
|
|
valueEnd = q + 1;
|
|
return StringSubstr(body, p + 1, q - p - 1);
|
|
}
|
|
int e = p;
|
|
while(e < StringLen(body))
|
|
{
|
|
ushort c = StringGetCharacter(body, e);
|
|
if((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')
|
|
e++;
|
|
else
|
|
break;
|
|
}
|
|
valueEnd = e;
|
|
return StringSubstr(body, p, e - p);
|
|
}
|
|
|
|
datetime SaturdayAfter(datetime reportDate)
|
|
{
|
|
MqlDateTime st;
|
|
TimeToStruct(reportDate, st);
|
|
int days = (6 - st.day_of_week + 7) % 7; // MQL5: 0=Sunday .. 6=Saturday
|
|
if(days == 0)
|
|
days = 7;
|
|
return reportDate + days * 86400;
|
|
}
|
|
|
|
string UrlEncodePart(string s)
|
|
{
|
|
string out = "";
|
|
for(int i = 0; i < StringLen(s); i++)
|
|
{
|
|
ushort c = StringGetCharacter(s, i);
|
|
//--- '%' MUST come first and MUST be escaped: SoQL like-predicates use it as the
|
|
//--- wildcard ('EURO FX -%'), and an unescaped one turns the rest of the query into
|
|
//--- a malformed percent-escape.
|
|
if(c == '%') out += "%25";
|
|
else if(c == ' ') out += "%20";
|
|
else if(c == '\'') out += "%27";
|
|
else if(c == '&') out += "%26";
|
|
else if(c == '>') out += "%3E";
|
|
else if(c == '=') out += "%3D";
|
|
else if(c == '+') out += "%2B";
|
|
else if(c == '#') out += "%23";
|
|
else out += ShortToString(c);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
//--- ------------------------------------------ raw cache load/save
|
|
bool LoadRaw(string name, SAltRawSeries &s, int cols)
|
|
{
|
|
s.rows = 0;
|
|
int h = FileOpen(ALTFETCH_DIR + name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
FileReadString(h); // header
|
|
int cap = 2048;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
string f[];
|
|
if(StringSplit(line, ';', f) < cols + 2)
|
|
continue;
|
|
if(s.rows >= cap)
|
|
{
|
|
cap = cap * 3 / 2;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
}
|
|
s.date[s.rows] = StringToTime(f[0]);
|
|
s.published[s.rows] = StringToTime(f[1]);
|
|
s.v1[s.rows] = StringToDouble(f[2]);
|
|
s.v2[s.rows] = cols > 1 ? StringToDouble(f[3]) : 0.0;
|
|
s.v3[s.rows] = cols > 2 ? StringToDouble(f[4]) : 0.0;
|
|
s.rows++;
|
|
}
|
|
FileClose(h);
|
|
return s.rows > 0;
|
|
}
|
|
|
|
//--- Whole-file rewrites go through a temp + FileMove swap: multiple charts share these caches
|
|
//--- (raw_VIXCLS serves every symbol; two timeframes of one symbol share all of them), and a
|
|
//--- reader hitting a truncate-then-write mid-flight parses a torn file.
|
|
bool SaveRaw(string name, SAltRawSeries &s, int cols, string header)
|
|
{
|
|
string tmp = ALTFETCH_DIR + name + ".tmp";
|
|
int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d)", tmp, GetLastError());
|
|
return false;
|
|
}
|
|
FileWriteString(h, header + "\n");
|
|
for(int i = 0; i < s.rows; i++)
|
|
{
|
|
string line = TimeToString(s.date[i], TIME_DATE) + ";" + TimeToString(s.published[i], TIME_DATE) + ";"
|
|
+ DoubleToString(s.v1[i], 6);
|
|
if(cols > 1)
|
|
line += ";" + DoubleToString(s.v2[i], 6);
|
|
if(cols > 2)
|
|
line += ";" + DoubleToString(s.v3[i], 6);
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
FileClose(h);
|
|
if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + name, FILE_COMMON | FILE_REWRITE))
|
|
{
|
|
PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - cache not updated this pass.",
|
|
tmp, name, GetLastError());
|
|
FileDelete(tmp, FILE_COMMON);
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
//--- ------------------------------------------------- FRED source appends observations after
|
|
//--- the cached tail; full history when the cache is empty vmin/vmax: per-series plausibility
|
|
//--- band. Rejected rows are counted and reported, never silently dropped.
|
|
bool UpdateFred(string seriesId, SAltRawSeries &s, int throttleSlot,
|
|
int staleDays = 4, double vmin = -1e18, double vmax = 1e18)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- daily series: current until ~2 business days behind (weekends/holidays).
|
|
//--- monthly series pass staleDays ~32 so a 30-day-old latest print does not
|
|
//--- trigger a pointless fetch attempt every hour for weeks.
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < staleDays * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string key = FredKey();
|
|
if(key == "")
|
|
return false;
|
|
string url = "https://api.stlouisfed.org/fred/series/observations?series_id=" + seriesId +
|
|
"&api_key=" + key + "&file_type=json&limit=100000";
|
|
//--- ALWAYS bound the start. 2005 leaves 5y of lookback margin ahead of the grid for the
|
|
//--- longest window (3y COT percentile analog).
|
|
string startDate = (s.rows > 0)
|
|
? TimeToString(s.date[s.rows - 1] + 86400, TIME_DATE)
|
|
: "2005.01.01";
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; FRED wants ISO dashes
|
|
url += "&observation_start=" + startDate;
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0, rejected = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = JsonField(body, "date", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2;
|
|
string v = JsonField(body, "value", e1, e2);
|
|
pos = (e2 > e1 ? e2 : e1);
|
|
if(v == "" || v == ".")
|
|
continue;
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || (s.rows > 0 && dt <= s.date[s.rows - 1]))
|
|
continue;
|
|
double val = StringToDouble(v);
|
|
if(val < vmin || val > vmax || !MathIsValidNumber(val))
|
|
{
|
|
rejected++;
|
|
continue;
|
|
}
|
|
int n = AltSeriesAppend(s, dt, dt + 86400); // unrevised daily: knowable next day
|
|
s.v1[n] = val;
|
|
added++;
|
|
}
|
|
if(rejected > 0)
|
|
PrintFormat("AltDataFetch: %s REJECTED %d value(s) outside the plausibility band "
|
|
"[%g, %g] - transport corruption or format drift; investigate if repeated.",
|
|
seriesId, rejected, vmin, vmax);
|
|
if(added > 0)
|
|
{
|
|
SaveRaw("raw_" + seriesId + ".csv", s, 1, "date;published;value");
|
|
PrintFormat("AltDataFetch: %s +%d rows (through %s)", seriesId, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: %s returned a response with ZERO usable observations and "
|
|
"the cache is empty - series id wrong or API format changed. Features "
|
|
"depending on it will stay empty (panel 0-fills).", seriesId);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- -------------------------------------------------- COT source
|
|
bool UpdateCot(string cacheName, string cotWhere, SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly: a new Tuesday report exists once the last one is >7d old; allow 3 extra
|
|
//--- days before nagging the API (Friday release + weekend)
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string where = cotWhere;
|
|
if(s.rows > 0)
|
|
{
|
|
string lastDate = TimeToString(s.date[s.rows - 1], TIME_DATE);
|
|
StringReplace(lastDate, ".", "-"); // ISO for the SoQL timestamp comparison
|
|
where += " AND report_date_as_yyyy_mm_dd > '" + lastDate + "'";
|
|
}
|
|
string url = "https://publicreporting.cftc.gov/resource/gpe5-46if.json?%24select=" +
|
|
"report_date_as_yyyy_mm_dd,open_interest_all,lev_money_positions_long,lev_money_positions_short" +
|
|
"&%24where=" + UrlEncodePart(where) +
|
|
"&%24order=report_date_as_yyyy_mm_dd&%24limit=50000";
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = JsonField(body, "report_date_as_yyyy_mm_dd", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2, e3, e4;
|
|
string oi = JsonField(body, "open_interest_all", e1, e2);
|
|
string ll = JsonField(body, "lev_money_positions_long", e2, e3);
|
|
string ls = JsonField(body, "lev_money_positions_short", e3, e4);
|
|
pos = (e4 > e1 ? e4 : e1);
|
|
datetime dt = StringToTime(StringSubstr(d, 0, 10));
|
|
if(dt <= 0 || oi == "" || ll == "" || ls == "")
|
|
continue;
|
|
if(s.rows > 0 && dt == s.date[s.rows - 1])
|
|
{
|
|
//--- same report date from a second contract-name variant: keep the larger-OI row
|
|
if(StringToDouble(oi) > s.v1[s.rows - 1])
|
|
{
|
|
s.v1[s.rows - 1] = StringToDouble(oi);
|
|
s.v2[s.rows - 1] = StringToDouble(ll);
|
|
s.v3[s.rows - 1] = StringToDouble(ls);
|
|
}
|
|
continue;
|
|
}
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
int n = AltSeriesAppend(s, dt, SaturdayAfter(dt));
|
|
s.v1[n] = StringToDouble(oi);
|
|
s.v2[n] = StringToDouble(ll);
|
|
s.v3[n] = StringToDouble(ls);
|
|
added++;
|
|
}
|
|
if(added > 0)
|
|
{
|
|
SaveRaw(cacheName, s, 3, "date;published;oi;lev_long;lev_short");
|
|
PrintFormat("AltDataFetch: %s +%d weekly reports (through %s)", cacheName, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: COT query for %s returned ZERO reports on an empty cache - "
|
|
"the contract-name predicate probably matches nothing at the endpoint "
|
|
"(the like-predicates for non-SP500/JPY contracts are unverified; see the "
|
|
"catalog). COT features stay empty (panel 0-fills) until it is corrected.",
|
|
cacheName);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ---------------------------------------------------- EIA source Weekly Petroleum Status
|
|
//--- Report via the EIA v2 API, three series in one call (2010->now x 3 ~ 2,600 rows, under the
|
|
//--- API's 5000-row page - revisit if series are added).
|
|
bool UpdateEia(SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly cadence, same nag guard as COT
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < 10 * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
string key = EiaKey();
|
|
if(key == "")
|
|
return false;
|
|
string startDate = TimeToString(s.rows > 0 ? s.date[s.rows - 1] + 86400 : ALTFETCH_GRID_START, TIME_DATE);
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; the API wants ISO dashes
|
|
string url = "https://api.eia.gov/v2/petroleum/sum/sndw/data/?api_key=" + key +
|
|
"&frequency=weekly&data%5B0%5D=value" +
|
|
"&facets%5Bseries%5D%5B0%5D=WCESTUS1" + // crude stocks ex-SPR
|
|
"&facets%5Bseries%5D%5B1%5D=WCRFPUS2" + // field production
|
|
"&facets%5Bseries%5D%5B2%5D=WPULEUS3" + // refinery utilization %
|
|
"&sort%5B0%5D%5Bcolumn%5D=period&sort%5B0%5D%5Bdirection%5D=asc" +
|
|
"&length=5000&start=" + startDate;
|
|
string body;
|
|
if(!HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1, e2, e3;
|
|
string d = JsonField(body, "period", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
string sid = JsonField(body, "series", e1, e2);
|
|
string v = JsonField(body, "value", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || sid == "" || v == "")
|
|
continue;
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
if(s.rows == 0 || dt > s.date[s.rows - 1])
|
|
{
|
|
int n = AltSeriesAppend(s, dt, dt + 6 * 86400);
|
|
added++;
|
|
}
|
|
int r = s.rows - 1;
|
|
if(sid == "WCESTUS1") s.v1[r] = StringToDouble(v);
|
|
else if(sid == "WCRFPUS2") s.v2[r] = StringToDouble(v);
|
|
else if(sid == "WPULEUS3") s.v3[r] = StringToDouble(v);
|
|
}
|
|
if(added > 0)
|
|
{
|
|
SaveRaw("raw_EIA_WPSR.csv", s, 3, "date;published;stocks;production;utilization");
|
|
PrintFormat("AltDataFetch: raw_EIA_WPSR.csv +%d weekly reports (through %s)", added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ------------------------------------ GEX forward recorder WHY THIS IS A RECORDER AND NOT A
|
|
//--- FEATURE: option open interest is a SNAPSHOT source. Revisit when the file holds a year or
|
|
//--- so, then screen it like every other candidate.
|
|
string CboeTicker(string canonical)
|
|
{
|
|
if(canonical == "SP500") return "_SPX"; // index options - where dealer positioning concentrates
|
|
if(canonical == "NAS100") return "_NDX";
|
|
if(canonical == "US30") return "_DJX";
|
|
if(canonical == "US2000") return "IWM";
|
|
if(canonical == "XAUUSD") return "GLD";
|
|
if(canonical == "XAGUSD") return "SLV";
|
|
if(canonical == "XTIUSD") return "USO";
|
|
if(canonical == "BTCUSD") return "IBIT";
|
|
return ""; // no listed-options proxy worth recording
|
|
}
|
|
|
|
//--- last date already in the file, or 0
|
|
datetime LastGexDate(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 1)
|
|
return 0;
|
|
return StringToTime(f[0]);
|
|
}
|
|
|
|
bool UpdateGex(string canonical)
|
|
{
|
|
string ticker = CboeTicker(canonical);
|
|
if(ticker == "")
|
|
return false;
|
|
//--- Once per UTC day, after the US cash close (21:00 UTC covers both DST regimes),
|
|
//--- and never on a weekend - the chain would just repeat Friday's settled numbers.
|
|
MqlDateTime g;
|
|
TimeToStruct(TimeGMT(), g);
|
|
if(g.day_of_week == 0 || g.day_of_week == 6)
|
|
return false;
|
|
if(g.hour < 21)
|
|
return false;
|
|
string file = ALTFETCH_DIR + "gex_" + canonical + ".csv";
|
|
datetime today = StringToTime(StringFormat("%04d.%02d.%02d", g.year, g.mon, g.day));
|
|
if(!m_gexDateLoaded)
|
|
{
|
|
m_lastGexDate = LastGexDate(file);
|
|
m_gexDateLoaded = true;
|
|
}
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- Re-read from DISK before spending the download: two charts of the same symbol
|
|
//--- (e.g. D1 + H4) each run their own fetcher, and a per-instance cached date would
|
|
//--- let the second chart append a duplicate row for the day the first just recorded.
|
|
m_lastGexDate = LastGexDate(file);
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- the SPX chain is ~13 MB: this needs a far longer timeout than the small feeds
|
|
string body;
|
|
if(!HttpGet("https://cdn.cboe.com/api/global/delayed_quotes/options/" + ticker + ".json",
|
|
body, ALTFETCH_GEX_TIMEOUT_MS))
|
|
return false;
|
|
if(StringLen(body) < 1000)
|
|
{
|
|
Print("AltDataFetch: GEX response for " + ticker + " was too short to be a chain - skipping.");
|
|
return false;
|
|
}
|
|
int e0;
|
|
string spotS = JsonField(body, "current_price", 0, e0); // absent from option rows, so
|
|
if(spotS == "") // a plain scan finds the header one
|
|
spotS = JsonField(body, "close", 0, e0);
|
|
double spot = StringToDouble(spotS);
|
|
if(spot <= 0.0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - no usable spot price in the response, skipping.");
|
|
return false;
|
|
}
|
|
//--- per-expiry accumulation (54 expiries on SPX; the cap is generous)
|
|
int expCode[128];
|
|
double expGex[128];
|
|
int nExp = 0;
|
|
double gexCall = 0.0, gexPut = 0.0;
|
|
double oiCall = 0.0, oiPut = 0.0;
|
|
int rows = 0;
|
|
int pos = 0;
|
|
//--- Row layout is option, ..., open_interest, volume, delta, gamma - so one forward
|
|
//--- sweep reads each row's three fields in the order they appear.
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string sym = JsonField(body, "option", pos, e1);
|
|
if(sym == "" || e1 < 0)
|
|
break;
|
|
int e2, e3;
|
|
string oiS = JsonField(body, "open_interest", e1, e2);
|
|
string gS = JsonField(body, "gamma", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
double oi = StringToDouble(oiS);
|
|
double gam = StringToDouble(gS);
|
|
if(oi <= 0.0 || gam == 0.0)
|
|
continue; // no exposure - skip the string work entirely
|
|
//--- OCC symbol: <root><YYMMDD><C|P><strike*1000, 8 digits>, so the tail is fixed
|
|
//--- width regardless of how long the root is (SPX, SPXW, IWM, ...).
|
|
int len = StringLen(sym);
|
|
if(len < 15)
|
|
continue;
|
|
string kind = StringSubstr(sym, len - 9, 1);
|
|
int expiry = (int)StringToInteger(StringSubstr(sym, len - 15, 6));
|
|
//--- dollar gamma per 1% move: gamma x OI x 100 (contract multiplier) x S^2 x 0.01
|
|
double dollar = gam * oi * 100.0 * spot * spot * 0.01;
|
|
if(kind == "C")
|
|
{
|
|
gexCall += dollar;
|
|
oiCall += oi;
|
|
}
|
|
else
|
|
{
|
|
gexPut -= dollar; // dealers short puts: opposite sign convention
|
|
oiPut += oi;
|
|
dollar = -dollar;
|
|
}
|
|
int slot = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
if(expCode[i] == expiry)
|
|
{
|
|
slot = i;
|
|
break;
|
|
}
|
|
if(slot < 0 && nExp < 128)
|
|
{
|
|
slot = nExp++;
|
|
expCode[slot] = expiry;
|
|
expGex[slot] = 0.0;
|
|
}
|
|
if(slot >= 0)
|
|
expGex[slot] += dollar;
|
|
rows++;
|
|
}
|
|
if(rows == 0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - parsed 0 contracts with exposure, skipping "
|
|
"(the CDN response format may have changed).");
|
|
return false;
|
|
}
|
|
//--- Partial-chain guard: a truncated 13 MB download parses cleanly and yields a
|
|
//--- plausible-but-wrong GEX. A day-over-day contract-count collapse (>50%) is the
|
|
//--- fingerprint of truncation, not of markets - OI does not halve overnight.
|
|
int prevRows = LastGexRowCount(file);
|
|
if(prevRows > 0 && rows < prevRows / 2)
|
|
{
|
|
PrintFormat("AltDataFetch: GEX %s - only %d contracts vs %d yesterday (<50%%): response "
|
|
"looks TRUNCATED, not recording. Will retry on the next timer pass.",
|
|
ticker, rows, prevRows);
|
|
return false;
|
|
}
|
|
//--- three nearest expiries, by selection (nExp is tiny)
|
|
double front[3] = {0.0, 0.0, 0.0};
|
|
int frontCode = 0, prevCode = 0;
|
|
for(int k = 0; k < 3; k++)
|
|
{
|
|
int best = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
{
|
|
if(expCode[i] <= prevCode)
|
|
continue;
|
|
if(best < 0 || expCode[i] < expCode[best])
|
|
best = i;
|
|
}
|
|
if(best < 0)
|
|
break;
|
|
front[k] = expGex[best] / 1e9;
|
|
prevCode = expCode[best];
|
|
if(k == 0)
|
|
frontCode = prevCode;
|
|
}
|
|
bool isNew = !FileIsExist(file, FILE_COMMON);
|
|
int h = FileOpen(file, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d) - GEX not recorded today.", file, GetLastError());
|
|
return false;
|
|
}
|
|
if(isNew)
|
|
FileWriteString(h, "date;recorded_utc;spot;gex_total;gex_call;gex_put;oi_call;oi_put;"
|
|
"gex_exp1;gex_exp2;gex_exp3;expiry1;n_expiry;n_rows\n");
|
|
FileSeek(h, 0, SEEK_END);
|
|
FileWriteString(h, StringFormat("%s;%s;%.4f;%.6f;%.6f;%.6f;%.0f;%.0f;%.6f;%.6f;%.6f;%d;%d;%d\n",
|
|
TimeToString(today, TIME_DATE),
|
|
TimeToString(TimeGMT(), TIME_DATE | TIME_MINUTES),
|
|
spot, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9,
|
|
oiCall, oiPut, front[0], front[1], front[2],
|
|
frontCode, nExp, rows));
|
|
FileClose(h);
|
|
m_lastGexDate = today;
|
|
//--- Progress matters here: the file is useless until it is long enough to screen, and
|
|
//--- the only way to know how far along it is, is to say so.
|
|
int have = 1;
|
|
datetime probe = LastGexDate(file);
|
|
if(probe > 0)
|
|
have = CountGexRows(file);
|
|
PrintFormat("AltDataFetch: GEX %s recorded - total %+.2f Bn/1%% (calls %+.2f, puts %+.2f), "
|
|
"%d contracts, %d expiries, spot %.2f. %d day(s) banked; this is a FORWARD "
|
|
"RECORDER - no free history exists, so it feeds no model until there is enough "
|
|
"to screen (~250 trading days).",
|
|
ticker, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9, rows, nExp, spot, have);
|
|
return true;
|
|
}
|
|
|
|
//--- n_rows (field 14) of the last data line - the previous session's contract count
|
|
int LastGexRowCount(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 14)
|
|
return 0;
|
|
return (int)StringToInteger(f[13]);
|
|
}
|
|
|
|
int CountGexRows(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
int n = 0;
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
n++;
|
|
}
|
|
FileClose(h);
|
|
return n;
|
|
}
|
|
|
|
//--- -------------------------------- feature math (mirrors export.py)
|
|
//--- pandas rolling(W, min_periods=M).rank(pct=True) on the LAST element: rank of the current
|
|
//--- value among the window's values (average rank for ties) divided by the window count.
|
|
double RollingPctRank(const double &v[], int i, int window, int minPeriods)
|
|
{
|
|
int lo = MathMax(0, i - window + 1);
|
|
int n = i - lo + 1;
|
|
if(n < minPeriods)
|
|
return EMPTY_VALUE;
|
|
double cur = v[i];
|
|
int below = 0, ties = 0;
|
|
for(int k = lo; k <= i; k++)
|
|
{
|
|
if(v[k] < cur)
|
|
below++;
|
|
else if(v[k] == cur)
|
|
ties++;
|
|
}
|
|
double rank = below + (1.0 + ties) / 2.0;
|
|
return rank / n;
|
|
}
|
|
|
|
//--- as-of index: largest row with published <= day, else -1
|
|
int AsOf(const SAltRawSeries &s, datetime day)
|
|
{
|
|
int lo = 0, hi = s.rows - 1, ans = -1;
|
|
while(lo <= hi)
|
|
{
|
|
int mid = (lo + hi) / 2;
|
|
if(s.published[mid] <= day)
|
|
{
|
|
ans = mid;
|
|
lo = mid + 1;
|
|
}
|
|
else
|
|
hi = mid - 1;
|
|
}
|
|
return ans;
|
|
}
|
|
|
|
//--- raw series kept resident between timer ticks (one chart = one symbol, so only the
|
|
//--- sources the chart's catalog row names are ever populated)
|
|
//--- m_ivol holds whichever instrument-specific implied-vol index THIS chart's catalog
|
|
//--- row names (GVZ for gold, OVX for oil, VXN/VXD/RVX for the index symbols) - one
|
|
//--- chart is one symbol, so a single slot serves regardless of which series it is.
|
|
SAltRawSeries m_vix, m_usd, m_cot, m_eia, m_ivol;
|
|
//--- US macro block. UNRATE is the exception: it IS revised (seasonal refits, ~0.1-0.2pp), there
|
|
//--- is no unrevised variant, and the EA cannot run the ALFRED first-print protocol - so a fresh
|
|
//--- backfill carries small revisions into history.
|
|
SAltRawSeries m_dgs10, m_curve, m_bei, m_dff, m_ecb, m_cpi, m_unrate;
|
|
|
|
//--- ------------------------------- symbol catalog (data, not code) An MT5 symbol cannot be
|
|
//--- mapped to a CFTC contract by rule - the naming universes are unrelated and every broker
|
|
//--- invents its own index tickers - so the catalog IS the mechanism: the chart symbol selects a
|
|
//--- row at runtime and ONLY that row's sources are fetched.
|
|
SAltSymbolSpec m_specs[];
|
|
string m_userFrom[]; // broker symbol (from symbol_map.cfg / the dialog)
|
|
string m_userTo[]; // canonical, or "NONE" = user declined alt data
|
|
bool m_userMapLoaded;
|
|
|
|
void AddSpec(string canonical, string aliases, string cotWhere,
|
|
double cotSign, string feats, string ivolSeries = "")
|
|
{
|
|
int n = ArraySize(m_specs);
|
|
ArrayResize(m_specs, n + 1);
|
|
m_specs[n].canonical = canonical;
|
|
m_specs[n].aliases = aliases;
|
|
m_specs[n].cotWhere = cotWhere;
|
|
m_specs[n].cotSign = cotSign;
|
|
m_specs[n].features = feats;
|
|
m_specs[n].ivolSeries = ivolSeries;
|
|
}
|
|
|
|
void BuildCatalog(void)
|
|
{
|
|
//--- feature bundles (locals, deliberately prefixed so no macro elsewhere can collide) POLICY
|
|
//--- (user directive 2026-08-16, twice): every feature the fetched sources can serve is wired
|
|
//--- on every symbol - the research screens inform priors but do NOT gate the input list ("we
|
|
//--- will leave the NN to do its job").
|
|
string f_eia = "eia_stk_idx1y;eia_stk_chg4;eia_util";
|
|
string f_risk = "vix_chg5;vix;usd_chg5"; // equity-risk complex
|
|
string f_cot = "cot_idx_1y;cot_idx_3y;cot_chg_4w"; // COT positioning family
|
|
//--- US macro block: 10y yield 20d change, curve slope, 5y breakeven 20d change, Fed-ECB
|
|
//--- policy gap, CPI yoy, unemployment 12m change.
|
|
string f_mac = "mac_y10;mac_curve;mac_bei;mac_gap;mac_cpi;mac_unemp";
|
|
//--- CFTC predicates. A predicate that matches nothing costs only an empty column.
|
|
string w_es = "market_and_exchange_names in('E-MINI S&P 500 - CHICAGO MERCANTILE EXCHANGE',"
|
|
"'E-MINI S&P 500 STOCK INDEX - CHICAGO MERCANTILE EXCHANGE')";
|
|
string w_jpy = "market_and_exchange_names in('JAPANESE YEN - CHICAGO MERCANTILE EXCHANGE')";
|
|
string w_eur = "market_and_exchange_names like 'EURO FX -%'";
|
|
string w_gbp = "market_and_exchange_names like 'BRITISH POUND%'";
|
|
string w_cad = "market_and_exchange_names like 'CANADIAN DOLLAR%'";
|
|
string w_aud = "market_and_exchange_names like 'AUSTRALIAN DOLLAR%'";
|
|
string w_nzd = "market_and_exchange_names like 'NZ DOLLAR%'";
|
|
string w_chf = "market_and_exchange_names like 'SWISS FRANC%'";
|
|
string w_btc = "market_and_exchange_names like 'BITCOIN%'";
|
|
|
|
//--- ---------------- equity indices (VIX/USD complex; only SP500 is screened)
|
|
//--- Screened findings preserved as priors, not gates: SP500 vix_chg5/vix/usd_chg5/
|
|
//--- cot_spec_net and USDJPY COT family cleared family-wise + incremental; XAUUSD
|
|
//--- gvz_chg5 (GVZ = gold's own IV) is the campaign's 2nd-strongest incremental
|
|
//--- feature (MI|vol 0.0197 p=0.002, 4.6x vix_chg5 on gold); EURUSD/USDJPY vix_chg5
|
|
//--- screened clean; the macro block screened null everywhere; EIA screened null on
|
|
//--- WTI. All ship regardless - see the policy note above.
|
|
AddSpec("SP500", "SP500;SPX500;US500;USA500;USA500IDX;US_500;SPXUSD;S&P500;SP_500;"
|
|
"SPX;USA500IDXUSD;US500Cash", w_es, 1.0,
|
|
f_risk + ";cot_spec_net;" + f_eia + ";" + f_mac);
|
|
AddSpec("NAS100", "NAS100;NASDAQ;US100;USA100;USATEC;USTEC;NDX100;NQ100;TECH100;"
|
|
"USA100IDX;US_100;NDX;USATECHIDX", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXNCLS");
|
|
AddSpec("US30", "US30;DJ30;DOW30;USA30;WALLSTREET;WS30;DJIUSD;USA30IDX;"
|
|
"US_30;DOWJONES;DJI", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "VXDCLS");
|
|
AddSpec("US2000", "US2000;RUSSELL2000;RUT;USA2000;RTY;US_2000", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "RVXCLS");
|
|
AddSpec("DE40", "DE40;GER40;DAX40;DAX;GER30;DE30;GERMANY40;DEU40;GRXEUR;"
|
|
"DE_40;GER_40", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("UK100", "UK100;FTSE100;FTSE;GBR100;UKX;GB100;UK_100;BRXGBP", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("JP225", "JP225;NIKKEI;NI225;JPN225;J225;JPXJPY;JP_225", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("EU50", "EU50;STOXX50;ESTX50;EUSTX50;SX5E;EUR50;E50EUR", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("FR40", "FR40;CAC40;FRA40;CAC;FRXEUR", "", 1.0, f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("AU200", "AU200;ASX200;AUS200;SPI200;AU_200;AUXAUD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("HK50", "HK50;HSI;HKG33;HK33;HANGSENG;HKXHKD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- FX majors (full complement: COT + risk complex + EIA + macro;
|
|
//--- no per-instrument IV - EVZ was discontinued 2025-03 and must never be wired)
|
|
AddSpec("EURUSD", "EURUSD;EUR_USD", w_eur, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDJPY", "USDJPY;USD_JPY", w_jpy, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("GBPUSD", "GBPUSD;GBP_USD", w_gbp, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDCAD", "USDCAD;USD_CAD", w_cad, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("AUDUSD", "AUDUSD;AUD_USD", w_aud, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("NZDUSD", "NZDUSD;NZD_USD", w_nzd, 1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
AddSpec("USDCHF", "USDCHF;USD_CHF", w_chf, -1.0, f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- metals (gold/silver COT is in the DISAGGREGATED CFTC dataset -
|
|
//--- different id AND column names - still not wired; silver has no free IV index)
|
|
AddSpec("XAUUSD", "XAUUSD;GOLD;XAU_USD;GOLDUSD", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "GVZCLS");
|
|
AddSpec("XAGUSD", "XAGUSD;SILVER;XAG_USD;SILVERUSD", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- energy
|
|
AddSpec("XTIUSD", "XTIUSD;USOIL;WTI;CRUDE;USCRUDE;OIL;WTICOUSD;"
|
|
"OILUSD;LIGHTCMDUSD;CRUDEOIL", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
|
|
AddSpec("XBRUSD", "XBRUSD;UKOIL;BRENT;UKOUSD;BCOUSD;BRENTCMDUSD", "", 1.0,
|
|
f_risk + ";ivol_chg5;ivol;" + f_eia + ";" + f_mac, "OVXCLS");
|
|
AddSpec("NATGAS", "NATGAS;NGAS;XNGUSD;NATURALGAS", "", 1.0,
|
|
f_risk + ";" + f_eia + ";" + f_mac);
|
|
|
|
//--- ---------------- crypto
|
|
AddSpec("BTCUSD", "BTCUSD;BITCOIN;XBTUSD;BTC_USD", w_btc, 1.0,
|
|
f_cot + ";" + f_risk + ";" + f_eia + ";" + f_mac);
|
|
}
|
|
|
|
//--- ------------------------------------- user mapping persistence
|
|
//--- symbol_map.cfg: one "BROKERSYMBOL=CANONICAL" per line, written by the mapping
|
|
//--- dialog. "NONE" records a deliberate decline so the dialog never nags again.
|
|
void LoadUserMap(void)
|
|
{
|
|
if(m_userMapLoaded)
|
|
return;
|
|
m_userMapLoaded = true;
|
|
int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return;
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
StringTrimLeft(line);
|
|
StringTrimRight(line);
|
|
if(line == "" || StringGetCharacter(line, 0) == '#')
|
|
continue;
|
|
string kv[];
|
|
if(StringSplit(line, '=', kv) != 2)
|
|
continue;
|
|
int n = ArraySize(m_userFrom);
|
|
ArrayResize(m_userFrom, n + 1);
|
|
ArrayResize(m_userTo, n + 1);
|
|
m_userFrom[n] = kv[0];
|
|
m_userTo[n] = kv[1];
|
|
}
|
|
FileClose(h);
|
|
if(ArraySize(m_userFrom) > 0)
|
|
PrintFormat("AltDataFetch: %d user symbol mapping(s) loaded from symbol_map.cfg.",
|
|
ArraySize(m_userFrom));
|
|
}
|
|
|
|
//--- exact, then prefix, over one row's alias list
|
|
bool AliasMatches(int spec, string symbol, bool prefixPass)
|
|
{
|
|
string a[];
|
|
int n = StringSplit(m_specs[spec].aliases, ';', a);
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
if(a[i] == "")
|
|
continue;
|
|
if(!prefixPass && symbol == a[i])
|
|
return true;
|
|
//--- require the alias to be a strict prefix AND at least 3 chars, so a short
|
|
//--- ticker can never swallow an unrelated symbol
|
|
if(prefixPass && StringLen(a[i]) >= 3 && StringFind(symbol, a[i]) == 0)
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
int FindSpec(string symbol)
|
|
{
|
|
LoadUserMap();
|
|
//--- a user mapping wins over everything, including a catalog row for the same name
|
|
for(int u = 0; u < ArraySize(m_userFrom); u++)
|
|
{
|
|
if(m_userFrom[u] != symbol)
|
|
continue;
|
|
if(m_userTo[u] == "NONE")
|
|
return -2; // deliberately declined - do not ask again
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(m_specs[i].canonical == m_userTo[u])
|
|
return i;
|
|
PrintFormat("AltDataFetch: symbol_map.cfg maps %s to unknown catalog entry '%s' - ignoring.",
|
|
symbol, m_userTo[u]);
|
|
}
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(AliasMatches(i, symbol, false))
|
|
return i;
|
|
for(int i = 0; i < ArraySize(m_specs); i++)
|
|
if(AliasMatches(i, symbol, true))
|
|
return i;
|
|
return -1; // unknown - the dialog asks the user
|
|
}
|
|
|
|
public:
|
|
CAltDataFetch(void) : m_fredKey(""), m_keyLoaded(false),
|
|
m_eiaKey(""), m_eiaWarned(false),
|
|
m_lastGexDate(0), m_gexDateLoaded(false),
|
|
m_urlAlerted(false),
|
|
m_userMapLoaded(false)
|
|
{
|
|
for(int i = 0; i < ArraySize(m_lastAttempt); i++)
|
|
m_lastAttempt[i] = 0;
|
|
BuildCatalog();
|
|
}
|
|
|
|
//--- ------------------------------------- symbol mapping interface
|
|
bool NeedsMapping(string symbol) { return FindSpec(symbol) == -1; }
|
|
|
|
int CatalogCount(void) { return ArraySize(m_specs); }
|
|
string CatalogName(int i)
|
|
{
|
|
return (i >= 0 && i < ArraySize(m_specs)) ? m_specs[i].canonical : "";
|
|
}
|
|
//--- one-line description for the dropdown: canonical + what it would contribute
|
|
string CatalogLabel(int i)
|
|
{
|
|
if(i < 0 || i >= ArraySize(m_specs))
|
|
return "";
|
|
string f[];
|
|
int n = StringSplit(m_specs[i].features, ';', f);
|
|
return m_specs[i].canonical + " (" + IntegerToString(n) + " features)";
|
|
}
|
|
|
|
//--- Persist the user's answer and apply it immediately. canonical == "NONE" records a
|
|
//--- deliberate decline. Appends to symbol_map.cfg, replacing any previous line for the
|
|
//--- same broker symbol, so the answer survives restarts and a wiped feature folder.
|
|
bool SaveUserMapping(string symbol, string canonical)
|
|
{
|
|
LoadUserMap();
|
|
bool replaced = false;
|
|
for(int i = 0; i < ArraySize(m_userFrom); i++)
|
|
if(m_userFrom[i] == symbol)
|
|
{
|
|
m_userTo[i] = canonical;
|
|
replaced = true;
|
|
}
|
|
if(!replaced)
|
|
{
|
|
int n = ArraySize(m_userFrom);
|
|
ArrayResize(m_userFrom, n + 1);
|
|
ArrayResize(m_userTo, n + 1);
|
|
m_userFrom[n] = symbol;
|
|
m_userTo[n] = canonical;
|
|
}
|
|
int h = FileOpen(ALTFETCH_DIR + "symbol_map.cfg", FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write symbol_map.cfg (%d) - the mapping applies to "
|
|
"this session only and will be asked again next attach.", GetLastError());
|
|
return false;
|
|
}
|
|
FileWriteString(h, "# Warrior EA alt-data symbol map: BROKERSYMBOL=CANONICAL (NONE = no alt data)\n");
|
|
for(int i = 0; i < ArraySize(m_userFrom); i++)
|
|
FileWriteString(h, m_userFrom[i] + "=" + m_userTo[i] + "\n");
|
|
FileClose(h);
|
|
PrintFormat("AltDataFetch: %s mapped to %s (saved in symbol_map.cfg).", symbol, canonical);
|
|
return true;
|
|
}
|
|
|
|
//--- Maintain the raw caches and the feature CSV for `symbol`. Returns true when the feature
|
|
//--- file was rebuilt (caller should reload the panels).
|
|
bool Update(string symbol)
|
|
{
|
|
int si = FindSpec(symbol);
|
|
if(si < 0)
|
|
return false; // unknown symbol or user declined - nothing to serve
|
|
bool needVix = StringFind(m_specs[si].features, "vix") >= 0;
|
|
bool needUsd = StringFind(m_specs[si].features, "usd_") >= 0;
|
|
bool needEia = StringFind(m_specs[si].features, "eia_") >= 0;
|
|
bool needCot = (m_specs[si].cotWhere != "");
|
|
bool needIvol = (m_specs[si].ivolSeries != "" && StringFind(m_specs[si].features, "ivol") >= 0);
|
|
bool needMac = StringFind(m_specs[si].features, "mac_") >= 0;
|
|
bool changed = false;
|
|
//--- plausibility bands: generous enough for any real print in the series' history
|
|
//--- (VIX peaked 89 in 2008; utilization-style checks live in FeatureValue), tight
|
|
//--- enough that transport garbage cannot pass
|
|
if(needVix)
|
|
{
|
|
if(m_vix.rows == 0)
|
|
LoadRaw("raw_VIXCLS.csv", m_vix, 1);
|
|
if(UpdateFred("VIXCLS", m_vix, 0, 4, 1.0, 200.0))
|
|
changed = true;
|
|
}
|
|
if(needUsd)
|
|
{
|
|
//--- staleDays 10: the H.10 dollar index posts with ~a week's lag (live fetch
|
|
//--- 2026-08-16 was current only through 08-07), so a 4-day horizon would fire a
|
|
//--- futile refetch attempt every hour in the gap between releases
|
|
if(m_usd.rows == 0)
|
|
LoadRaw("raw_DTWEXBGS.csv", m_usd, 1);
|
|
if(UpdateFred("DTWEXBGS", m_usd, 1, 10, 50.0, 250.0))
|
|
changed = true;
|
|
}
|
|
if(needCot)
|
|
{
|
|
//--- cache is named by CANONICAL, so two brokers' names for the same instrument
|
|
//--- share one download instead of fetching the same contract twice
|
|
string cache = "raw_COT_" + m_specs[si].canonical + ".csv";
|
|
if(m_cot.rows == 0)
|
|
LoadRaw(cache, m_cot, 3);
|
|
if(UpdateCot(cache, m_specs[si].cotWhere, m_cot, 2))
|
|
changed = true;
|
|
}
|
|
if(needEia)
|
|
{
|
|
if(m_eia.rows == 0)
|
|
LoadRaw("raw_EIA_WPSR.csv", m_eia, 3);
|
|
if(UpdateEia(m_eia, 3))
|
|
changed = true;
|
|
}
|
|
if(needIvol)
|
|
{
|
|
if(m_ivol.rows == 0)
|
|
LoadRaw("raw_" + m_specs[si].ivolSeries + ".csv", m_ivol, 1);
|
|
if(UpdateFred(m_specs[si].ivolSeries, m_ivol, 4, 4, 1.0, 300.0))
|
|
changed = true;
|
|
}
|
|
if(needMac)
|
|
{
|
|
if(m_dgs10.rows == 0) LoadRaw("raw_DGS10.csv", m_dgs10, 1);
|
|
if(m_curve.rows == 0) LoadRaw("raw_T10Y2Y.csv", m_curve, 1);
|
|
if(m_bei.rows == 0) LoadRaw("raw_T5YIE.csv", m_bei, 1);
|
|
if(m_dff.rows == 0) LoadRaw("raw_DFF.csv", m_dff, 1);
|
|
if(m_ecb.rows == 0) LoadRaw("raw_ECBDFR.csv", m_ecb, 1);
|
|
if(m_cpi.rows == 0) LoadRaw("raw_CPIAUCNS.csv", m_cpi, 1);
|
|
if(m_unrate.rows == 0) LoadRaw("raw_UNRATE.csv", m_unrate, 1);
|
|
if(UpdateFred("DGS10", m_dgs10, 5, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("T10Y2Y", m_curve, 6, 4, -10.0, 10.0)) changed = true;
|
|
if(UpdateFred("T5YIE", m_bei, 7, 4, -5.0, 15.0)) changed = true;
|
|
if(UpdateFred("DFF", m_dff, 8, 4, -2.0, 30.0)) changed = true;
|
|
if(UpdateFred("ECBDFR", m_ecb, 9, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("CPIAUCNS", m_cpi, 10, 32, 20.0, 1000.0)) changed = true;
|
|
if(UpdateFred("UNRATE", m_unrate, 11, 32, 0.0, 35.0)) changed = true;
|
|
}
|
|
//--- Rebuild when stale/missing and ANY needed source has rows: a temporarily dead source
|
|
//--- leaves its columns empty (header intact) and the panel 0-fills them - degradation must
|
|
//--- never take the working features down with it (the silent-FRED incident took exactly that
|
|
//--- shape under the old all-sources-required gate).
|
|
UpdateGex(m_specs[si].canonical);
|
|
bool anyReady = (needVix && m_vix.rows > 0) || (needUsd && m_usd.rows > 0)
|
|
|| (needCot && m_cot.rows > 0) || (needEia && m_eia.rows > 0)
|
|
|| (needIvol && m_ivol.rows > 0) || (needMac && m_dgs10.rows > 0);
|
|
string outFile = AltDataFileSymbol(symbol) + "_D1.csv"; // sanitizer shared with the panel
|
|
if((changed || !FileIsExist(ALTFETCH_DIR + outFile, FILE_COMMON)) && anyReady)
|
|
return RebuildFeatures(si, outFile);
|
|
return false;
|
|
}
|
|
|
|
protected:
|
|
//--- --------------------------------- catalog-driven feature writer Walks the daily grid and
|
|
//--- emits exactly export.py's format; the transforms are the FIXED a-priori constants
|
|
//--- documented there.
|
|
bool RebuildFeatures(int si, string outFile)
|
|
{
|
|
string feats[];
|
|
int nf = StringSplit(m_specs[si].features, ';', feats);
|
|
if(nf <= 0)
|
|
return false;
|
|
//--- weekly COT net-spec series with the catalog sign applied
|
|
double cspec[];
|
|
ArrayResize(cspec, m_cot.rows);
|
|
for(int i = 0; i < m_cot.rows; i++)
|
|
cspec[i] = (m_cot.v1[i] > 0) ? m_specs[si].cotSign * (m_cot.v2[i] - m_cot.v3[i]) / m_cot.v1[i] : 0.0;
|
|
//--- EIA crude-stocks copy for the rolling percentile
|
|
double estk[];
|
|
ArrayResize(estk, m_eia.rows);
|
|
for(int i = 0; i < m_eia.rows; i++)
|
|
estk[i] = m_eia.v1[i];
|
|
//--- temp + swap, same reasoning as SaveRaw: panels on other charts read this file
|
|
string tmp = ALTFETCH_DIR + outFile + ".tmp";
|
|
int h = FileOpen(tmp, FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
string header = "date";
|
|
for(int f = 0; f < nf; f++)
|
|
header += ";" + feats[f];
|
|
FileWriteString(h, header + "\n");
|
|
datetime today = TimeCurrent();
|
|
for(datetime day = ALTFETCH_GRID_START; day <= today; day += 86400)
|
|
{
|
|
string line = TimeToString(day, TIME_DATE);
|
|
bool any = false;
|
|
for(int f = 0; f < nf; f++)
|
|
{
|
|
string val = FeatureValue(feats[f], day, cspec, estk);
|
|
if(val != "")
|
|
any = true;
|
|
line += ";" + val;
|
|
}
|
|
if(any)
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
FileClose(h);
|
|
if(!FileMove(tmp, FILE_COMMON, ALTFETCH_DIR + outFile, FILE_COMMON | FILE_REWRITE))
|
|
{
|
|
PrintFormat("AltDataFetch: FileMove %s -> %s failed (%d) - feature file not updated.",
|
|
tmp, outFile, GetLastError());
|
|
FileDelete(tmp, FILE_COMMON);
|
|
return false;
|
|
}
|
|
PrintFormat("AltDataFetch: %s rebuilt (EA-maintained) - %d features: %s.",
|
|
outFile, nf, m_specs[si].features);
|
|
return true;
|
|
}
|
|
|
|
//--- One feature value at one day; "" while its source history is not yet deep enough.
|
|
//--- Each branch does its own as-of lookup (binary search, cheap at rebuild frequency),
|
|
//--- so adding a source never widens a parameter list again.
|
|
string FeatureValue(string id, datetime day,
|
|
const double &cspec[], const double &estk[])
|
|
{
|
|
if(id == "vix_chg5")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 5) ? DoubleToString((m_vix.v1[i] - m_vix.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "vix")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 0) ? DoubleToString(m_vix.v1[i] / 100.0, 6) : "";
|
|
}
|
|
//--- the chart's OWN implied-vol index; same a-priori scales as vix/vix_chg5 so the
|
|
//--- pairs are directly comparable to the first batch-norm layer
|
|
if(id == "ivol_chg5")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 5) ? DoubleToString((m_ivol.v1[i] - m_ivol.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "ivol")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 0) ? DoubleToString(m_ivol.v1[i] / 100.0, 6) : "";
|
|
}
|
|
if(id == "usd_chg5")
|
|
{
|
|
int i = AsOf(m_usd, day);
|
|
return (i >= 5) ? DoubleToString(m_usd.v1[i] - m_usd.v1[i - 5], 6) : "";
|
|
}
|
|
if(id == "cot_spec_net")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 0 && m_cot.v1[i] > 0) ? DoubleToString(cspec[i], 6) : "";
|
|
}
|
|
if(id == "cot_idx_1y" || id == "cot_idx_3y")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
if(i < 0)
|
|
return "";
|
|
bool oneYear = (id == "cot_idx_1y");
|
|
double r = RollingPctRank(cspec, i, oneYear ? 52 : 156, oneYear ? 26 : 78);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "cot_chg_4w")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 4) ? DoubleToString(cspec[i] - cspec[i - 4], 6) : "";
|
|
}
|
|
if(id == "eia_stk_idx1y")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
if(i < 0 || estk[i] <= 0)
|
|
return "";
|
|
double r = RollingPctRank(estk, i, 52, 26);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "eia_stk_chg4")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 4 && estk[i] > 0 && estk[i - 4] > 0)
|
|
? DoubleToString((estk[i] / estk[i - 4] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "eia_util")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 0 && m_eia.v3[i] > 0)
|
|
? DoubleToString((m_eia.v3[i] - 90.0) / 10.0, 6) : "";
|
|
}
|
|
//--- US macro block. Index arithmetic is in OBSERVATIONS (business days for the
|
|
//--- daily series, months for CPI/UNRATE), matching the research screen exactly.
|
|
if(id == "mac_y10")
|
|
{
|
|
int i = AsOf(m_dgs10, day);
|
|
return (i >= 20) ? DoubleToString(m_dgs10.v1[i] - m_dgs10.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_curve")
|
|
{
|
|
int i = AsOf(m_curve, day);
|
|
return (i >= 0) ? DoubleToString(m_curve.v1[i], 6) : "";
|
|
}
|
|
if(id == "mac_bei")
|
|
{
|
|
int i = AsOf(m_bei, day);
|
|
return (i >= 20) ? DoubleToString(m_bei.v1[i] - m_bei.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_gap")
|
|
{
|
|
int i = AsOf(m_dff, day), j = AsOf(m_ecb, day);
|
|
return (i >= 0 && j >= 0)
|
|
? DoubleToString((m_dff.v1[i] - m_ecb.v1[j]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_cpi")
|
|
{
|
|
int i = AsOf(m_cpi, day);
|
|
return (i >= 12 && m_cpi.v1[i - 12] > 0)
|
|
? DoubleToString((m_cpi.v1[i] / m_cpi.v1[i - 12] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_unemp")
|
|
{
|
|
int i = AsOf(m_unrate, day);
|
|
return (i >= 12) ? DoubleToString((m_unrate.v1[i] - m_unrate.v1[i - 12]) / 10.0, 6) : "";
|
|
}
|
|
return "";
|
|
}
|
|
};
|