forked from animatedread/Warrior_EA
53de361e9c
CAltDataFetch (System/AltDataFetch.mqh) mixed six unrelated concerns in one 1406-line class. This is plain composition, not the Expert/AIBase view+adapter pattern - CAltDataFetch has no single-inheritance parent forcing an adapter, same shape as Database/DatabaseManager.mqh composing its four Database* managers. Extracted, grep-confirmed zero external callers of any moved method (only Warrior_EA.mq5's already-public Update/NeedsMapping/CatalogCount/CatalogName/ CatalogLabel/SaveUserMapping surface, unchanged): - CAltDataHttpClient (System/AltDataHttpClient.mqh): HttpGet/HostOf/MaskUrl/ BlockedIndex (4014 per-host backoff), JsonField, UrlEncodePart. Owns the m_blockedHost/m_blockedUntil/m_urlAlerted state - STATEFUL, moved verbatim. - CAltDataCatalog (System/AltDataCatalog.mqh): the SAltSymbolSpec catalog (AddSpec/BuildCatalog/AliasMatches/FindSpec) plus symbol_map.cfg user-mapping persistence (LoadUserMap/SaveUserMapping) and the public NeedsMapping/CatalogCount/CatalogName/CatalogLabel surface. Owns m_specs/m_userFrom/m_userTo/m_userMapLoaded - STATEFUL, moved verbatim. Added one new Spec(i) getter (by value - 4 strings + 1 double, cheap) so CAltDataFetch::Update()/RebuildFeatures() can read a resolved catalog row without reaching into the collaborator's array. - LoadRaw/SaveRaw (raw-series CSV <-> file mapping) had zero member-state dependency - turned into free functions AltRawLoad/AltRawSave, matching the file's own existing AltSeriesAppend precedent, instead of a needless class. CAltDataFetch itself keeps three concerns as a deliberate partial, same judgment already applied to Topology's boot sequence / Features' shared- indicator lifecycle elsewhere in this campaign: the four per-source fetch pipelines (UpdateFred/UpdateCot/UpdateEia/UpdateGex, including FredKey/EiaKey/ LoadCommonKey/ShouldAttemptFetch/the GEX CBOE helpers) and the feature-CSV builder (RebuildFeatures/FeatureValue/RollingPctRank/AsOf) both read/write the 9 SAltRawSeries caches kept resident on the orchestrator between timer ticks - splitting them out would mean either relocating that cache's ownership or a 9-13 parameter signature per method, a larger design decision better made as its own pass rather than forced through unattended given the finding's own "high risk" estimate. Every moved method body is copied verbatim (statement-by-statement diffed against git show HEAD~1:System/AltDataFetch.mqh) with only the mechanical substitution HttpGet/JsonField/UrlEncodePart -> m_http.*, LoadRaw/SaveRaw -> AltRawLoad/AltRawSave, FindSpec -> m_catalog.FindSpec, and m_specs[si].X -> spec.X (spec = m_catalog.Spec(si), resolved once per Update() call instead of re-indexing). ALTFETCH_TIMEOUT_MS/ALTFETCH_GEX_TIMEOUT_MS macros moved into AltDataHttpClient.mqh (their logical owner); ALTFETCH_DIR stays in AltDataFetch.mqh, defined before both new #includes since CAltDataCatalog's ctor path (symbol_map.cfg) and CAltDataHttpClient's HttpGet default param reference it. Self-compiled 0 errors, 0 warnings (_claude_stage, MetaEditor64 /compile), twice - once before and once after a stale doc-comment fix (a leftover "same reasoning as SaveRaw" mention updated to AltRawSave). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1008 lines
43 KiB
MQL5
1008 lines
43 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| AltDataFetch.mqh |
|
|
//| AnimateDread |
|
|
//| missing history at attach time and keeps appending forward while |
|
|
//| deployed, so online learning never depends on an external Python |
|
|
//| process. research/altdata remains the RESEARCH side (screening, |
|
|
//| new-source adjudication); this module maintains the exact same |
|
|
//| files in production. |
|
|
//+------------------------------------------------------------------+
|
|
#include "AltData.mqh"
|
|
#include "AtomicFile.mqh"
|
|
|
|
#define ALTFETCH_DIR "Warrior_EA\\AltData\\"
|
|
#define ALTFETCH_RETRY_SECONDS 3600
|
|
#define ALTFETCH_GRID_START D'2010.01.01'
|
|
|
|
//--- generic HTTP client (backoff/masking/JSON parse) and the symbol catalog + user-mapping
|
|
//--- persistence are separate collaborators - see those files' headers for why each is its own
|
|
//--- responsibility. Declared here, after ALTFETCH_DIR, since CAltDataCatalog's ctor path uses it.
|
|
#include "AltDataHttpClient.mqh"
|
|
#include "AltDataCatalog.mqh"
|
|
|
|
//--- one raw series: weekly COT positioning, a daily FRED series, or the EIA WPSR block
|
|
struct SAltRawSeries
|
|
{
|
|
datetime date[]; // observation date, ascending
|
|
datetime published[];
|
|
double v1[]; // FRED: value | COT: open interest | EIA: crude stocks ex-SPR
|
|
double v2[]; // COT: spec long | EIA: field production
|
|
double v3[]; // COT: spec short | EIA: refinery utilization %
|
|
int rows;
|
|
};
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| Append one empty row and return its index. |
|
|
//| |
|
|
//| The five arrays are ONE record split across five buffers, so a |
|
|
//| resize missed on any of them reads out of range on the next |
|
|
//| append rather than failing here. Callers fill the values they |
|
|
//| have; the rest stay 0. |
|
|
//+------------------------------------------------------------------+
|
|
int AltSeriesAppend(SAltRawSeries &s, const datetime date, const datetime published)
|
|
{
|
|
int n = s.rows;
|
|
ArrayResize(s.date, n + 1);
|
|
ArrayResize(s.published, n + 1);
|
|
ArrayResize(s.v1, n + 1);
|
|
ArrayResize(s.v2, n + 1);
|
|
ArrayResize(s.v3, n + 1);
|
|
s.date[n] = date;
|
|
s.published[n] = published;
|
|
s.v1[n] = 0.0;
|
|
s.v2[n] = 0.0;
|
|
s.v3[n] = 0.0;
|
|
s.rows = n + 1;
|
|
return n;
|
|
}
|
|
|
|
//--- Whole-file raw-series cache load/save (concern separated from the fetch pipelines that use
|
|
//--- it): pure functions, no member state - just the SAltRawSeries <-> ';'-delimited CSV mapping.
|
|
|
|
bool AltRawLoad(string name, SAltRawSeries &s, int cols)
|
|
{
|
|
s.rows = 0;
|
|
int h = FileOpen(ALTFETCH_DIR + name, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
FileReadString(h); // header
|
|
int cap = 2048;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
string f[];
|
|
if(StringSplit(line, ';', f) < cols + 2)
|
|
continue;
|
|
if(s.rows >= cap)
|
|
{
|
|
cap = cap * 3 / 2;
|
|
ArrayResize(s.date, cap);
|
|
ArrayResize(s.published, cap);
|
|
ArrayResize(s.v1, cap);
|
|
ArrayResize(s.v2, cap);
|
|
ArrayResize(s.v3, cap);
|
|
}
|
|
s.date[s.rows] = StringToTime(f[0]);
|
|
s.published[s.rows] = StringToTime(f[1]);
|
|
s.v1[s.rows] = StringToDouble(f[2]);
|
|
s.v2[s.rows] = cols > 1 ? StringToDouble(f[3]) : 0.0;
|
|
s.v3[s.rows] = cols > 2 ? StringToDouble(f[4]) : 0.0;
|
|
s.rows++;
|
|
}
|
|
FileClose(h);
|
|
return s.rows > 0;
|
|
}
|
|
|
|
//--- Whole-file rewrites go through System\AtomicFile.mqh's temp + FileMove swap: multiple
|
|
//--- charts share these caches (raw_VIXCLS serves every symbol; two timeframes of one symbol
|
|
//--- share all of them), and a reader hitting a truncate-then-write mid-flight parses a torn file.
|
|
bool AltRawSave(string name, SAltRawSeries &s, int cols, string header)
|
|
{
|
|
string tmpName = "";
|
|
int h = AtomicWriteBegin(ALTFETCH_DIR + name, FILE_COMMON, tmpName, FILE_TXT | FILE_ANSI);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d)", tmpName, GetLastError());
|
|
return false;
|
|
}
|
|
FileWriteString(h, header + "\n");
|
|
for(int i = 0; i < s.rows; i++)
|
|
{
|
|
string line = TimeToString(s.date[i], TIME_DATE) + ";" + TimeToString(s.published[i], TIME_DATE) + ";"
|
|
+ DoubleToString(s.v1[i], 6);
|
|
if(cols > 1)
|
|
line += ";" + DoubleToString(s.v2[i], 6);
|
|
if(cols > 2)
|
|
line += ";" + DoubleToString(s.v3[i], 6);
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
return AtomicWriteEnd(h, ALTFETCH_DIR + name, tmpName, FILE_COMMON, true, __FUNCTION__);
|
|
}
|
|
|
|
class CAltDataFetch
|
|
{
|
|
protected:
|
|
string m_fredKey;
|
|
bool m_keyLoaded;
|
|
string m_eiaKey;
|
|
bool m_eiaWarned;
|
|
datetime m_lastGexDate; // last UTC day whose GEX row is already on disk
|
|
bool m_gexDateLoaded;
|
|
datetime m_lastAttempt[16]; // per-source retry throttle: 0=VIX 1=USD 2=COT 3=EIA 4=IVOL
|
|
// 5=DGS10 6=T10Y2Y 7=T5YIE 8=DFF 9=ECBDFR 10=CPI 11=UNRATE
|
|
|
|
//--- ---------------------------------------------------------- utils
|
|
//--- Shared contract behind FredKey()/EiaKey(): the EA input is the key's home - it travels
|
|
//--- with the EA and survives the Common\Files wipe that starts every fresh test (the
|
|
//--- 2026-08-16 incident: keys.txt was wiped with the folder, FredKey() returned "" with NO
|
|
//--- log line, COT updated but FRED never ran and the SP500 rebuild - gated on all three raws -
|
|
//--- never fired). keys.txt is only the fallback for a blanked input, re-read on every
|
|
//--- hourly-throttled attempt until a key is found; warnedFlag means "warned once", never fail
|
|
//--- silently. fileMissingMsg == "" means "no distinct message for a missing file" (EIA's
|
|
//--- shape) - the missing-file case then just falls through to notFoundMsg below.
|
|
string LoadCommonKey(const string inputValue, string &cache, bool &warnedFlag,
|
|
const string prefix, const string fileMissingMsg, const string notFoundMsg)
|
|
{
|
|
if(inputValue != "")
|
|
return inputValue;
|
|
if(cache != "")
|
|
return cache;
|
|
int h = FileOpen(ALTFETCH_DIR + "keys.txt", FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
if(fileMissingMsg != "")
|
|
{
|
|
if(!warnedFlag)
|
|
Print(fileMissingMsg);
|
|
warnedFlag = true;
|
|
return "";
|
|
}
|
|
}
|
|
else
|
|
{
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringFind(line, prefix) == 0)
|
|
cache = StringSubstr(line, StringLen(prefix));
|
|
}
|
|
FileClose(h);
|
|
}
|
|
if(cache == "" && !warnedFlag)
|
|
{
|
|
warnedFlag = true;
|
|
Print(notFoundMsg);
|
|
}
|
|
return cache;
|
|
}
|
|
|
|
string FredKey(void)
|
|
{
|
|
return LoadCommonKey(FredApiKey, m_fredKey, m_keyLoaded, "fred=",
|
|
"AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt "
|
|
"does not exist in Common\\Files - FRED features (VIX, USD index) cannot update, "
|
|
"so the SP500/XAUUSD feature files will not build. Set the FredApiKey input "
|
|
"(free key: fred.stlouisfed.org) or create keys.txt with one line: fred=<key>. "
|
|
"The file is re-checked hourly; no restart needed.",
|
|
"AltDataFetch: the FredApiKey input is blank and " + ALTFETCH_DIR + "keys.txt has "
|
|
"no 'fred=<key>' line - FRED-based features will not update until one is provided "
|
|
"(free key: fred.stlouisfed.org).");
|
|
}
|
|
|
|
string EiaKey(void)
|
|
{
|
|
return LoadCommonKey(EiaApiKey, m_eiaKey, m_eiaWarned, "eia=", "",
|
|
"AltDataFetch: the EiaApiKey input is blank and no 'eia=<key>' line exists in " +
|
|
ALTFETCH_DIR + "keys.txt - the EIA feature columns will stay empty (panel 0-fills; "
|
|
"free key: eia.gov/opendata). Re-checked hourly; no restart needed.");
|
|
}
|
|
|
|
//--- Shared gate at the top of every per-source Update method (Fred/Cot/Eia): skip while the
|
|
//--- cached tail is still fresh (within staleDays), then skip if a fetch attempt for this
|
|
//--- source's throttle slot already happened within ALTFETCH_RETRY_SECONDS - stamping the
|
|
//--- attempt time only when neither check skips, so a genuine fetch below is exactly one
|
|
//--- attempt per hour. UpdateGex has its own unrelated day/hour gate and does not use this.
|
|
bool ShouldAttemptFetch(const SAltRawSeries &s, int staleDays, int throttleSlot, datetime now)
|
|
{
|
|
if(s.rows > 0 && now - s.date[s.rows - 1] < staleDays * 86400)
|
|
return false;
|
|
if(m_lastAttempt[throttleSlot] != 0 && now - m_lastAttempt[throttleSlot] < ALTFETCH_RETRY_SECONDS)
|
|
return false;
|
|
m_lastAttempt[throttleSlot] = now;
|
|
return true;
|
|
}
|
|
|
|
datetime SaturdayAfter(datetime reportDate)
|
|
{
|
|
MqlDateTime st;
|
|
TimeToStruct(reportDate, st);
|
|
int days = (6 - st.day_of_week + 7) % 7; // MQL5: 0=Sunday .. 6=Saturday
|
|
if(days == 0)
|
|
days = 7;
|
|
return reportDate + days * 86400;
|
|
}
|
|
|
|
//--- ------------------------------------------------- FRED source appends observations after
|
|
//--- the cached tail; full history when the cache is empty vmin/vmax: per-series plausibility
|
|
//--- band. Rejected rows are counted and reported, never silently dropped.
|
|
bool UpdateFred(string seriesId, SAltRawSeries &s, int throttleSlot,
|
|
int staleDays = 4, double vmin = -1e18, double vmax = 1e18)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- daily series: current until ~2 business days behind (weekends/holidays).
|
|
//--- monthly series pass staleDays ~32 so a 30-day-old latest print does not
|
|
//--- trigger a pointless fetch attempt every hour for weeks.
|
|
if(!ShouldAttemptFetch(s, staleDays, throttleSlot, now))
|
|
return false;
|
|
string key = FredKey();
|
|
if(key == "")
|
|
return false;
|
|
string url = "https://api.stlouisfed.org/fred/series/observations?series_id=" + seriesId +
|
|
"&api_key=" + key + "&file_type=json&limit=100000";
|
|
//--- ALWAYS bound the start. 2005 leaves 5y of lookback margin ahead of the grid for the
|
|
//--- longest window (3y COT percentile analog).
|
|
string startDate = (s.rows > 0)
|
|
? TimeToString(s.date[s.rows - 1] + 86400, TIME_DATE)
|
|
: "2005.01.01";
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; FRED wants ISO dashes
|
|
url += "&observation_start=" + startDate;
|
|
string body;
|
|
if(!m_http.HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0, rejected = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = m_http.JsonField(body, "date", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2;
|
|
string v = m_http.JsonField(body, "value", e1, e2);
|
|
pos = (e2 > e1 ? e2 : e1);
|
|
if(v == "" || v == ".")
|
|
continue;
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || (s.rows > 0 && dt <= s.date[s.rows - 1]))
|
|
continue;
|
|
double val = StringToDouble(v);
|
|
if(val < vmin || val > vmax || !MathIsValidNumber(val))
|
|
{
|
|
rejected++;
|
|
continue;
|
|
}
|
|
int n = AltSeriesAppend(s, dt, dt + 86400); // unrevised daily: knowable next day
|
|
s.v1[n] = val;
|
|
added++;
|
|
}
|
|
if(rejected > 0)
|
|
PrintFormat("AltDataFetch: %s REJECTED %d value(s) outside the plausibility band "
|
|
"[%g, %g] - transport corruption or format drift; investigate if repeated.",
|
|
seriesId, rejected, vmin, vmax);
|
|
if(added > 0)
|
|
{
|
|
AltRawSave("raw_" + seriesId + ".csv", s, 1, "date;published;value");
|
|
PrintFormat("AltDataFetch: %s +%d rows (through %s)", seriesId, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: %s returned a response with ZERO usable observations and "
|
|
"the cache is empty - series id wrong or API format changed. Features "
|
|
"depending on it will stay empty (panel 0-fills).", seriesId);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- -------------------------------------------------- COT source
|
|
bool UpdateCot(string cacheName, string cotWhere, SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly: a new Tuesday report exists once the last one is >7d old; allow 3 extra
|
|
//--- days before nagging the API (Friday release + weekend)
|
|
if(!ShouldAttemptFetch(s, 10, throttleSlot, now))
|
|
return false;
|
|
string where = cotWhere;
|
|
if(s.rows > 0)
|
|
{
|
|
string lastDate = TimeToString(s.date[s.rows - 1], TIME_DATE);
|
|
StringReplace(lastDate, ".", "-"); // ISO for the SoQL timestamp comparison
|
|
where += " AND report_date_as_yyyy_mm_dd > '" + lastDate + "'";
|
|
}
|
|
string url = "https://publicreporting.cftc.gov/resource/gpe5-46if.json?%24select=" +
|
|
"report_date_as_yyyy_mm_dd,open_interest_all,lev_money_positions_long,lev_money_positions_short" +
|
|
"&%24where=" + m_http.UrlEncodePart(where) +
|
|
"&%24order=report_date_as_yyyy_mm_dd&%24limit=50000";
|
|
string body;
|
|
if(!m_http.HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string d = m_http.JsonField(body, "report_date_as_yyyy_mm_dd", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
int e2, e3, e4;
|
|
string oi = m_http.JsonField(body, "open_interest_all", e1, e2);
|
|
string ll = m_http.JsonField(body, "lev_money_positions_long", e2, e3);
|
|
string ls = m_http.JsonField(body, "lev_money_positions_short", e3, e4);
|
|
pos = (e4 > e1 ? e4 : e1);
|
|
datetime dt = StringToTime(StringSubstr(d, 0, 10));
|
|
if(dt <= 0 || oi == "" || ll == "" || ls == "")
|
|
continue;
|
|
if(s.rows > 0 && dt == s.date[s.rows - 1])
|
|
{
|
|
//--- same report date from a second contract-name variant: keep the larger-OI row
|
|
if(StringToDouble(oi) > s.v1[s.rows - 1])
|
|
{
|
|
s.v1[s.rows - 1] = StringToDouble(oi);
|
|
s.v2[s.rows - 1] = StringToDouble(ll);
|
|
s.v3[s.rows - 1] = StringToDouble(ls);
|
|
}
|
|
continue;
|
|
}
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
int n = AltSeriesAppend(s, dt, SaturdayAfter(dt));
|
|
s.v1[n] = StringToDouble(oi);
|
|
s.v2[n] = StringToDouble(ll);
|
|
s.v3[n] = StringToDouble(ls);
|
|
added++;
|
|
}
|
|
if(added > 0)
|
|
{
|
|
AltRawSave(cacheName, s, 3, "date;published;oi;lev_long;lev_short");
|
|
PrintFormat("AltDataFetch: %s +%d weekly reports (through %s)", cacheName, added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
else if(s.rows == 0)
|
|
PrintFormat("AltDataFetch: COT query for %s returned ZERO reports on an empty cache - "
|
|
"the contract-name predicate probably matches nothing at the endpoint "
|
|
"(the like-predicates for non-SP500/JPY contracts are unverified; see the "
|
|
"catalog). COT features stay empty (panel 0-fills) until it is corrected.",
|
|
cacheName);
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ---------------------------------------------------- EIA source Weekly Petroleum Status
|
|
//--- Report via the EIA v2 API, three series in one call (2010->now x 3 ~ 2,600 rows, under the
|
|
//--- API's 5000-row page - revisit if series are added).
|
|
bool UpdateEia(SAltRawSeries &s, int throttleSlot)
|
|
{
|
|
datetime now = TimeCurrent();
|
|
//--- weekly cadence, same nag guard as COT
|
|
if(!ShouldAttemptFetch(s, 10, throttleSlot, now))
|
|
return false;
|
|
string key = EiaKey();
|
|
if(key == "")
|
|
return false;
|
|
string startDate = TimeToString(s.rows > 0 ? s.date[s.rows - 1] + 86400 : ALTFETCH_GRID_START, TIME_DATE);
|
|
StringReplace(startDate, ".", "-"); // TimeToString is yyyy.mm.dd; the API wants ISO dashes
|
|
string url = "https://api.eia.gov/v2/petroleum/sum/sndw/data/?api_key=" + key +
|
|
"&frequency=weekly&data%5B0%5D=value" +
|
|
"&facets%5Bseries%5D%5B0%5D=WCESTUS1" + // crude stocks ex-SPR
|
|
"&facets%5Bseries%5D%5B1%5D=WCRFPUS2" + // field production
|
|
"&facets%5Bseries%5D%5B2%5D=WPULEUS3" + // refinery utilization %
|
|
"&sort%5B0%5D%5Bcolumn%5D=period&sort%5B0%5D%5Bdirection%5D=asc" +
|
|
"&length=5000&start=" + startDate;
|
|
string body;
|
|
if(!m_http.HttpGet(url, body))
|
|
return false;
|
|
int pos = 0, added = 0;
|
|
while(true)
|
|
{
|
|
int e1, e2, e3;
|
|
string d = m_http.JsonField(body, "period", pos, e1);
|
|
if(d == "" || e1 < 0)
|
|
break;
|
|
string sid = m_http.JsonField(body, "series", e1, e2);
|
|
string v = m_http.JsonField(body, "value", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
datetime dt = StringToTime(d); // accepts yyyy-mm-dd
|
|
if(dt <= 0 || sid == "" || v == "")
|
|
continue;
|
|
if(s.rows > 0 && dt < s.date[s.rows - 1])
|
|
continue;
|
|
if(s.rows == 0 || dt > s.date[s.rows - 1])
|
|
{
|
|
int n = AltSeriesAppend(s, dt, dt + 6 * 86400);
|
|
added++;
|
|
}
|
|
int r = s.rows - 1;
|
|
if(sid == "WCESTUS1") s.v1[r] = StringToDouble(v);
|
|
else if(sid == "WCRFPUS2") s.v2[r] = StringToDouble(v);
|
|
else if(sid == "WPULEUS3") s.v3[r] = StringToDouble(v);
|
|
}
|
|
if(added > 0)
|
|
{
|
|
AltRawSave("raw_EIA_WPSR.csv", s, 3, "date;published;stocks;production;utilization");
|
|
PrintFormat("AltDataFetch: raw_EIA_WPSR.csv +%d weekly reports (through %s)", added,
|
|
TimeToString(s.date[s.rows - 1], TIME_DATE));
|
|
}
|
|
return added > 0;
|
|
}
|
|
|
|
//--- ------------------------------------ GEX forward recorder WHY THIS IS A RECORDER AND NOT A
|
|
//--- FEATURE: option open interest is a SNAPSHOT source. Revisit when the file holds a year or
|
|
//--- so, then screen it like every other candidate.
|
|
string CboeTicker(string canonical)
|
|
{
|
|
if(canonical == "SP500") return "_SPX"; // index options - where dealer positioning concentrates
|
|
if(canonical == "NAS100") return "_NDX";
|
|
if(canonical == "US30") return "_DJX";
|
|
if(canonical == "US2000") return "IWM";
|
|
if(canonical == "XAUUSD") return "GLD";
|
|
if(canonical == "XAGUSD") return "SLV";
|
|
if(canonical == "XTIUSD") return "USO";
|
|
if(canonical == "BTCUSD") return "IBIT";
|
|
return ""; // no listed-options proxy worth recording
|
|
}
|
|
|
|
//--- last date already in the file, or 0
|
|
datetime LastGexDate(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 1)
|
|
return 0;
|
|
return StringToTime(f[0]);
|
|
}
|
|
|
|
bool UpdateGex(string canonical)
|
|
{
|
|
string ticker = CboeTicker(canonical);
|
|
if(ticker == "")
|
|
return false;
|
|
//--- Once per UTC day, after the US cash close (21:00 UTC covers both DST regimes),
|
|
//--- and never on a weekend - the chain would just repeat Friday's settled numbers.
|
|
MqlDateTime g;
|
|
TimeToStruct(TimeGMT(), g);
|
|
if(g.day_of_week == 0 || g.day_of_week == 6)
|
|
return false;
|
|
if(g.hour < 21)
|
|
return false;
|
|
string file = ALTFETCH_DIR + "gex_" + canonical + ".csv";
|
|
datetime today = StringToTime(StringFormat("%04d.%02d.%02d", g.year, g.mon, g.day));
|
|
if(!m_gexDateLoaded)
|
|
{
|
|
m_lastGexDate = LastGexDate(file);
|
|
m_gexDateLoaded = true;
|
|
}
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- Re-read from DISK before spending the download: two charts of the same symbol
|
|
//--- (e.g. D1 + H4) each run their own fetcher, and a per-instance cached date would
|
|
//--- let the second chart append a duplicate row for the day the first just recorded.
|
|
m_lastGexDate = LastGexDate(file);
|
|
if(m_lastGexDate >= today)
|
|
return false;
|
|
//--- the SPX chain is ~13 MB: this needs a far longer timeout than the small feeds
|
|
string body;
|
|
if(!m_http.HttpGet("https://cdn.cboe.com/api/global/delayed_quotes/options/" + ticker + ".json",
|
|
body, ALTFETCH_GEX_TIMEOUT_MS))
|
|
return false;
|
|
if(StringLen(body) < 1000)
|
|
{
|
|
Print("AltDataFetch: GEX response for " + ticker + " was too short to be a chain - skipping.");
|
|
return false;
|
|
}
|
|
int e0;
|
|
string spotS = m_http.JsonField(body, "current_price", 0, e0); // absent from option rows, so
|
|
if(spotS == "") // a plain scan finds the header one
|
|
spotS = m_http.JsonField(body, "close", 0, e0);
|
|
double spot = StringToDouble(spotS);
|
|
if(spot <= 0.0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - no usable spot price in the response, skipping.");
|
|
return false;
|
|
}
|
|
//--- per-expiry accumulation (54 expiries on SPX; the cap is generous)
|
|
int expCode[128];
|
|
double expGex[128];
|
|
int nExp = 0;
|
|
double gexCall = 0.0, gexPut = 0.0;
|
|
double oiCall = 0.0, oiPut = 0.0;
|
|
int rows = 0;
|
|
int pos = 0;
|
|
//--- Row layout is option, ..., open_interest, volume, delta, gamma - so one forward
|
|
//--- sweep reads each row's three fields in the order they appear.
|
|
while(true)
|
|
{
|
|
int e1;
|
|
string sym = m_http.JsonField(body, "option", pos, e1);
|
|
if(sym == "" || e1 < 0)
|
|
break;
|
|
int e2, e3;
|
|
string oiS = m_http.JsonField(body, "open_interest", e1, e2);
|
|
string gS = m_http.JsonField(body, "gamma", (e2 > e1 ? e2 : e1), e3);
|
|
pos = (e3 > e1 ? e3 : (e2 > e1 ? e2 : e1));
|
|
double oi = StringToDouble(oiS);
|
|
double gam = StringToDouble(gS);
|
|
if(oi <= 0.0 || gam == 0.0)
|
|
continue; // no exposure - skip the string work entirely
|
|
//--- OCC symbol: <root><YYMMDD><C|P><strike*1000, 8 digits>, so the tail is fixed
|
|
//--- width regardless of how long the root is (SPX, SPXW, IWM, ...).
|
|
int len = StringLen(sym);
|
|
if(len < 15)
|
|
continue;
|
|
string kind = StringSubstr(sym, len - 9, 1);
|
|
int expiry = (int)StringToInteger(StringSubstr(sym, len - 15, 6));
|
|
//--- dollar gamma per 1% move: gamma x OI x 100 (contract multiplier) x S^2 x 0.01
|
|
double dollar = gam * oi * 100.0 * spot * spot * 0.01;
|
|
if(kind == "C")
|
|
{
|
|
gexCall += dollar;
|
|
oiCall += oi;
|
|
}
|
|
else
|
|
{
|
|
gexPut -= dollar; // dealers short puts: opposite sign convention
|
|
oiPut += oi;
|
|
dollar = -dollar;
|
|
}
|
|
int slot = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
if(expCode[i] == expiry)
|
|
{
|
|
slot = i;
|
|
break;
|
|
}
|
|
if(slot < 0 && nExp < 128)
|
|
{
|
|
slot = nExp++;
|
|
expCode[slot] = expiry;
|
|
expGex[slot] = 0.0;
|
|
}
|
|
if(slot >= 0)
|
|
expGex[slot] += dollar;
|
|
rows++;
|
|
}
|
|
if(rows == 0)
|
|
{
|
|
Print("AltDataFetch: GEX " + ticker + " - parsed 0 contracts with exposure, skipping "
|
|
"(the CDN response format may have changed).");
|
|
return false;
|
|
}
|
|
//--- Partial-chain guard: a truncated 13 MB download parses cleanly and yields a
|
|
//--- plausible-but-wrong GEX. A day-over-day contract-count collapse (>50%) is the
|
|
//--- fingerprint of truncation, not of markets - OI does not halve overnight.
|
|
int prevRows = LastGexRowCount(file);
|
|
if(prevRows > 0 && rows < prevRows / 2)
|
|
{
|
|
PrintFormat("AltDataFetch: GEX %s - only %d contracts vs %d yesterday (<50%%): response "
|
|
"looks TRUNCATED, not recording. Will retry on the next timer pass.",
|
|
ticker, rows, prevRows);
|
|
return false;
|
|
}
|
|
//--- three nearest expiries, by selection (nExp is tiny)
|
|
double front[3] = {0.0, 0.0, 0.0};
|
|
int frontCode = 0, prevCode = 0;
|
|
for(int k = 0; k < 3; k++)
|
|
{
|
|
int best = -1;
|
|
for(int i = 0; i < nExp; i++)
|
|
{
|
|
if(expCode[i] <= prevCode)
|
|
continue;
|
|
if(best < 0 || expCode[i] < expCode[best])
|
|
best = i;
|
|
}
|
|
if(best < 0)
|
|
break;
|
|
front[k] = expGex[best] / 1e9;
|
|
prevCode = expCode[best];
|
|
if(k == 0)
|
|
frontCode = prevCode;
|
|
}
|
|
bool isNew = !FileIsExist(file, FILE_COMMON);
|
|
int h = FileOpen(file, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
{
|
|
PrintFormat("AltDataFetch: cannot write %s (%d) - GEX not recorded today.", file, GetLastError());
|
|
return false;
|
|
}
|
|
if(isNew)
|
|
FileWriteString(h, "date;recorded_utc;spot;gex_total;gex_call;gex_put;oi_call;oi_put;"
|
|
"gex_exp1;gex_exp2;gex_exp3;expiry1;n_expiry;n_rows\n");
|
|
FileSeek(h, 0, SEEK_END);
|
|
FileWriteString(h, StringFormat("%s;%s;%.4f;%.6f;%.6f;%.6f;%.0f;%.0f;%.6f;%.6f;%.6f;%d;%d;%d\n",
|
|
TimeToString(today, TIME_DATE),
|
|
TimeToString(TimeGMT(), TIME_DATE | TIME_MINUTES),
|
|
spot, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9,
|
|
oiCall, oiPut, front[0], front[1], front[2],
|
|
frontCode, nExp, rows));
|
|
FileClose(h);
|
|
m_lastGexDate = today;
|
|
//--- Progress matters here: the file is useless until it is long enough to screen, and
|
|
//--- the only way to know how far along it is, is to say so.
|
|
int have = 1;
|
|
datetime probe = LastGexDate(file);
|
|
if(probe > 0)
|
|
have = CountGexRows(file);
|
|
PrintFormat("AltDataFetch: GEX %s recorded - total %+.2f Bn/1%% (calls %+.2f, puts %+.2f), "
|
|
"%d contracts, %d expiries, spot %.2f. %d day(s) banked; this is a FORWARD "
|
|
"RECORDER - no free history exists, so it feeds no model until there is enough "
|
|
"to screen (~250 trading days).",
|
|
ticker, (gexCall + gexPut) / 1e9, gexCall / 1e9, gexPut / 1e9, rows, nExp, spot, have);
|
|
return true;
|
|
}
|
|
|
|
//--- n_rows (field 14) of the last data line - the previous session's contract count
|
|
int LastGexRowCount(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
string last = "";
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
last = line;
|
|
}
|
|
FileClose(h);
|
|
if(last == "")
|
|
return 0;
|
|
string f[];
|
|
if(StringSplit(last, ';', f) < 14)
|
|
return 0;
|
|
return (int)StringToInteger(f[13]);
|
|
}
|
|
|
|
int CountGexRows(string file)
|
|
{
|
|
int h = FileOpen(file, FILE_READ | FILE_TXT | FILE_ANSI | FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_COMMON);
|
|
if(h == INVALID_HANDLE)
|
|
return 0;
|
|
int n = 0;
|
|
while(!FileIsEnding(h))
|
|
{
|
|
string line = FileReadString(h);
|
|
if(StringLen(line) > 8 && StringFind(line, "date;") != 0)
|
|
n++;
|
|
}
|
|
FileClose(h);
|
|
return n;
|
|
}
|
|
|
|
//--- -------------------------------- feature math (mirrors export.py)
|
|
//--- pandas rolling(W, min_periods=M).rank(pct=True) on the LAST element: rank of the current
|
|
//--- value among the window's values (average rank for ties) divided by the window count.
|
|
double RollingPctRank(const double &v[], int i, int window, int minPeriods)
|
|
{
|
|
int lo = MathMax(0, i - window + 1);
|
|
int n = i - lo + 1;
|
|
if(n < minPeriods)
|
|
return EMPTY_VALUE;
|
|
double cur = v[i];
|
|
int below = 0, ties = 0;
|
|
for(int k = lo; k <= i; k++)
|
|
{
|
|
if(v[k] < cur)
|
|
below++;
|
|
else if(v[k] == cur)
|
|
ties++;
|
|
}
|
|
double rank = below + (1.0 + ties) / 2.0;
|
|
return rank / n;
|
|
}
|
|
|
|
//--- as-of index: largest row with published <= day, else -1
|
|
int AsOf(const SAltRawSeries &s, datetime day)
|
|
{
|
|
int lo = 0, hi = s.rows - 1, ans = -1;
|
|
while(lo <= hi)
|
|
{
|
|
int mid = (lo + hi) / 2;
|
|
if(s.published[mid] <= day)
|
|
{
|
|
ans = mid;
|
|
lo = mid + 1;
|
|
}
|
|
else
|
|
hi = mid - 1;
|
|
}
|
|
return ans;
|
|
}
|
|
|
|
//--- raw series kept resident between timer ticks (one chart = one symbol, so only the
|
|
//--- sources the chart's catalog row names are ever populated)
|
|
//--- m_ivol holds whichever instrument-specific implied-vol index THIS chart's catalog
|
|
//--- row names (GVZ for gold, OVX for oil, VXN/VXD/RVX for the index symbols) - one
|
|
//--- chart is one symbol, so a single slot serves regardless of which series it is.
|
|
SAltRawSeries m_vix, m_usd, m_cot, m_eia, m_ivol;
|
|
//--- US macro block. UNRATE is the exception: it IS revised (seasonal refits, ~0.1-0.2pp), there
|
|
//--- is no unrevised variant, and the EA cannot run the ALFRED first-print protocol - so a fresh
|
|
//--- backfill carries small revisions into history.
|
|
SAltRawSeries m_dgs10, m_curve, m_bei, m_dff, m_ecb, m_cpi, m_unrate;
|
|
|
|
//--- generic HTTP client and symbol catalog/user-mapping are separate collaborators - see
|
|
//--- AltDataHttpClient.mqh/AltDataCatalog.mqh. Plain composition (no view/adapter): this class
|
|
//--- has no single-inheritance parent forcing one, unlike the Expert\AIBase\* partials.
|
|
CAltDataHttpClient m_http;
|
|
CAltDataCatalog m_catalog;
|
|
|
|
public:
|
|
CAltDataFetch(void) : m_fredKey(""), m_keyLoaded(false),
|
|
m_eiaKey(""), m_eiaWarned(false),
|
|
m_lastGexDate(0), m_gexDateLoaded(false)
|
|
{
|
|
for(int i = 0; i < ArraySize(m_lastAttempt); i++)
|
|
m_lastAttempt[i] = 0;
|
|
}
|
|
|
|
//--- ------------------------------------- symbol mapping interface (forwards to the catalog)
|
|
bool NeedsMapping(string symbol) { return m_catalog.NeedsMapping(symbol); }
|
|
int CatalogCount(void) { return m_catalog.CatalogCount(); }
|
|
string CatalogName(int i) { return m_catalog.CatalogName(i); }
|
|
string CatalogLabel(int i) { return m_catalog.CatalogLabel(i); }
|
|
bool SaveUserMapping(string symbol, string canonical)
|
|
{
|
|
return m_catalog.SaveUserMapping(symbol, canonical);
|
|
}
|
|
|
|
//--- Maintain the raw caches and the feature CSV for `symbol`. Returns true when the feature
|
|
//--- file was rebuilt (caller should reload the panels).
|
|
bool Update(string symbol)
|
|
{
|
|
int si = m_catalog.FindSpec(symbol);
|
|
if(si < 0)
|
|
return false; // unknown symbol or user declined - nothing to serve
|
|
SAltSymbolSpec spec = m_catalog.Spec(si);
|
|
bool needVix = StringFind(spec.features, "vix") >= 0;
|
|
bool needUsd = StringFind(spec.features, "usd_") >= 0;
|
|
bool needEia = StringFind(spec.features, "eia_") >= 0;
|
|
bool needCot = (spec.cotWhere != "");
|
|
bool needIvol = (spec.ivolSeries != "" && StringFind(spec.features, "ivol") >= 0);
|
|
bool needMac = StringFind(spec.features, "mac_") >= 0;
|
|
bool changed = false;
|
|
//--- plausibility bands: generous enough for any real print in the series' history
|
|
//--- (VIX peaked 89 in 2008; utilization-style checks live in FeatureValue), tight
|
|
//--- enough that transport garbage cannot pass
|
|
if(needVix)
|
|
{
|
|
if(m_vix.rows == 0)
|
|
AltRawLoad("raw_VIXCLS.csv", m_vix, 1);
|
|
if(UpdateFred("VIXCLS", m_vix, 0, 4, 1.0, 200.0))
|
|
changed = true;
|
|
}
|
|
if(needUsd)
|
|
{
|
|
//--- staleDays 10: the H.10 dollar index posts with ~a week's lag (live fetch
|
|
//--- 2026-08-16 was current only through 08-07), so a 4-day horizon would fire a
|
|
//--- futile refetch attempt every hour in the gap between releases
|
|
if(m_usd.rows == 0)
|
|
AltRawLoad("raw_DTWEXBGS.csv", m_usd, 1);
|
|
if(UpdateFred("DTWEXBGS", m_usd, 1, 10, 50.0, 250.0))
|
|
changed = true;
|
|
}
|
|
if(needCot)
|
|
{
|
|
//--- cache is named by CANONICAL, so two brokers' names for the same instrument
|
|
//--- share one download instead of fetching the same contract twice
|
|
string cache = "raw_COT_" + spec.canonical + ".csv";
|
|
if(m_cot.rows == 0)
|
|
AltRawLoad(cache, m_cot, 3);
|
|
if(UpdateCot(cache, spec.cotWhere, m_cot, 2))
|
|
changed = true;
|
|
}
|
|
if(needEia)
|
|
{
|
|
if(m_eia.rows == 0)
|
|
AltRawLoad("raw_EIA_WPSR.csv", m_eia, 3);
|
|
if(UpdateEia(m_eia, 3))
|
|
changed = true;
|
|
}
|
|
if(needIvol)
|
|
{
|
|
if(m_ivol.rows == 0)
|
|
AltRawLoad("raw_" + spec.ivolSeries + ".csv", m_ivol, 1);
|
|
if(UpdateFred(spec.ivolSeries, m_ivol, 4, 4, 1.0, 300.0))
|
|
changed = true;
|
|
}
|
|
if(needMac)
|
|
{
|
|
if(m_dgs10.rows == 0) AltRawLoad("raw_DGS10.csv", m_dgs10, 1);
|
|
if(m_curve.rows == 0) AltRawLoad("raw_T10Y2Y.csv", m_curve, 1);
|
|
if(m_bei.rows == 0) AltRawLoad("raw_T5YIE.csv", m_bei, 1);
|
|
if(m_dff.rows == 0) AltRawLoad("raw_DFF.csv", m_dff, 1);
|
|
if(m_ecb.rows == 0) AltRawLoad("raw_ECBDFR.csv", m_ecb, 1);
|
|
if(m_cpi.rows == 0) AltRawLoad("raw_CPIAUCNS.csv", m_cpi, 1);
|
|
if(m_unrate.rows == 0) AltRawLoad("raw_UNRATE.csv", m_unrate, 1);
|
|
if(UpdateFred("DGS10", m_dgs10, 5, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("T10Y2Y", m_curve, 6, 4, -10.0, 10.0)) changed = true;
|
|
if(UpdateFred("T5YIE", m_bei, 7, 4, -5.0, 15.0)) changed = true;
|
|
if(UpdateFred("DFF", m_dff, 8, 4, -2.0, 30.0)) changed = true;
|
|
if(UpdateFred("ECBDFR", m_ecb, 9, 4, -5.0, 30.0)) changed = true;
|
|
if(UpdateFred("CPIAUCNS", m_cpi, 10, 32, 20.0, 1000.0)) changed = true;
|
|
if(UpdateFred("UNRATE", m_unrate, 11, 32, 0.0, 35.0)) changed = true;
|
|
}
|
|
//--- Rebuild when stale/missing and ANY needed source has rows: a temporarily dead source
|
|
//--- leaves its columns empty (header intact) and the panel 0-fills them - degradation must
|
|
//--- never take the working features down with it (the silent-FRED incident took exactly that
|
|
//--- shape under the old all-sources-required gate).
|
|
UpdateGex(spec.canonical);
|
|
bool anyReady = (needVix && m_vix.rows > 0) || (needUsd && m_usd.rows > 0)
|
|
|| (needCot && m_cot.rows > 0) || (needEia && m_eia.rows > 0)
|
|
|| (needIvol && m_ivol.rows > 0) || (needMac && m_dgs10.rows > 0);
|
|
string outFile = AltDataFileSymbol(symbol) + "_D1.csv"; // sanitizer shared with the panel
|
|
if((changed || !FileIsExist(ALTFETCH_DIR + outFile, FILE_COMMON)) && anyReady)
|
|
return RebuildFeatures(spec, outFile);
|
|
return false;
|
|
}
|
|
|
|
protected:
|
|
//--- --------------------------------- catalog-driven feature writer Walks the daily grid and
|
|
//--- emits exactly export.py's format; the transforms are the FIXED a-priori constants
|
|
//--- documented there.
|
|
bool RebuildFeatures(const SAltSymbolSpec &spec, string outFile)
|
|
{
|
|
string feats[];
|
|
int nf = StringSplit(spec.features, ';', feats);
|
|
if(nf <= 0)
|
|
return false;
|
|
//--- weekly COT net-spec series with the catalog sign applied
|
|
double cspec[];
|
|
ArrayResize(cspec, m_cot.rows);
|
|
for(int i = 0; i < m_cot.rows; i++)
|
|
cspec[i] = (m_cot.v1[i] > 0) ? spec.cotSign * (m_cot.v2[i] - m_cot.v3[i]) / m_cot.v1[i] : 0.0;
|
|
//--- EIA crude-stocks copy for the rolling percentile
|
|
double estk[];
|
|
ArrayResize(estk, m_eia.rows);
|
|
for(int i = 0; i < m_eia.rows; i++)
|
|
estk[i] = m_eia.v1[i];
|
|
//--- temp + swap via AtomicFile.mqh, same reasoning as AltRawSave: panels on other charts read this file
|
|
string tmpName = "";
|
|
int h = AtomicWriteBegin(ALTFETCH_DIR + outFile, FILE_COMMON, tmpName, FILE_TXT | FILE_ANSI);
|
|
if(h == INVALID_HANDLE)
|
|
return false;
|
|
string header = "date";
|
|
for(int f = 0; f < nf; f++)
|
|
header += ";" + feats[f];
|
|
FileWriteString(h, header + "\n");
|
|
datetime today = TimeCurrent();
|
|
for(datetime day = ALTFETCH_GRID_START; day <= today; day += 86400)
|
|
{
|
|
string line = TimeToString(day, TIME_DATE);
|
|
bool any = false;
|
|
for(int f = 0; f < nf; f++)
|
|
{
|
|
string val = FeatureValue(feats[f], day, cspec, estk);
|
|
if(val != "")
|
|
any = true;
|
|
line += ";" + val;
|
|
}
|
|
if(any)
|
|
FileWriteString(h, line + "\n");
|
|
}
|
|
if(!AtomicWriteEnd(h, ALTFETCH_DIR + outFile, tmpName, FILE_COMMON, true, __FUNCTION__))
|
|
return false;
|
|
PrintFormat("AltDataFetch: %s rebuilt (EA-maintained) - %d features: %s.",
|
|
outFile, nf, spec.features);
|
|
return true;
|
|
}
|
|
|
|
//--- One feature value at one day; "" while its source history is not yet deep enough.
|
|
//--- Each branch does its own as-of lookup (binary search, cheap at rebuild frequency),
|
|
//--- so adding a source never widens a parameter list again.
|
|
string FeatureValue(string id, datetime day,
|
|
const double &cspec[], const double &estk[])
|
|
{
|
|
if(id == "vix_chg5")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 5) ? DoubleToString((m_vix.v1[i] - m_vix.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "vix")
|
|
{
|
|
int i = AsOf(m_vix, day);
|
|
return (i >= 0) ? DoubleToString(m_vix.v1[i] / 100.0, 6) : "";
|
|
}
|
|
//--- the chart's OWN implied-vol index; same a-priori scales as vix/vix_chg5 so the
|
|
//--- pairs are directly comparable to the first batch-norm layer
|
|
if(id == "ivol_chg5")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 5) ? DoubleToString((m_ivol.v1[i] - m_ivol.v1[i - 5]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "ivol")
|
|
{
|
|
int i = AsOf(m_ivol, day);
|
|
return (i >= 0) ? DoubleToString(m_ivol.v1[i] / 100.0, 6) : "";
|
|
}
|
|
if(id == "usd_chg5")
|
|
{
|
|
int i = AsOf(m_usd, day);
|
|
return (i >= 5) ? DoubleToString(m_usd.v1[i] - m_usd.v1[i - 5], 6) : "";
|
|
}
|
|
if(id == "cot_spec_net")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 0 && m_cot.v1[i] > 0) ? DoubleToString(cspec[i], 6) : "";
|
|
}
|
|
if(id == "cot_idx_1y" || id == "cot_idx_3y")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
if(i < 0)
|
|
return "";
|
|
bool oneYear = (id == "cot_idx_1y");
|
|
double r = RollingPctRank(cspec, i, oneYear ? 52 : 156, oneYear ? 26 : 78);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "cot_chg_4w")
|
|
{
|
|
int i = AsOf(m_cot, day);
|
|
return (i >= 4) ? DoubleToString(cspec[i] - cspec[i - 4], 6) : "";
|
|
}
|
|
if(id == "eia_stk_idx1y")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
if(i < 0 || estk[i] <= 0)
|
|
return "";
|
|
double r = RollingPctRank(estk, i, 52, 26);
|
|
return (r != EMPTY_VALUE) ? DoubleToString(r - 0.5, 6) : "";
|
|
}
|
|
if(id == "eia_stk_chg4")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 4 && estk[i] > 0 && estk[i - 4] > 0)
|
|
? DoubleToString((estk[i] / estk[i - 4] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "eia_util")
|
|
{
|
|
int i = AsOf(m_eia, day);
|
|
return (i >= 0 && m_eia.v3[i] > 0)
|
|
? DoubleToString((m_eia.v3[i] - 90.0) / 10.0, 6) : "";
|
|
}
|
|
//--- US macro block. Index arithmetic is in OBSERVATIONS (business days for the
|
|
//--- daily series, months for CPI/UNRATE), matching the research screen exactly.
|
|
if(id == "mac_y10")
|
|
{
|
|
int i = AsOf(m_dgs10, day);
|
|
return (i >= 20) ? DoubleToString(m_dgs10.v1[i] - m_dgs10.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_curve")
|
|
{
|
|
int i = AsOf(m_curve, day);
|
|
return (i >= 0) ? DoubleToString(m_curve.v1[i], 6) : "";
|
|
}
|
|
if(id == "mac_bei")
|
|
{
|
|
int i = AsOf(m_bei, day);
|
|
return (i >= 20) ? DoubleToString(m_bei.v1[i] - m_bei.v1[i - 20], 6) : "";
|
|
}
|
|
if(id == "mac_gap")
|
|
{
|
|
int i = AsOf(m_dff, day), j = AsOf(m_ecb, day);
|
|
return (i >= 0 && j >= 0)
|
|
? DoubleToString((m_dff.v1[i] - m_ecb.v1[j]) / 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_cpi")
|
|
{
|
|
int i = AsOf(m_cpi, day);
|
|
return (i >= 12 && m_cpi.v1[i - 12] > 0)
|
|
? DoubleToString((m_cpi.v1[i] / m_cpi.v1[i - 12] - 1.0) * 10.0, 6) : "";
|
|
}
|
|
if(id == "mac_unemp")
|
|
{
|
|
int i = AsOf(m_unrate, day);
|
|
return (i >= 12) ? DoubleToString((m_unrate.v1[i] - m_unrate.v1[i - 12]) / 10.0, 6) : "";
|
|
}
|
|
return "";
|
|
}
|
|
};
|