//+------------------------------------------------------------------+ //| CalendarRecorder.mq5 | //| AnimateDread | //| https://www.mql5.com | //+------------------------------------------------------------------+ #property copyright "AnimateDread" #property link "https://www.mql5.com" #property version "1.00" #property description "Records economic calendar releases AT RELEASE TIME, write-once." #property description "Standalone: attach to any spare chart. Does not touch the trading EA." //--- WHY THIS EXISTS, AND WHY IT CANNOT BE A BACKTEST //--- //--- The calendar is the only non-price data source this project found that MQL5 actually //--- carries with content (swap has no history, TICK_FLAG_BUY/SELL are empty on FX, DOM is //--- absent on retail FX and never in the tester). But it CANNOT be researched from //--- history, because MqlCalendarValue.actual_value returns the CURRENT figure - the one //--- after every subsequent revision. Reading 2019's NFP today gives a number that nobody //--- could have known in 2019. Any "surprise = actual - forecast" feature built from //--- history is therefore contaminated by lookahead, and it is the flattering kind: it //--- makes the model look prescient exactly on the events that were revised most. //--- //--- The only sound way to get an honest surprise series is to write down what the //--- terminal reported at the moment of release and never touch that row again. That is //--- all this does. It buys a dataset that starts empty and grows in real time - months //--- before it is usable, which is precisely why it should start running now rather than //--- when someone wants it. //--- //--- WRITE-ONCE IS THE WHOLE CONTRACT. A row is appended the first time a value_id is seen //--- with an actual figure attached, and is never rewritten. Re-recording on a later poll //--- would silently import the revision this file exists to avoid. //--- Deliberately NO includes from the EA tree. This is a data-collection side-task that //--- may run for months on a spare chart; coupling it to the trading system's headers //--- would mean a refactor there can stop the recorder, and a silent gap in a write-once //--- series cannot be backfilled by definition. The one helper it needs is eight lines. input int InpPollSeconds = 30; // Poll interval (seconds) input int InpLookbackHours = 48; // How far back each poll looks for newly-filled values input int InpLookaheadHours = 4; // Upcoming window (captures scheduled-but-unreleased) input int InpLiveLagSeconds = 600; // Max release->observation lag still counted LIVE input bool InpAllCountries = true; // Record every country (false = this symbol's only) input int InpMinImportance = 1; // 0=none 1=low 2=moderate 3=high input string InpFileName = "Warrior_EA/Research/calendar_live.csv"; //--- LONG_MIN is the terminal's "this field has no value" sentinel on MqlCalendarValue. #define CAL_NO_VALUE LONG_MIN #define CAL_SCALE 1000000.0 //--- CAPTURE CLASS - the single most important column in this file. //--- //--- Every poll looks BACK as well as forward, so it necessarily re-reads events that were //--- released before the recorder was watching: the first poll after attaching picked up 83 //--- of them. Those rows carry whatever the terminal holds NOW, which is the revised figure //--- - the exact contamination this recorder exists to avoid. The very first batch proved //--- it was not hypothetical: a "Retail Sales y/y" row came back with previous 3.5 and //--- revised_prev 3.4, and a "Core CPI m/m" row already had revision=1. //--- //--- Backfill is still worth keeping (it is a fine snapshot of the revised series, and it //--- carries the event metadata), but it must never be silently mixed with release-time //--- observations. So the lag is written explicitly and classified, rather than left to be //--- re-derived by whoever loads the CSV later and may not think to. //--- //--- LIVE observed within InpLiveLagSeconds of the release -> usable as a surprise //--- BACKFILL seen long after the fact -> MUST NOT be used for surprise research #define CSV_HEADER "observed_time,capture,lag_sec,value_id,event_id,event_time,country," \ "currency,importance,name,actual,forecast,previous,revised_prev," \ "revision,impact,unit,digits,multiplier,symbol,bid,ask,spread_pts" ulong g_seen[]; // value_ids already written, kept SORTED for binary search string g_countries[]; datetime g_lastPoll = 0; int g_live = 0; // release-time captures this session (the only usable rows) //+------------------------------------------------------------------+ //| Sorted insert into the seen-set | //+------------------------------------------------------------------+ bool SeenContains(const ulong id) { int n = ArraySize(g_seen); if(n == 0) return false; int lo = 0, hi = n - 1; while(lo <= hi) { int mid = (lo + hi) >> 1; if(g_seen[mid] == id) return true; if(g_seen[mid] < id) lo = mid + 1; else hi = mid - 1; } return false; } void SeenInsert(const ulong id) { int n = ArraySize(g_seen); int pos = n; for(int i = 0; i < n; i++) if(g_seen[i] > id) { pos = i; break; } ArrayResize(g_seen, n + 1); for(int i = n; i > pos; i--) g_seen[i] = g_seen[i - 1]; g_seen[pos] = id; } //+------------------------------------------------------------------+ //| Rebuild the seen-set from the file so a restart never duplicates | //| or, worse, re-records a value that has since been revised. | //+------------------------------------------------------------------+ void LoadSeen() { ArrayResize(g_seen, 0); //--- Read whole LINES and split, rather than walking FILE_CSV field by field with a //--- counter. The counter version had an off-by-one and was keying the seen-set on //--- event_id instead of value_id - and because event_id identifies the event TYPE, not //--- the release, that would have made every restart permanently skip all future //--- releases of every event already in the file. Indexing a split array by name-checked //--- position cannot drift like that when the schema changes. //--- FILE_SHARE_READ|FILE_SHARE_WRITE is mandatory in this codebase: an exclusive open //--- fails 5004 whenever anything else holds the file, and looks like "no history yet". int h = FileOpen(InpFileName, FILE_READ | FILE_TXT | FILE_ANSI | FILE_COMMON | FILE_SHARE_READ | FILE_SHARE_WRITE); if(h == INVALID_HANDLE) { PrintFormat("%s: no existing file (%s) - starting fresh", __FUNCTION__, InpFileName); return; } string header = FileIsEnding(h) ? "" : FileReadString(h); StringTrimRight(header); StringTrimLeft(header); if(StringLen(header) > 0 && header != CSV_HEADER) { //--- Old schema. Rotate rather than append: mixing layouts silently mis-parses every //--- old row, and the existing rows are backfill that will be re-derived anyway. FileClose(h); string bak = InpFileName + "." + IntegerToString((int)TimeCurrent()) + ".old"; if(FileMove(InpFileName, FILE_COMMON, bak, FILE_COMMON)) PrintFormat("%s: schema changed - previous file rotated to %s", __FUNCTION__, bak); else PrintFormat("%s: schema changed but rotate FAILED (error %d) - " "move %s aside by hand", __FUNCTION__, GetLastError(), InpFileName); return; } int vcol = -1; string hf[]; if(StringSplit(header, ',', hf) > 0) for(int i = 0; i < ArraySize(hf); i++) if(hf[i] == "value_id") vcol = i; if(vcol < 0) { FileClose(h); Print(__FUNCTION__ + ": header has no value_id column - refusing to guess"); return; } while(!FileIsEnding(h)) { string line = FileReadString(h); if(StringLen(line) < 5) continue; string f[]; if(StringSplit(line, ',', f) > vcol) SeenInsert((ulong)StringToInteger(f[vcol])); } FileClose(h); PrintFormat("%s: loaded %d previously recorded values from %s", __FUNCTION__, ArraySize(g_seen), InpFileName); } //+------------------------------------------------------------------+ //| Append one row. Header written only when the file is created. | //+------------------------------------------------------------------+ bool AppendRow(const string row) { int h = FileOpen(InpFileName, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_COMMON | FILE_SHARE_READ | FILE_SHARE_WRITE); if(h == INVALID_HANDLE) { PrintFormat("%s: cannot open %s, error %d", __FUNCTION__, InpFileName, GetLastError()); return false; } if(FileSize(h) == 0) FileWriteString(h, CSV_HEADER + "\r\n"); FileSeek(h, 0, SEEK_END); FileWriteString(h, row + "\r\n"); FileFlush(h); FileClose(h); return true; } //+------------------------------------------------------------------+ double CalValue(const long v) { return (v == CAL_NO_VALUE) ? EMPTY_VALUE : (double)v / CAL_SCALE; } string CalField(const long v) { return (v == CAL_NO_VALUE) ? "" : DoubleToString((double)v / CAL_SCALE, 6); } //+------------------------------------------------------------------+ int OnInit() { if(InpAllCountries) { MqlCalendarCountry countries[]; int total = CalendarCountries(countries); ArrayResize(g_countries, 0); for(int i = 0; i < total; i++) { int n = ArraySize(g_countries); ArrayResize(g_countries, n + 1); g_countries[n] = countries[i].code; } } else { //--- this symbol's base+quote currencies only string baseCcy = SymbolInfoString(_Symbol, SYMBOL_CURRENCY_BASE); string profCcy = SymbolInfoString(_Symbol, SYMBOL_CURRENCY_PROFIT); MqlCalendarCountry countries[]; int total = CalendarCountries(countries); ArrayResize(g_countries, 0); for(int i = 0; i < total; i++) if(countries[i].currency == baseCcy || countries[i].currency == profCcy) { int n = ArraySize(g_countries); ArrayResize(g_countries, n + 1); g_countries[n] = countries[i].code; } } if(ArraySize(g_countries) == 0) { Print(__FUNCTION__ + ": no countries resolved - calendar unavailable on this terminal?"); return INIT_FAILED; } LoadSeen(); EventSetTimer(MathMax(5, InpPollSeconds)); PrintFormat("CalendarRecorder: watching %d countries, poll %ds, file %s", ArraySize(g_countries), InpPollSeconds, InpFileName); Poll(); return INIT_SUCCEEDED; } void OnDeinit(const int reason) { EventKillTimer(); } void OnTimer() { Poll(); } //+------------------------------------------------------------------+ //| One sweep. Cheap enough to run every 30s: the terminal serves the | //| calendar from a local cache, and the window is a couple of days. | //+------------------------------------------------------------------+ void Poll() { datetime now = TimeCurrent(); datetime from = now - (datetime)InpLookbackHours * 3600; datetime to = now + (datetime)InpLookaheadHours * 3600; double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID); double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK); long spr = SymbolInfoInteger(_Symbol, SYMBOL_SPREAD); int added = 0; int liveBefore = g_live; for(int c = 0; c < ArraySize(g_countries); c++) { MqlCalendarValue values[]; if(!CalendarValueHistory(values, from, to, g_countries[c])) continue; // country simply has nothing in the window for(int i = 0; i < ArraySize(values); i++) { //--- Only record once the ACTUAL figure exists. A scheduled-but-unreleased row has //--- no surprise to measure, and recording it early would burn the value_id in the //--- seen-set so the real release is never captured. if(values[i].actual_value == CAL_NO_VALUE) continue; if(SeenContains(values[i].id)) continue; MqlCalendarEvent ev; if(!CalendarEventById(values[i].event_id, ev)) continue; if((int)ev.importance < InpMinImportance) { SeenInsert(values[i].id); // remember, so it is not re-examined every poll continue; } MqlCalendarCountry ctry; string currency = ""; if(CalendarCountryById(ev.country_id, ctry)) currency = ctry.currency; string name = ev.name; StringReplace(name, ",", ";"); // keep the CSV parseable long lag = (long)now - (long)values[i].time; string cap = (lag >= 0 && lag <= InpLiveLagSeconds) ? "LIVE" : "BACKFILL"; if(cap == "LIVE") g_live++; string row = StringFormat( "%s,%s,%I64d,%I64u,%I64u,%s,%s,%s,%d,%s,%s,%s,%s,%s,%d,%d,%d,%d,%.6f,%s,%.5f,%.5f,%d", TimeToString(now, TIME_DATE | TIME_SECONDS), cap, lag, values[i].id, values[i].event_id, TimeToString(values[i].time, TIME_DATE | TIME_SECONDS), g_countries[c], currency, (int)ev.importance, name, CalField(values[i].actual_value), CalField(values[i].forecast_value), CalField(values[i].prev_value), CalField(values[i].revised_prev_value), values[i].revision, (int)values[i].impact_type, (int)ev.unit, (int)ev.digits, (double)ev.multiplier, _Symbol, bid, ask, (int)spr); if(AppendRow(row)) { SeenInsert(values[i].id); added++; } } } if(added > 0) { int live = g_live - liveBefore; PrintFormat("CalendarRecorder: +%d row(s): %d LIVE, %d BACKFILL (usable this session: " "%d, file total: %d)", added, live, added - live, g_live, ArraySize(g_seen)); if(live == 0) Print("CalendarRecorder: all BACKFILL - post-revision figures, " "NOT usable for surprise research. Only LIVE rows are."); } g_lastPoll = now; } //+------------------------------------------------------------------+