Warrior_EA/Scripts/CalendarRecorder.mq5

//+------------------------------------------------------------------+
//|                                            CalendarRecorder.mq5   |
//|                                                     AnimateDread |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "AnimateDread"
#property link      "https://www.mql5.com"
#property version   "1.00"
#property description "Records economic calendar releases AT RELEASE TIME, write-once."
#property description "Standalone: attach to any spare chart. Does not touch the trading EA."

//--- WHY THIS EXISTS, AND WHY IT CANNOT BE A BACKTEST
//---
//--- The calendar is the only non-price data source this project found that MQL5 actually
//--- carries with content (swap has no history, TICK_FLAG_BUY/SELL are empty on FX, DOM is
//--- absent on retail FX and never in the tester). But it CANNOT be researched from
//--- history, because MqlCalendarValue.actual_value returns the CURRENT figure - the one
//--- after every subsequent revision. Reading 2019's NFP today gives a number that nobody
//--- could have known in 2019. Any "surprise = actual - forecast" feature built from
//--- history is therefore contaminated by lookahead, and it is the flattering kind: it
//--- makes the model look prescient exactly on the events that were revised most.
//---
//--- The only sound way to get an honest surprise series is to write down what the
//--- terminal reported at the moment of release and never touch that row again. That is
//--- all this does. It buys a dataset that starts empty and grows in real time - months
//--- before it is usable, which is precisely why it should start running now rather than
//--- when someone wants it.
//---
//--- WRITE-ONCE IS THE WHOLE CONTRACT. A row is appended the first time a value_id is seen
//--- with an actual figure attached, and is never rewritten. Re-recording on a later poll
//--- would silently import the revision this file exists to avoid.

//--- Deliberately NO includes from the EA tree. This is a data-collection side-task that
//--- may run for months on a spare chart; coupling it to the trading system's headers
//--- would mean a refactor there can stop the recorder, and a silent gap in a write-once
//--- series cannot be backfilled by definition. The one helper it needs is eight lines.

input int    InpPollSeconds     = 30;     // Poll interval (seconds)
input int    InpLookbackHours   = 48;     // How far back each poll looks for newly-filled values
input int    InpLookaheadHours  = 4;      // Upcoming window (captures scheduled-but-unreleased)
input int    InpLiveLagSeconds  = 600;    // Max release->observation lag still counted LIVE
input bool   InpAllCountries    = true;   // Record every country (false = this symbol's only)
input int    InpMinImportance   = 1;      // 0=none 1=low 2=moderate 3=high
input string InpFileName        = "Warrior_EA/Research/calendar_live.csv";

//--- LONG_MIN is the terminal's "this field has no value" sentinel on MqlCalendarValue.
#define CAL_NO_VALUE  LONG_MIN
#define CAL_SCALE     1000000.0

//--- CAPTURE CLASS - the single most important column in this file.
//---
//--- Every poll looks BACK as well as forward, so it necessarily re-reads events that were
//--- released before the recorder was watching: the first poll after attaching picked up 83
//--- of them. Those rows carry whatever the terminal holds NOW, which is the revised figure
//--- - the exact contamination this recorder exists to avoid. The very first batch proved
//--- it was not hypothetical: a "Retail Sales y/y" row came back with previous 3.5 and
//--- revised_prev 3.4, and a "Core CPI m/m" row already had revision=1.
//---
//--- Backfill is still worth keeping (it is a fine snapshot of the revised series, and it
//--- carries the event metadata), but it must never be silently mixed with release-time
//--- observations. So the lag is written explicitly and classified, rather than left to be
//--- re-derived by whoever loads the CSV later and may not think to.
//---
//---   LIVE     observed within InpLiveLagSeconds of the release -> usable as a surprise
//---   BACKFILL seen long after the fact          -> MUST NOT be used for surprise research
#define CSV_HEADER "observed_time,capture,lag_sec,value_id,event_id,event_time,country," \
                   "currency,importance,name,actual,forecast,previous,revised_prev," \
                   "revision,impact,unit,digits,multiplier,symbol,bid,ask,spread_pts"

ulong  g_seen[];            // value_ids already written, kept SORTED for binary search
string g_countries[];
datetime g_lastPoll = 0;
int    g_live = 0;          // release-time captures this session (the only usable rows)

//+------------------------------------------------------------------+
//| Sorted insert into the seen-set                                   |
//+------------------------------------------------------------------+
bool SeenContains(const ulong id)
  {
   int n = ArraySize(g_seen);
   if(n == 0)
      return false;
   int lo = 0, hi = n - 1;
   while(lo <= hi)
     {
      int mid = (lo + hi) >> 1;
      if(g_seen[mid] == id)
         return true;
      if(g_seen[mid] < id)
         lo = mid + 1;
      else
         hi = mid - 1;
     }
   return false;
  }

void SeenInsert(const ulong id)
  {
   int n = ArraySize(g_seen);
   int pos = n;
   for(int i = 0; i < n; i++)
      if(g_seen[i] > id)
        {
         pos = i;
         break;
        }
   ArrayResize(g_seen, n + 1);
   for(int i = n; i > pos; i--)
      g_seen[i] = g_seen[i - 1];
   g_seen[pos] = id;
  }

//+------------------------------------------------------------------+
//| Rebuild the seen-set from the file so a restart never duplicates  |
//| or, worse, re-records a value that has since been revised.        |
//+------------------------------------------------------------------+
void LoadSeen()
  {
   ArrayResize(g_seen, 0);
   //--- Read whole LINES and split, rather than walking FILE_CSV field by field with a
   //--- counter. The counter version had an off-by-one and was keying the seen-set on
   //--- event_id instead of value_id - and because event_id identifies the event TYPE, not
   //--- the release, that would have made every restart permanently skip all future
   //--- releases of every event already in the file. Indexing a split array by name-checked
   //--- position cannot drift like that when the schema changes.
   //--- FILE_SHARE_READ|FILE_SHARE_WRITE is mandatory in this codebase: an exclusive open
   //--- fails 5004 whenever anything else holds the file, and looks like "no history yet".
   int h = FileOpen(InpFileName, FILE_READ | FILE_TXT | FILE_ANSI | FILE_COMMON |
                    FILE_SHARE_READ | FILE_SHARE_WRITE);
   if(h == INVALID_HANDLE)
     {
      PrintFormat("%s: no existing file (%s) - starting fresh", __FUNCTION__, InpFileName);
      return;
     }
   string header = FileIsEnding(h) ? "" : FileReadString(h);
   StringTrimRight(header);
   StringTrimLeft(header);
   if(StringLen(header) > 0 && header != CSV_HEADER)
     {
      //--- Old schema. Rotate rather than append: mixing layouts silently mis-parses every
      //--- old row, and the existing rows are backfill that will be re-derived anyway.
      FileClose(h);
      string bak = InpFileName + "." + IntegerToString((int)TimeCurrent()) + ".old";
      if(FileMove(InpFileName, FILE_COMMON, bak, FILE_COMMON))
         PrintFormat("%s: schema changed - previous file rotated to %s", __FUNCTION__, bak);
      else
         PrintFormat("%s: schema changed but rotate FAILED (error %d) - "
                     "move %s aside by hand", __FUNCTION__, GetLastError(), InpFileName);
      return;
     }
   int vcol = -1;
   string hf[];
   if(StringSplit(header, ',', hf) > 0)
      for(int i = 0; i < ArraySize(hf); i++)
         if(hf[i] == "value_id")
            vcol = i;
   if(vcol < 0)
     {
      FileClose(h);
      Print(__FUNCTION__ + ": header has no value_id column - refusing to guess");
      return;
     }
   while(!FileIsEnding(h))
     {
      string line = FileReadString(h);
      if(StringLen(line) < 5)
         continue;
      string f[];
      if(StringSplit(line, ',', f) > vcol)
         SeenInsert((ulong)StringToInteger(f[vcol]));
     }
   FileClose(h);
   PrintFormat("%s: loaded %d previously recorded values from %s",
               __FUNCTION__, ArraySize(g_seen), InpFileName);
  }

//+------------------------------------------------------------------+
//| Append one row. Header written only when the file is created.     |
//+------------------------------------------------------------------+
bool AppendRow(const string row)
  {
   int h = FileOpen(InpFileName, FILE_READ | FILE_WRITE | FILE_TXT | FILE_ANSI | FILE_COMMON |
                    FILE_SHARE_READ | FILE_SHARE_WRITE);
   if(h == INVALID_HANDLE)
     {
      PrintFormat("%s: cannot open %s, error %d", __FUNCTION__, InpFileName, GetLastError());
      return false;
     }
   if(FileSize(h) == 0)
      FileWriteString(h, CSV_HEADER + "\r\n");
   FileSeek(h, 0, SEEK_END);
   FileWriteString(h, row + "\r\n");
   FileFlush(h);
   FileClose(h);
   return true;
  }

//+------------------------------------------------------------------+
double CalValue(const long v)
  {
   return (v == CAL_NO_VALUE) ? EMPTY_VALUE : (double)v / CAL_SCALE;
  }

string CalField(const long v)
  {
   return (v == CAL_NO_VALUE) ? "" : DoubleToString((double)v / CAL_SCALE, 6);
  }

//+------------------------------------------------------------------+
int OnInit()
  {
   if(InpAllCountries)
     {
      MqlCalendarCountry countries[];
      int total = CalendarCountries(countries);
      ArrayResize(g_countries, 0);
      for(int i = 0; i < total; i++)
        {
         int n = ArraySize(g_countries);
         ArrayResize(g_countries, n + 1);
         g_countries[n] = countries[i].code;
        }
     }
   else
     {
      //--- this symbol's base+quote currencies only
      string baseCcy = SymbolInfoString(_Symbol, SYMBOL_CURRENCY_BASE);
      string profCcy = SymbolInfoString(_Symbol, SYMBOL_CURRENCY_PROFIT);
      MqlCalendarCountry countries[];
      int total = CalendarCountries(countries);
      ArrayResize(g_countries, 0);
      for(int i = 0; i < total; i++)
         if(countries[i].currency == baseCcy || countries[i].currency == profCcy)
           {
            int n = ArraySize(g_countries);
            ArrayResize(g_countries, n + 1);
            g_countries[n] = countries[i].code;
           }
     }

   if(ArraySize(g_countries) == 0)
     {
      Print(__FUNCTION__ + ": no countries resolved - calendar unavailable on this terminal?");
      return INIT_FAILED;
     }
   LoadSeen();
   EventSetTimer(MathMax(5, InpPollSeconds));
   PrintFormat("CalendarRecorder: watching %d countries, poll %ds, file %s",
               ArraySize(g_countries), InpPollSeconds, InpFileName);
   Poll();
   return INIT_SUCCEEDED;
  }

void OnDeinit(const int reason)
  {
   EventKillTimer();
  }

void OnTimer()
  {
   Poll();
  }

//+------------------------------------------------------------------+
//| One sweep. Cheap enough to run every 30s: the terminal serves the |
//| calendar from a local cache, and the window is a couple of days.  |
//+------------------------------------------------------------------+
void Poll()
  {
   datetime now  = TimeCurrent();
   datetime from = now - (datetime)InpLookbackHours * 3600;
   datetime to   = now + (datetime)InpLookaheadHours * 3600;

   double bid = SymbolInfoDouble(_Symbol, SYMBOL_BID);
   double ask = SymbolInfoDouble(_Symbol, SYMBOL_ASK);
   long   spr = SymbolInfoInteger(_Symbol, SYMBOL_SPREAD);
   int    added = 0;
   int    liveBefore = g_live;

   for(int c = 0; c < ArraySize(g_countries); c++)
     {
      MqlCalendarValue values[];
      if(!CalendarValueHistory(values, from, to, g_countries[c]))
         continue;                       // country simply has nothing in the window

      for(int i = 0; i < ArraySize(values); i++)
        {
         //--- Only record once the ACTUAL figure exists. A scheduled-but-unreleased row has
         //--- no surprise to measure, and recording it early would burn the value_id in the
         //--- seen-set so the real release is never captured.
         if(values[i].actual_value == CAL_NO_VALUE)
            continue;
         if(SeenContains(values[i].id))
            continue;

         MqlCalendarEvent ev;
         if(!CalendarEventById(values[i].event_id, ev))
            continue;
         if((int)ev.importance < InpMinImportance)
           {
            SeenInsert(values[i].id);     // remember, so it is not re-examined every poll
            continue;
           }

         MqlCalendarCountry ctry;
         string currency = "";
         if(CalendarCountryById(ev.country_id, ctry))
            currency = ctry.currency;

         string name = ev.name;
         StringReplace(name, ",", ";");   // keep the CSV parseable

         long   lag = (long)now - (long)values[i].time;
         string cap = (lag >= 0 && lag <= InpLiveLagSeconds) ? "LIVE" : "BACKFILL";
         if(cap == "LIVE")
            g_live++;

         string row = StringFormat(
                         "%s,%s,%I64d,%I64u,%I64u,%s,%s,%s,%d,%s,%s,%s,%s,%s,%d,%d,%d,%d,%.6f,%s,%.5f,%.5f,%d",
                         TimeToString(now, TIME_DATE | TIME_SECONDS), cap, lag,
                         values[i].id, values[i].event_id,
                         TimeToString(values[i].time, TIME_DATE | TIME_SECONDS),
                         g_countries[c], currency, (int)ev.importance, name,
                         CalField(values[i].actual_value),
                         CalField(values[i].forecast_value),
                         CalField(values[i].prev_value),
                         CalField(values[i].revised_prev_value),
                         values[i].revision, (int)values[i].impact_type,
                         (int)ev.unit, (int)ev.digits, (double)ev.multiplier,
                         _Symbol, bid, ask, (int)spr);

         if(AppendRow(row))
           {
            SeenInsert(values[i].id);
            added++;
           }
        }
     }
   if(added > 0)
     {
      int live = g_live - liveBefore;
      PrintFormat("CalendarRecorder: +%d row(s): %d LIVE, %d BACKFILL (usable this session: "
                  "%d, file total: %d)", added, live, added - live, g_live, ArraySize(g_seen));
      if(live == 0)
         Print("CalendarRecorder: all BACKFILL - post-revision figures, "
               "NOT usable for surprise research. Only LIVE rows are.");
     }
   g_lastPoll = now;
  }
//+------------------------------------------------------------------+