Warrior_EA/System/AltDataHttpClient.mqh

//+------------------------------------------------------------------+
//|                                          AltDataHttpClient.mqh   |
//|                                                     AnimateDread |
//| Generic HTTP client for AltDataFetch's per-source pipelines: the |
//| per-host 4014 backoff, URL masking for the journal, and the      |
//| minimal JSON/URL-encoding helpers the Socrata/FRED/EIA/CBOE      |
//| responses need. Knows nothing about FRED/COT/EIA/GEX - only how  |
//| to fetch a URL and parse a flat JSON body.                       |
//+------------------------------------------------------------------+
#ifndef ALTDATAHTTPCLIENT_MQH
#define ALTDATAHTTPCLIENT_MQH

#define ALTFETCH_TIMEOUT_MS     15000
//--- the SPX option chain is ~13 MB; the small feeds' 15 s would time out on a slow link
#define ALTFETCH_GEX_TIMEOUT_MS 90000

class CAltDataHttpClient
  {
private:
   string            m_blockedHost[];   // 4014 backoff, PER HOST - one missing whitelist entry
   datetime          m_blockedUntil[];  // must not silence the hosts that are whitelisted
   bool              m_urlAlerted;      // full four-line reference printed once per session

   //--- "https://host/..." -> "https://host" (what the MT5 whitelist dialog wants)
   string            HostOf(string url)
     {
      int p = StringFind(url, "://");
      int q = StringFind(url, "/", p < 0 ? 0 : p + 3);
      return (q < 0) ? url : StringSubstr(url, 0, q);
     }

   //--- URL safe for the JOURNAL: api_key values masked. The 2026-08-16 EIA failure
   //--- printed the first 80 chars of the URL - which included most of the key.
   //--- Every error path that echoes a URL must go through this.
   string            MaskUrl(string url)
     {
      int p = StringFind(url, "api_key=");
      if(p < 0)
         return StringSubstr(url, 0, 100);
      int e = StringFind(url, "&", p);
      string masked = StringSubstr(url, 0, p + 8) + "***";
      if(e > 0)
         masked += StringSubstr(url, e);
      return StringSubstr(masked, 0, 100);
     }

   int               BlockedIndex(string host)
     {
      for(int i = 0; i < ArraySize(m_blockedHost); i++)
         if(m_blockedHost[i] == host)
            return i;
      return -1;
     }

public:
                     CAltDataHttpClient(void) : m_urlAlerted(false) {}

   bool              HttpGet(string url, string &body, int timeoutMs = ALTFETCH_TIMEOUT_MS)
     {
      //--- PER-HOST backoff. An in-flight request cannot be cancelled; refusing to START another
      //--- one is the whole of the remedy.
      if(IsStopped())
         return false;
      string host = HostOf(url);
      int bi = BlockedIndex(host);
      if(bi >= 0 && TimeCurrent() < m_blockedUntil[bi])
         return false;
      char data[], result[];
      string rh;
      ResetLastError();
      //--- identify ourselves: some API gateways reject requests with an empty User-Agent
      //--- at the transport level (candidate cause of the EIA 1003 on first live fetch)
      int status = WebRequest("GET", url, "User-Agent: WarriorEA/1.0\r\n", timeoutMs, data, result, rh);
      if(status == -1)
        {
         int err = GetLastError();
         if(err == 4014)
           {
            //--- Backoff, not a permanent latch: the whitelist takes effect for a RUNNING EA
            //--- the moment the user saves the options dialog, so retrying hourly picks the
            //--- fix up without a re-attach.
            bool firstTimeThisHost = (bi < 0);
            if(bi < 0)
              {
               bi = ArraySize(m_blockedHost);
               ArrayResize(m_blockedHost, bi + 1);
               ArrayResize(m_blockedUntil, bi + 1);
               m_blockedHost[bi] = host;
              }
            m_blockedUntil[bi] = TimeCurrent() + 3600;
            if(firstTimeThisHost)
              {
               //--- ALWAYS name the offender - the whole point of this message is telling
               //--- the user which entry is missing, not that "something" is
               Alert("Warrior EA: web request to ", host, " is blocked (4014) - this exact ",
                     "address is missing from the WebRequest whitelist (or the checkbox is ",
                     "off). Copy-paste line and steps are in the Experts log.");
               Print("AltDataFetch: ==================================================================");
               Print("AltDataFetch: BLOCKED host (add this exact line as its own whitelist entry):");
               Print(host);
               Print("AltDataFetch: MT5 menu: Tools > Options > Expert Advisors ->");
               Print("AltDataFetch: tick 'Allow WebRequest for listed URL', add the line, click OK.");
               Print("AltDataFetch: Takes effect immediately - the EA retries within the hour.");
               if(!m_urlAlerted)
                 {
                  m_urlAlerted = true;
                  Print("AltDataFetch: For reference, the FULL set this EA uses (each its own entry):");
                  Print("https://publicreporting.cftc.gov");
                  Print("https://api.stlouisfed.org");
                  Print("https://api.eia.gov");
                  Print("https://cdn.cboe.com");
                  Print("AltDataFetch: Until fixed it runs normally on cached alt-data files - features");
                  Print("AltDataFetch: stay valid but stop updating past their last downloaded date.");
                 }
               Print("AltDataFetch: ==================================================================");
              }
            else
               Print("AltDataFetch: still blocked (4014): " + host + " - retrying in an hour.");
           }
         else
            PrintFormat("AltDataFetch: WebRequest error %d for %s", err, MaskUrl(url));
         return false;
        }
      if(status != 200)
        {
         PrintFormat("AltDataFetch: HTTP %d for %s%s", status, MaskUrl(url),
                     status >= 1000 ? " (1xxx = MT5 transport-layer failure, not a server "
                     "response - retried on the normal hourly schedule)" : "");
         return false;
        }
      body = CharArrayToString(result, 0, WHOLE_ARRAY, CP_UTF8);
      return true;
     }

   //--- minimal targeted JSON scan: value of "key":"..." (or numeric) at/after position `from`.
   //--- Sufficient for Socrata/FRED responses, which are flat arrays of flat objects.
   string            JsonField(const string &body, string key, int from, int &valueEnd)
     {
      valueEnd = -1;
      string tag = "\"" + key + "\":";
      int p = StringFind(body, tag, from);
      if(p < 0)
         return "";
      p += StringLen(tag);
      if(StringGetCharacter(body, p) == '"')
        {
         int q = StringFind(body, "\"", p + 1);
         if(q < 0)
            return "";
         valueEnd = q + 1;
         return StringSubstr(body, p + 1, q - p - 1);
        }
      int e = p;
      while(e < StringLen(body))
        {
         ushort c = StringGetCharacter(body, e);
         if((c >= '0' && c <= '9') || c == '.' || c == '-' || c == '+' || c == 'e' || c == 'E')
            e++;
         else
            break;
        }
      valueEnd = e;
      return StringSubstr(body, p, e - p);
     }

   string            UrlEncodePart(string s)
     {
      string out = "";
      for(int i = 0; i < StringLen(s); i++)
        {
         ushort c = StringGetCharacter(s, i);
         //--- '%' MUST come first and MUST be escaped: SoQL like-predicates use it as the
         //--- wildcard ('EURO FX -%'), and an unescaped one turns the rest of the query into
         //--- a malformed percent-escape.
         if(c == '%')       out += "%25";
         else if(c == ' ')  out += "%20";
         else if(c == '\'') out += "%27";
         else if(c == '&')  out += "%26";
         else if(c == '>')  out += "%3E";
         else if(c == '=')  out += "%3D";
         else if(c == '+')  out += "%2B";
         else if(c == '#')  out += "%23";
         else               out += ShortToString(c);
        }
      return out;
     }
  };
#endif