//+------------------------------------------------------------------+
//|                                        ImportDukascopyBars.mq5   |
//|                                                     AnimateDread |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "AnimateDread"
#property link      "https://www.mql5.com"
#property version   "1.00"
#property script_show_inputs
#property description "Imports a Dukascopy M1 CSV (the SQX customdata export) as an MT5 custom"
#property description "symbol, so the same EA config can be tested against a SECOND data vendor."

//--- WHY THIS EXISTS
//---
//--- Every result this project has ever produced came from one broker's history. That is not a
//--- validation, it is a single sample: a result that survives only on Five Percent Online's bars
//--- is a result about Five Percent Online. The SQX install ships Dukascopy M1 exports for the
//--- same instruments, which makes a genuine like-for-like cross-vendor check possible - same EA,
//--- same window, same settings, independent prices.
//---
//--- WHAT THE FILES ACTUALLY CONTAIN (measured 2026-09-11, all 11 CSVs, counted by year):
//---   * The header says M1 and the filename says 1990, but the early years are DAILY bars, one
//---     per trading day stamped 00:00. EURUSD/USDJPY carry 257-261 bars/yr from 1990-1998 and
//---     then jump to ~300k/yr from 1999: a clean handover, no overlap, no mixed band.
//---   * The INDICES are not like that and must not be treated as if they were. SP500 is daily
//---     2008-2016, then a PARTIAL intraday band 2017-2021 - 2017 holds 1,472 bars, about six per
//---     session - and only becomes true M1 in 2022. Those middle years would still build D1
//---     bars, from a few minutes of each session, so their highs and lows are systematically
//---     compressed: understated range, understated ATR, stops too tight. NAS100, UK100 and
//---     XTIUSD share the shape. Import the indices only for 2022+, or as daily-era data, never
//---     across the join.
//---   * <SPREAD> is synthetic and drifts with the vendor's era - 50 points in 1999, 40 in 2004,
//---     13 in 2010, 7 in 2015, 0 by 2023. It is NOT a cost model and is deliberately ignored
//---     here; the custom symbol inherits its costs from the broker symbol it is cloned from.
//---
//--- HOW IT IMPORTS. CustomSymbolCreate() with an `origin` clones the broker symbol's contract
//--- specification - digits, tick size, tick value, contract size, margin - so a backtest on the
//--- custom symbol is priced like the real one and only the BARS differ. That is the whole point:
//--- if the two disagree, the disagreement is the data, not the instrument definition.
//+------------------------------------------------------------------+

input string InpCsv      = "EURUSD_M1_dukascopy.csv"; // CSV in MQL5\Files (tab-separated MT5 export)
input string InpOrigin   = "EURUSD";                  // Broker symbol to clone the spec from
input string InpName     = "EURUSD.dk";               // Name for the custom symbol
input string InpFrom     = "2016.01.01";              // Import bars on/after this date ("" = all)
input string InpTo       = "";                        // Import bars before this date ("" = all)

#define BATCH 50000                                   // bars per CustomRatesUpdate call

//+------------------------------------------------------------------+
void OnStart(void)
  {
   const datetime from = (InpFrom == "") ? 0 : StringToTime(InpFrom);
   const datetime to   = (InpTo   == "") ? 0 : StringToTime(InpTo);

   //--- THE SYMBOL. Created once and then reused: deleting and recreating on every run would drop
   //--- any bars already imported, and this file takes minutes to walk.
   if(!SymbolSelect(InpName, true))
     {
      if(!CustomSymbolCreate(InpName, "Custom\\Dukascopy", InpOrigin))
        {
         PrintFormat("CustomSymbolCreate(%s, origin=%s) failed: %d - is '%s' in Market Watch?",
                     InpName, InpOrigin, GetLastError(), InpOrigin);
         return;
        }
      SymbolSelect(InpName, true);
      PrintFormat("created custom symbol %s cloned from %s", InpName, InpOrigin);
     }

   const int h = FileOpen(InpCsv, FILE_READ | FILE_TXT | FILE_ANSI);
   if(h == INVALID_HANDLE)
     {
      PrintFormat("cannot open MQL5\\Files\\%s: %d", InpCsv, GetLastError());
      return;
     }

   MqlRates r[];
   ArrayResize(r, BATCH);
   int      n = 0;
   long     read = 0, kept = 0, bad = 0;
   datetime firstKept = 0, lastKept = 0;
   string   parts[];

   while(!FileIsEnding(h))
     {
      const string line = FileReadString(h);
      read++;
      if(StringLen(line) < 20 || StringGetCharacter(line, 0) == '<')
         continue;                                     // header, or a truncated tail line
      if(StringSplit(line, '\t', parts) < 6)
        { bad++; continue; }

      //--- "1990.01.02" + " " + "00:00:00" is exactly what StringToTime expects.
      const datetime t = StringToTime(parts[0] + " " + parts[1]);
      if(t == 0)
        { bad++; continue; }
      if(from > 0 && t <  from) continue;
      if(to   > 0 && t >= to)   continue;

      r[n].time         = t;
      r[n].open         = StringToDouble(parts[2]);
      r[n].high         = StringToDouble(parts[3]);
      r[n].low          = StringToDouble(parts[4]);
      r[n].close        = StringToDouble(parts[5]);
      r[n].tick_volume  = (ArraySize(parts) > 6) ? (long)StringToInteger(parts[6]) : 0;
      r[n].real_volume  = 0;
      //--- The vendor's spread column is era-dependent fiction (see the note above). Left at 0 so
      //--- the tester prices from the cloned symbol's own spread rather than from it.
      r[n].spread       = 0;

      //--- A zero or inverted bar would import as a valid bar and then quietly poison every ATR
      //--- and range feature built from it. Refuse it here, where it can still be counted.
      if(r[n].open <= 0.0 || r[n].high < r[n].low || r[n].high <= 0.0)
        { bad++; continue; }

      if(firstKept == 0) firstKept = t;
      lastKept = t;
      kept++;
      if(++n >= BATCH)
        {
         if(CustomRatesUpdate(InpName, r, n) < 0)
            PrintFormat("CustomRatesUpdate failed at %s: %d", TimeToString(t), GetLastError());
         n = 0;
         if(kept % 1000000 == 0)
            PrintFormat("  ... %d bars imported (at %s)", kept, TimeToString(t));
        }
     }
   if(n > 0 && CustomRatesUpdate(InpName, r, n) < 0)
      PrintFormat("final CustomRatesUpdate failed: %d", GetLastError());
   FileClose(h);

   PrintFormat("%s: read %d line(s), imported %d bar(s), rejected %d; range %s .. %s",
               InpName, read, kept, bad,
               TimeToString(firstKept, TIME_DATE | TIME_MINUTES),
               TimeToString(lastKept,  TIME_DATE | TIME_MINUTES));
   PrintFormat("Bars(%s, M1) now reports %d; D1 reports %d.",
               InpName, Bars(InpName, PERIOD_M1), Bars(InpName, PERIOD_D1));
  }
//+------------------------------------------------------------------+