mql5-tick-compression/scripts/TkzExport.mq5

//+------------------------------------------------------------------+
//|                                                    TkzExport.mq5 |
//|  Exports tick history (time_msc, bid, ask) into a .tkz archive.  |
//+------------------------------------------------------------------+
#property copyright   "TKZ tick compression"
#property version     "1.1"
#property description "Exports the tick history of a symbol (time_msc, bid, ask) into a compressed .tkz archive."
#property script_show_inputs

#include "../lib/TkzArchive.mqh"

enum ENUM_TKZ_CHUNK
  {
   TKZ_CHUNK_DAY=0,    // Day
   TKZ_CHUNK_WEEK=1,   // Week
   TKZ_CHUNK_MONTH=2   // Month
  };

enum ENUM_TKZ_TICKS
  {
   TKZ_TICKS_ALL=0,    // All ticks (COPY_TICKS_ALL)
   TKZ_TICKS_INFO=1    // Bid/Ask changes only (COPY_TICKS_INFO)
  };

input string         InpSymbol="";                 // Symbol (empty = chart symbol)
input datetime       InpFrom=D'2024.01.01 00:00';  // From (server time)
input datetime       InpTo=0;                      // To (0 = now)
input ENUM_TKZ_CHUNK InpChunk=TKZ_CHUNK_MONTH;     // One chunk per
input ENUM_TKZ_TICKS InpTicks=TKZ_TICKS_ALL;       // Tick selection
input string         InpFile="";                   // File name (empty = <symbol>.tkz)
input bool           InpCommon=false;              // Use the common data folder
input bool           InpVerify=true;               // Decode every chunk again and compare

//+------------------------------------------------------------------+
long DayStart(const datetime t)
  {
   long v=(long)t;
   return v-v%86400;
  }

//+------------------------------------------------------------------+
//| First moment of the next chunk after t                           |
//+------------------------------------------------------------------+
datetime NextBoundary(const datetime t,const ENUM_TKZ_CHUNK mode)
  {
   long d=DayStart(t);
   if(mode==TKZ_CHUNK_DAY)
      return (datetime)(d+86400);
   MqlDateTime s;
   TimeToStruct((datetime)d,s);
   if(mode==TKZ_CHUNK_WEEK)
      return (datetime)(d+(long)(7-s.day_of_week)*86400);   // next Sunday 00:00
   s.day=1;
   s.hour=0;
   s.min=0;
   s.sec=0;
   s.mon++;
   if(s.mon>12)
     {
      s.mon=1;
      s.year++;
     }
   return StructToTime(s);
  }

//+------------------------------------------------------------------+
//| Ticks with from <= time < to, retried while history loads        |
//+------------------------------------------------------------------+
int FetchTicks(const string sym,const uint flags,const datetime from,const datetime to,MqlTick &ticks[])
  {
   ulong from_msc=(ulong)from*1000;
   ulong to_msc=(ulong)to*1000-1;
   for(int attempt=0; attempt<5 && !IsStopped(); attempt++)
     {
      ResetLastError();
      int n=CopyTicksRange(sym,ticks,flags,from_msc,to_msc);
      if(n>=0)
         return n;
      PrintFormat("TKZ: CopyTicksRange(%s, %s) failed, error %d, retrying",sym,TimeToString(from),GetLastError());
      Sleep(2000);
     }
   return -1;
  }

//+------------------------------------------------------------------+
void OnStart()
  {
   string sym=(InpSymbol=="") ? _Symbol : InpSymbol;
   if(!SymbolSelect(sym,true))
     {
      PrintFormat("TKZ: symbol %s is not available",sym);
      return;
     }
   int digits=(int)SymbolInfoInteger(sym,SYMBOL_DIGITS);
   datetime to=(InpTo==0) ? (datetime)(TimeCurrent()+1) : InpTo;
   if(to<=InpFrom)
     {
      Print("TKZ: empty time range");
      return;
     }
   string file=(InpFile=="") ? sym+".tkz" : InpFile;
   uint flags=(InpTicks==TKZ_TICKS_INFO) ? COPY_TICKS_INFO : COPY_TICKS_ALL;

   CTkzWriter writer;
   writer.SetVerify(InpVerify);
   if(!writer.Open(file,sym,digits,InpCommon))
     {
      PrintFormat("TKZ: cannot create %s: %s",file,writer.LastError());
      return;
     }

   long     tms[];
   double   bid[];
   double   ask[];
   MqlTick  day[];
   int      n=0;
   int      chunks=0;
   bool     ok=true;
   ulong    t_start=GetMicrosecondCount();
   ulong    fetch_us=0;
   datetime cur=InpFrom;
   datetime boundary=NextBoundary(cur,InpChunk);

   while(cur<to && !IsStopped())
     {
      datetime next=(datetime)(DayStart(cur)+86400);
      if(next>boundary)
         next=boundary;
      if(next>to)
         next=to;
      ulong f0=GetMicrosecondCount();
      int got=FetchTicks(sym,flags,cur,next,day);
      fetch_us+=GetMicrosecondCount()-f0;
      if(got<0)
        {
         PrintFormat("TKZ: no tick history for %s at %s, stopping",sym,TimeToString(cur));
         ok=false;
         break;
        }
      if(got>0)
        {
         if(ArrayResize(tms,n+got,1000000)<0 || ArrayResize(bid,n+got,1000000)<0 || ArrayResize(ask,n+got,1000000)<0)
           {
            Print("TKZ: out of memory, use a shorter chunk period");
            ok=false;
            break;
           }
         for(int i=0; i<got; i++)
           {
            tms[n+i]=day[i].time_msc;
            bid[n+i]=day[i].bid;
            ask[n+i]=day[i].ask;
           }
         n+=got;
        }
      cur=next;
      if(cur>=boundary || cur>=to)
        {
         if(n>0)
           {
            if(!writer.Write(tms,bid,ask,0,n))
              {
               PrintFormat("TKZ: chunk ending %s failed: %s",TimeToString(cur),writer.LastError());
               ok=false;
               break;
              }
            chunks++;
            long wt=writer.TicksWritten();
            Comment(StringFormat("TKZ export %s: up to %s, %I64d ticks, %.3f bytes/tick",
                                 sym,TimeToString(cur,TIME_DATE),wt,(double)writer.BytesWritten()/(double)(wt>0 ? wt : (long)1)));
           }
         n=0;
         boundary=NextBoundary(cur,InpChunk);
        }
     }

   long   ticks=writer.TicksWritten();
   long   bytes=writer.BytesWritten();
   double enc_s=(double)writer.EncodeMicroseconds()/1e6;
   double ver_s=(double)writer.VerifyMicroseconds()/1e6;
   writer.Close();
   Comment("");
   double total_s=(double)(GetMicrosecondCount()-t_start)/1e6;
   string folder=InpCommon ? TerminalInfoString(TERMINAL_COMMONDATA_PATH)+"\\Files\\" :
                 TerminalInfoString(TERMINAL_DATA_PATH)+"\\MQL5\\Files\\";
   PrintFormat("TKZ: %s %s -> %s%s",ok ? "exported" : "PARTIALLY exported",sym,folder,file);
   if(ticks>0)
     {
      PrintFormat("TKZ: %I64d ticks in %d chunks, %I64d bytes: %.4f bytes/tick, %.1fx smaller than 24-byte records, %.1fx smaller than MqlTick",
                  ticks,chunks,bytes,(double)bytes/(double)ticks,24.0*(double)ticks/(double)bytes,(double)(ticks*sizeof(MqlTick))/(double)bytes);
      PrintFormat("TKZ: encode %.2f s (%.0f ticks/s), verify %.2f s (%.0f ticks/s), history %.2f s, total %.2f s",
                  enc_s,enc_s>0 ? (double)ticks/enc_s : 0.0,ver_s,ver_s>0 ? (double)ticks/ver_s : 0.0,(double)fetch_us/1e6,total_s);
     }
   else
      Print("TKZ: no ticks in the requested range");
  }
//+------------------------------------------------------------------+