mql5-tick-compression/lib/TkzArchive.mqh

//+------------------------------------------------------------------+
//|                                                   TkzArchive.mqh |
//|  .tkz archive files: a 64-byte file header followed by TKZ1      |
//|  chunks (see TkzCodec.mqh and docs/FORMAT.md).                   |
//+------------------------------------------------------------------+
#ifndef TKZ_ARCHIVE_MQH
#define TKZ_ARCHIVE_MQH

#include "TkzCodec.mqh"

#define TKZ_FILE_MAGIC        0x465A4B54   // bytes "TKZF"
#define TKZ_FILE_VERSION      1
#define TKZ_FILE_HEADER_SIZE  64
#define TKZ_SYMBOL_BYTES      32

//+------------------------------------------------------------------+
//| Little-endian helpers shared by writer and reader                |
//+------------------------------------------------------------------+
class CTkzBytes
  {
public:
   void              PutU16(uchar &dst[],const int pos,const int v)
     {
      dst[pos]=(uchar)(v&255);
      dst[pos+1]=(uchar)((v>>8)&255);
     }
   void              PutU32(uchar &dst[],const int pos,const uint v)
     {
      dst[pos]=(uchar)(v&255);
      dst[pos+1]=(uchar)((v>>8)&255);
      dst[pos+2]=(uchar)((v>>16)&255);
      dst[pos+3]=(uchar)((v>>24)&255);
     }
   int               GetU16(const uchar &src[],const int pos)
     {
      return (int)src[pos]|((int)src[pos+1]<<8);
     }
   uint              GetU32(const uchar &src[],const int pos)
     {
      return (uint)src[pos]|((uint)src[pos+1]<<8)|((uint)src[pos+2]<<16)|((uint)src[pos+3]<<24);
     }
  };

//+------------------------------------------------------------------+
//| Writer: creates (or appends to) a .tkz file                      |
//+------------------------------------------------------------------+
class CTkzWriter
  {
private:
   int               m_h;
   CTkzCodec         m_codec;
   CTkzCodec         m_check;
   CTkzBytes         m_io;
   uchar             m_buf[];
   string            m_error;
   long              m_ticks;
   long              m_bytes;
   bool              m_verify;
   ulong             m_enc_us;
   ulong             m_ver_us;
   long              m_vt[];
   double            m_vb[];
   double            m_va[];

public:
                     CTkzWriter(void)
     {
      m_h=INVALID_HANDLE;
      m_error="";
      m_ticks=0;
      m_bytes=0;
      m_verify=false;
      m_enc_us=0;
      m_ver_us=0;
     }
                    ~CTkzWriter(void)
     {
      Close();
     }

   string            LastError(void) { return m_error; }
   long              TicksWritten(void) { return m_ticks; }
   long              BytesWritten(void) { return m_bytes; }
   ulong             EncodeMicroseconds(void) { return m_enc_us; }
   ulong             VerifyMicroseconds(void) { return m_ver_us; }
   //--- decode every written chunk again and compare it with the input
   void              SetVerify(const bool verify) { m_verify=verify; }

   //--- create a new file (or append chunks to an existing one when append=true)
   bool              Open(const string filename,const string symbol,const int digits,const bool common,const bool append=false)
     {
      Close();
      m_error="";
      m_ticks=0;
      m_bytes=0;
      int flags=FILE_BIN|FILE_WRITE;
      if(common)
         flags|=FILE_COMMON;
      if(append && FileIsExist(filename,common ? FILE_COMMON : 0))
        {
         m_h=FileOpen(filename,flags|FILE_READ);
         if(m_h==INVALID_HANDLE)
           {
            m_error="cannot open file for append";
            return false;
           }
         uchar old[];
         ArrayResize(old,TKZ_FILE_HEADER_SIZE);
         if((int)FileReadArray(m_h,old,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE ||
            m_io.GetU32(old,0)!=(uint)TKZ_FILE_MAGIC)
           {
            m_error="existing file is not a TKZ archive";
            FileClose(m_h);
            m_h=INVALID_HANDLE;
            return false;
           }
         FileSeek(m_h,0,SEEK_END);
         m_bytes=(long)FileTell(m_h);
         return true;
        }
      m_h=FileOpen(filename,flags);
      if(m_h==INVALID_HANDLE)
        {
         m_error="cannot create file";
         return false;
        }
      uchar hdr[];
      ArrayResize(hdr,TKZ_FILE_HEADER_SIZE);
      ArrayInitialize(hdr,0);
      m_io.PutU32(hdr,0,(uint)TKZ_FILE_MAGIC);
      m_io.PutU16(hdr,4,TKZ_FILE_VERSION);
      m_io.PutU16(hdr,6,TKZ_FILE_HEADER_SIZE);
      uchar name[];
      int len=StringToCharArray(symbol,name,0,-1,CP_UTF8);
      for(int i=0; i<len && i<TKZ_SYMBOL_BYTES-1; i++)
         hdr[8+i]=name[i];
      m_io.PutU32(hdr,40,(uint)digits);
      if((int)FileWriteArray(m_h,hdr,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE)
        {
         m_error="cannot write file header";
         Close();
         return false;
        }
      m_bytes=TKZ_FILE_HEADER_SIZE;
      return true;
     }

   //--- compress ticks[start..start+count) into one chunk and append it
   bool              WriteTicks(const MqlTick &ticks[],const int start,const int count)
     {
      if(m_h==INVALID_HANDLE)
        {
         m_error="file is not open";
         return false;
        }
      ArrayResize(m_buf,0);
      ulong t0=GetMicrosecondCount();
      int sz=m_codec.EncodeTicks(ticks,start,count,m_buf);
      m_enc_us+=GetMicrosecondCount()-t0;
      if(sz<0)
        {
         m_error=m_codec.LastError();
         return false;
        }
      if(m_verify)
        {
         t0=GetMicrosecondCount();
         int n=m_check.Decode(m_buf,0,m_vt,m_vb,m_va);
         m_ver_us+=GetMicrosecondCount()-t0;
         if(n!=count)
           {
            m_error="verification failed: "+m_check.LastError();
            return false;
           }
         for(int i=0; i<n; i++)
            if(m_vt[i]!=ticks[start+i].time_msc || !m_check.SameBits(m_vb[i],ticks[start+i].bid) ||
               !m_check.SameBits(m_va[i],ticks[start+i].ask))
              {
               m_error="verification failed at tick "+IntegerToString(start+i);
               return false;
              }
        }
      return Flush(sz,count);
     }
   bool              Write(const long &time_msc[],const double &bid[],const double &ask[],const int start,const int count)
     {
      if(m_h==INVALID_HANDLE)
        {
         m_error="file is not open";
         return false;
        }
      ArrayResize(m_buf,0);
      ulong t0=GetMicrosecondCount();
      int sz=m_codec.Encode(time_msc,bid,ask,start,count,m_buf);
      m_enc_us+=GetMicrosecondCount()-t0;
      if(sz<0)
        {
         m_error=m_codec.LastError();
         return false;
        }
      if(m_verify)
        {
         t0=GetMicrosecondCount();
         int n=m_check.Decode(m_buf,0,m_vt,m_vb,m_va);
         m_ver_us+=GetMicrosecondCount()-t0;
         if(n!=count)
           {
            m_error="verification failed: "+m_check.LastError();
            return false;
           }
         for(int i=0; i<n; i++)
            if(m_vt[i]!=time_msc[start+i] || !m_check.SameBits(m_vb[i],bid[start+i]) ||
               !m_check.SameBits(m_va[i],ask[start+i]))
              {
               m_error="verification failed at tick "+IntegerToString(start+i);
               return false;
              }
        }
      return Flush(sz,count);
     }
   void              Close(void)
     {
      if(m_h!=INVALID_HANDLE)
        {
         FileClose(m_h);
         m_h=INVALID_HANDLE;
        }
      m_codec.Release();
      m_check.Release();
      ArrayFree(m_buf);
      ArrayFree(m_vt);
      ArrayFree(m_vb);
      ArrayFree(m_va);
     }

private:
   bool              Flush(const int sz,const int count)
     {
      if((int)FileWriteArray(m_h,m_buf,0,sz)!=sz)
        {
         m_error="file write failed";
         return false;
        }
      FileFlush(m_h);
      m_ticks+=count;
      m_bytes+=sz;
      return true;
     }
  };

//+------------------------------------------------------------------+
//| Reader: chunk index, random access by chunk or by time range     |
//+------------------------------------------------------------------+
class CTkzReader
  {
private:
   int               m_h;
   CTkzCodec         m_codec;
   CTkzBytes         m_io;
   uchar             m_buf[];
   string            m_error;
   string            m_symbol;
   int               m_digits;
   int               m_count;
   long              m_offset[];
   STkzChunkInfo     m_info[];

public:
                     CTkzReader(void)
     {
      m_h=INVALID_HANDLE;
      m_error="";
      m_symbol="";
      m_digits=-1;
      m_count=0;
     }
                    ~CTkzReader(void)
     {
      Close();
     }

   string            LastError(void) { return m_error; }
   string            ArchiveSymbol(void) { return m_symbol; }
   int               ArchiveDigits(void) { return m_digits; }
   int               ChunkCount(void) { return m_count; }

   bool              Open(const string filename,const bool common)
     {
      Close();
      m_error="";
      int flags=FILE_BIN|FILE_READ|FILE_SHARE_READ;
      if(common)
         flags|=FILE_COMMON;
      m_h=FileOpen(filename,flags);
      if(m_h==INVALID_HANDLE)
        {
         m_error="cannot open file";
         return false;
        }
      long fsize=(long)FileSize(m_h);
      uchar hdr[];
      ArrayResize(hdr,TKZ_FILE_HEADER_SIZE);
      if(fsize<TKZ_FILE_HEADER_SIZE || (int)FileReadArray(m_h,hdr,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE ||
         m_io.GetU32(hdr,0)!=(uint)TKZ_FILE_MAGIC)
        {
         m_error="not a TKZ archive";
         Close();
         return false;
        }
      if(m_io.GetU16(hdr,4)!=TKZ_FILE_VERSION)
        {
         m_error="unsupported archive version";
         Close();
         return false;
        }
      int hsize=m_io.GetU16(hdr,6);
      m_symbol=CharArrayToString(hdr,8,TKZ_SYMBOL_BYTES,CP_UTF8);
      m_digits=(int)m_io.GetU32(hdr,40);
      //--- scan chunk headers
      long pos=hsize;
      uchar ch[];
      ArrayResize(ch,TKZ_HEADER_SIZE);
      while(pos+TKZ_HEADER_SIZE<=fsize)
        {
         if(!FileSeek(m_h,pos,SEEK_SET) || (int)FileReadArray(m_h,ch,0,TKZ_HEADER_SIZE)!=TKZ_HEADER_SIZE)
           {
            m_error="read error";
            break;
           }
         STkzChunkInfo info;
         if(!m_codec.ReadInfo(ch,0,info))
           {
            m_error="bad chunk at offset "+IntegerToString(pos)+": "+m_codec.LastError();
            break;
           }
         if(pos+info.total_size>fsize)
           {
            m_error="truncated last chunk";
            break;
           }
         ArrayResize(m_offset,m_count+1,256);
         ArrayResize(m_info,m_count+1,256);
         m_offset[m_count]=pos;
         m_info[m_count]=info;
         m_count++;
         pos+=info.total_size;
        }
      return true;
     }

   void              Close(void)
     {
      if(m_h!=INVALID_HANDLE)
        {
         FileClose(m_h);
         m_h=INVALID_HANDLE;
        }
      m_count=0;
      ArrayFree(m_offset);
      ArrayFree(m_info);
      ArrayFree(m_buf);
      m_codec.Release();
     }

   bool              GetChunkInfo(const int i,STkzChunkInfo &info)
     {
      if(i<0 || i>=m_count)
         return false;
      info=m_info[i];
      return true;
     }

   long              TickCount(void)
     {
      long n=0;
      for(int i=0; i<m_count; i++)
         n+=m_info[i].tick_count;
      return n;
     }

   //--- decode one chunk; ticks are appended when append=true
   int               ReadChunk(const int i,MqlTick &ticks[],const bool append=false)
     {
      if(i<0 || i>=m_count || m_h==INVALID_HANDLE)
        {
         m_error="bad chunk index";
         return -1;
        }
      int sz=m_info[i].total_size;
      ArrayResize(m_buf,sz);
      if(!FileSeek(m_h,m_offset[i],SEEK_SET) || (int)FileReadArray(m_h,m_buf,0,sz)!=sz)
        {
         m_error="read error";
         return -1;
        }
      int n=m_codec.DecodeTicks(m_buf,0,ticks,append);
      if(n<0)
         m_error=m_codec.LastError();
      return n;
     }

   //--- all ticks of the archive
   int               ReadAll(MqlTick &ticks[])
     {
      ArrayResize(ticks,0);
      for(int i=0; i<m_count; i++)
         if(ReadChunk(i,ticks,true)<0)
            return -1;
      return ArraySize(ticks);
     }

   //--- ticks with from_msc <= time_msc < to_msc
   int               ReadRange(const long from_msc,const long to_msc,MqlTick &ticks[])
     {
      ArrayResize(ticks,0);
      MqlTick tmp[];
      int total=0;
      for(int i=0; i<m_count; i++)
        {
         if(m_info[i].tick_count==0 || m_info[i].max_msc<from_msc || m_info[i].min_msc>=to_msc)
            continue;
         int n=ReadChunk(i,tmp,false);
         if(n<0)
            return -1;
         ArrayResize(ticks,total+n,n);
         for(int k=0; k<n; k++)
           {
            if(tmp[k].time_msc>=from_msc && tmp[k].time_msc<to_msc)
              {
               ticks[total]=tmp[k];
               total++;
              }
           }
         ArrayResize(ticks,total);
        }
      return total;
     }
  };

#endif // TKZ_ARCHIVE_MQH
//+------------------------------------------------------------------+