mql5-tick-compression/lib/TkzSelfTest.mqh

//+------------------------------------------------------------------+
//|                                                  TkzSelfTest.mqh |
//|  Self-test of the TKZ codec: synthetic edge cases, round trips   |
//|  of real ticks and archive file handling.                        |
//+------------------------------------------------------------------+
#ifndef TKZ_SELFTEST_MQH
#define TKZ_SELFTEST_MQH

#include "TkzArchive.mqh"

// Reference output of the C++ build of this exact code for the golden data
// set below. A different value means the compiled codec does not produce the
// reference byte stream (files would not be portable).
#define TKZ_GOLDEN_SIZE   117563
#define TKZ_GOLDEN_HASH   3252087341

class CTkzSelfTest
  {
private:
   CTkzCodec         m_enc;
   CTkzCodec         m_dec;
   ulong             m_seed;
   int               m_pass;
   int               m_fail;
   int               m_n;
   long              m_t[];
   double            m_b[];
   double            m_a[];
   long              m_ot[];
   double            m_ob[];
   double            m_oa[];
   uchar             m_buf[];

   ulong             Rnd(void)
     {
      m_seed^=m_seed<<13;
      m_seed^=m_seed>>7;
      m_seed^=m_seed<<17;
      return m_seed;
     }
   long              RndInt(const long range) { return (long)(Rnd()%(ulong)range); }
   double            FromBits(const long v)
     {
      UTkzBits u;
      u.l=v;
      return u.d;
     }
   long              ToBits(const double v)
     {
      UTkzBits u;
      u.d=v;
      return u.l;
     }
   void              Clear(void) { m_n=0; }
   void              Push(const long t,const double b,const double a)
     {
      if(m_n>=ArraySize(m_t))
        {
         int cap=m_n*2+1024;
         ArrayResize(m_t,cap);
         ArrayResize(m_b,cap);
         ArrayResize(m_a,cap);
        }
      m_t[m_n]=t;
      m_b[m_n]=b;
      m_a[m_n]=a;
      m_n++;
     }
   void              Result(const bool ok,const string name,const string details)
     {
      if(ok)
         m_pass++;
      else
         m_fail++;
      PrintFormat("TKZ self-test %s: %s: %s",ok ? "PASS" : "FAIL",name,details);
     }
   double            Rate(const int n,const ulong us)
     {
      if(us==0)
         return 0.0;
      return (double)n*1e6/(double)us;
     }

public:
                     CTkzSelfTest(void)
     {
      m_seed=88172645463325252;
      m_pass=0;
      m_fail=0;
      m_n=0;
     }
   int               Passed(void) { return m_pass; }
   int               Failed(void) { return m_fail; }

   //--- encode, decode and compare bitwise; returns the chunk size or -1
   int               RoundTrip(const long &t[],const double &b[],const double &a[],const int start,const int n,const string name)
     {
      ArrayResize(m_buf,0);
      ulong t0=GetMicrosecondCount();
      int sz=m_enc.Encode(t,b,a,start,n,m_buf);
      ulong t1=GetMicrosecondCount();
      if(sz<0)
        {
         Result(false,name,"encode error: "+m_enc.LastError());
         return -1;
        }
      int dn=m_dec.Decode(m_buf,0,m_ot,m_ob,m_oa);
      ulong t2=GetMicrosecondCount();
      if(dn!=n)
        {
         Result(false,name,"decode error: "+m_dec.LastError());
         return -1;
        }
      for(int i=0; i<n; i++)
         if(m_ot[i]!=t[start+i] || ToBits(m_ob[i])!=ToBits(b[start+i]) || ToBits(m_oa[i])!=ToBits(a[start+i]))
           {
            Result(false,name,"mismatch at tick "+IntegerToString(i));
            return -1;
           }
      Result(true,name,StringFormat("%d ticks, %d bytes, %.4f bytes/tick, encode %.0f ticks/s, decode %.0f ticks/s",
                                    n,sz,n>0 ? (double)sz/(double)n : 0.0,Rate(n,t1-t0),Rate(n,t2-t1)));
      return sz;
     }

   //--- FNV-1a of a byte range
   uint              Fnv(const uchar &data[],const int n)
     {
      uint h=(uint)2166136261;
      for(int i=0; i<n; i++)
         h=(uint)((h^data[i])*(uint)16777619);
      return h;
     }

   //--- byte-exact comparison with the reference implementation: three chunks
   //--- that exercise negative operands of /, % and >>, the decimal sub-grid,
   //--- escapes, crossed quotes, a time step with negative steps and a price step
   void              RunGolden(void)
     {
      ulong saved=m_seed;
      m_seed=0x2545F4914F6CDD1D;
      ArrayResize(m_buf,0);
      int i;
      //--- 1. forex-like stream
      Clear();
      long t=1700000000000;
      long p=108000;
      long sp=2;
      for(i=0; i<50000; i++)
        {
         long r=RndInt(1000);
         t+=(r<100) ? (long)0 : 1+RndInt(2000);
         if(r%3==0)
            p+=RndInt(5)-2;
         if(r%7==0)
            sp=1+RndInt(4);
         Push(t,(double)p/1e5,(double)(p+sp)/1e5);
        }
      int ok1=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
      //--- 2. negative prices on a decimal sub-grid, crossed quotes, escapes, time going back
      Clear();
      t=1700000000000;
      long q=-5000;
      for(i=0; i<20000; i++)
        {
         long r=RndInt(1000);
         t+=(r<50) ? -RndInt(300) : RndInt(700);
         if(r%2==0)
            q+=(RndInt(5)-2)*10;
         long qa=q+((r%13==0) ? -20 : 30);
         long rb=(r%17==0) ? 8 : 5;
         double b=(double)(q+rb)/1e3;
         double a=(double)(qa+5)/1e3;
         if(i%500==7)
            b=FromBits((long)Rnd());
         Push(t,b,a);
        }
      int ok2=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
      //--- 3. 250 ms time step with negative steps, tick size of 25 points
      Clear();
      t=1700000000000;
      long z=400000;
      for(i=0; i<10000; i++)
        {
         long r=RndInt(100);
         t+=250*((r<5) ? -RndInt(8) : RndInt(12));
         z+=25*(RndInt(7)-3);
         Push(t,(double)z/100.0,(double)(z+50)/100.0);
        }
      int ok3=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
      m_seed=saved;
      int sz=ArraySize(m_buf);
      uint h=Fnv(m_buf,sz);
      bool ok=(ok1>0 && ok2>0 && ok3>0 && sz==TKZ_GOLDEN_SIZE && h==(uint)TKZ_GOLDEN_HASH);
      //--- the chunks must have taken the intended paths
      STkzChunkInfo i1,i2,i3;
      if(ok)
         ok=m_enc.ReadInfo(m_buf,0,i1) && m_enc.ReadInfo(m_buf,ok1,i2) && m_enc.ReadInfo(m_buf,ok1+ok2,i3);
      if(ok)
         ok=(i1.modulus==1 && i2.modulus==10 && i2.flags==1 && i3.time_step==250 && i3.price_step==25);
      Result(ok,"golden output",StringFormat("size %d hash %u (reference %d / %u)",sz,h,TKZ_GOLDEN_SIZE,(uint)TKZ_GOLDEN_HASH));
     }

   //--- synthetic edge cases
   void              RunSynthetic(void)
     {
      int i;
      double nan=FromBits(0x7FF8000000000000);
      double inf=FromBits(0x7FF0000000000000);
      double negzero=FromBits(((long)1)<<63);
      //--- empty and single tick
      Clear();
      RoundTrip(m_t,m_b,m_a,0,0,"empty chunk");
      Clear();
      Push(1700000000123,1.23456,1.23467);
      RoundTrip(m_t,m_b,m_a,0,m_n,"single tick");
      //--- constant prices and times
      Clear();
      for(i=0; i<1000; i++)
         Push(1700000000000,1.1,1.1);
      RoundTrip(m_t,m_b,m_a,0,m_n,"constant");
      //--- forex-like random walk, 5 digits
      Clear();
      long t=1700000000000;
      long p=108000;
      for(i=0; i<100000; i++)
        {
         t+=RndInt(3000);
         p+=RndInt(7)-3;
         long sp=1+RndInt(4);
         Push(t,(double)p/1e5,(double)(p+sp)/1e5);
        }
      RoundTrip(m_t,m_b,m_a,0,m_n,"random walk");
      //--- time anomalies: negative steps, huge gaps, duplicates, extreme values
      Clear();
      t=1700000000000;
      for(i=0; i<30000; i++)
        {
         long r=RndInt(100);
         if(r==0)
            t-=RndInt(100000);
         else
            if(r==1)
               t+=RndInt(4000000000000);
            else
               if(r>=30)
                  t+=RndInt(500);
         Push(t,2000.0+(double)RndInt(1000)/100.0,2001.0+(double)RndInt(1000)/100.0);
        }
      Push(-5000000000000,1.0,2.0);
      Push(9000000000000000000,1.0,2.0);
      Push(-9000000000000000000,1.0,2.0);
      RoundTrip(m_t,m_b,m_a,0,m_n,"time anomalies");
      //--- irregular doubles: NaN, Inf, -0.0, denormals, non-normalized values
      Clear();
      t=1700000000000;
      for(i=0; i<20000; i++)
        {
         t+=100;
         double b=1.2345+(double)(i%50)*1e-4;
         double a=b+0.0002;
         int r=i%97;
         if(r==3)
            b=nan;
         if(r==5)
            a=inf;
         if(r==7)
            b=-inf;
         if(r==11)
            a=negzero;
         if(r==13)
            b=FromBits(1);
         if(r==17)
            a=0.1+0.2;
         if(r==19)
            b=1e300;
         if(r==23)
            a=-1e300;
         if(r==29)
            b=FromBits(ToBits(1.2345)+1);
         Push(t,b,a);
        }
      RoundTrip(m_t,m_b,m_a,0,m_n,"irregular doubles");
      //--- random bit patterns
      Clear();
      for(i=0; i<3000; i++)
         Push(1700000000000+(long)i*7,FromBits((long)Rnd()),FromBits((long)Rnd()));
      RoundTrip(m_t,m_b,m_a,0,m_n,"random bit patterns");
      //--- 0.25 tick grid with an offset and whole-second timestamps
      Clear();
      t=1700000000000;
      long q=20000;
      for(i=0; i<50000; i++)
        {
         t+=1000*(1+RndInt(5));
         q+=RndInt(9)-4;
         double b=(double)q*0.25+0.01;
         Push(t,b,b+0.5);
        }
      RoundTrip(m_t,m_b,m_a,0,m_n,"0.25 grid, 1 s timestamps");
      //--- negative prices and zero ask
      Clear();
      t=1700000000000;
      for(i=0; i<30000; i++)
        {
         t+=RndInt(200);
         double b=-37.63+(double)(RndInt(2000)-1000)/100.0;
         double a=(i%10==0) ? 0.0 : b+0.03;
         Push(t,b,a);
        }
      RoundTrip(m_t,m_b,m_a,0,m_n,"negative prices");
      //--- 12 decimal digits and large values
      Clear();
      for(i=0; i<5000; i++)
         Push(1700000000000+i,0.000012345678+(double)i*1e-12,0.000012345699+(double)i*1e-12);
      RoundTrip(m_t,m_b,m_a,0,m_n,"12 digits");
      Clear();
      for(i=0; i<5000; i++)
         Push(1700000000000+i,3999999999.5+(double)i,3999999999.5+2.0*(double)i);
      RoundTrip(m_t,m_b,m_a,0,m_n,"large values");
      //--- corruption must be detected, never crash
      Clear();
      t=1700000000000;
      for(i=0; i<20000; i++)
        {
         t+=RndInt(300);
         Push(t,1.1+(double)RndInt(100)/1e5,1.1002+(double)RndInt(100)/1e5);
        }
      ArrayResize(m_buf,0);
      int sz=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
      uchar damaged[];
      int detected=0;
      int trials=100;
      for(int k=0; k<trials && sz>TKZ_HEADER_SIZE; k++)
        {
         ArrayResize(damaged,0);
         ArrayCopy(damaged,m_buf,0,0,sz);
         int pos=TKZ_HEADER_SIZE+(int)RndInt(sz-TKZ_HEADER_SIZE);
         damaged[pos]=(uchar)(damaged[pos]^(uchar)(1+RndInt(255)));
         if(m_dec.Decode(damaged,0,m_ot,m_ob,m_oa)<0)
            detected++;
        }
      Result(detected>=trials-2,"corruption detection",StringFormat("%d of %d corrupted chunks rejected",detected,trials));
     }

   //--- round trip of real ticks, as one chunk and as several chunks
   void              RunTicks(const MqlTick &ticks[],const int n,const string name)
     {
      Clear();
      for(int i=0; i<n; i++)
         Push(ticks[i].time_msc,ticks[i].bid,ticks[i].ask);
      RoundTrip(m_t,m_b,m_a,0,m_n,name+" (one chunk)");
      int parts=8;
      int step=n/parts+1;
      long total=0;
      bool ok=true;
      for(int s=0; s<n; s+=step)
        {
         int c=(n-s<step) ? n-s : step;
         ArrayResize(m_buf,0);
         int sz=m_enc.Encode(m_t,m_b,m_a,s,c,m_buf);
         if(sz<0 || m_dec.Decode(m_buf,0,m_ot,m_ob,m_oa)!=c)
           {
            ok=false;
            break;
           }
         for(int i=0; i<c; i++)
            if(m_ot[i]!=m_t[s+i] || ToBits(m_ob[i])!=ToBits(m_b[s+i]) || ToBits(m_oa[i])!=ToBits(m_a[s+i]))
              {
               ok=false;
               break;
              }
         total+=sz;
        }
      Result(ok,name+" (8 chunks)",StringFormat("%I64d bytes, %.4f bytes/tick",total,n>0 ? (double)total/(double)n : 0.0));
     }

   //--- archive writer/reader with a temporary file
   void              RunArchive(void)
     {
      string file="tkz_selftest.tkz";
      Clear();
      long t=1700000000000;
      long p=150000;
      for(int i=0; i<60000; i++)
        {
         t+=1+RndInt(800);
         p+=RndInt(5)-2;
         Push(t,(double)p/1e3,(double)(p+20+RndInt(3))/1e3);
        }
      CTkzWriter w;
      w.SetVerify(true);
      bool ok=w.Open(file,"SELFTEST",3,false);
      for(int s=0; ok && s<40000; s+=10000)
         ok=w.Write(m_t,m_b,m_a,s,10000);
      w.Close();
      CTkzWriter w2;
      if(ok)
         ok=w2.Open(file,"IGNORED",0,false,true);
      if(ok)
         ok=w2.Write(m_t,m_b,m_a,40000,20000);
      w2.Close();
      if(!ok)
        {
         Result(false,"archive","write failed: "+w.LastError()+" "+w2.LastError());
         return;
        }
      CTkzReader r;
      if(!r.Open(file,false))
        {
         Result(false,"archive","open failed: "+r.LastError());
         return;
        }
      MqlTick all[];
      int got=r.ReadAll(all);
      ok=(got==m_n && r.ChunkCount()==5 && r.ArchiveSymbol()=="SELFTEST" && r.ArchiveDigits()==3);
      for(int i=0; ok && i<got; i++)
         if(all[i].time_msc!=m_t[i] || ToBits(all[i].bid)!=ToBits(m_b[i]) || ToBits(all[i].ask)!=ToBits(m_a[i]))
            ok=false;
      long from=m_t[15000];
      long to=m_t[45000];
      MqlTick part[];
      int pn=r.ReadRange(from,to,part);
      int expect=0;
      for(int i=0; i<m_n; i++)
         if(m_t[i]>=from && m_t[i]<to)
            expect++;
      ok=ok && (pn==expect);
      r.Close();
      FileDelete(file);
      Result(ok,"archive",StringFormat("%d chunks, %d ticks, range query %d ticks",5,got,pn));
     }
  };

#endif // TKZ_SELFTEST_MQH
//+------------------------------------------------------------------+