| //+------------------------------------------------------------------+
|
| //| TkzSelfTest.mqh |
|
| //| Self-test of the TKZ codec: synthetic edge cases, round trips |
|
| //| of real ticks and archive file handling. |
|
| //+------------------------------------------------------------------+
|
| #ifndef TKZ_SELFTEST_MQH
|
| #define TKZ_SELFTEST_MQH
|
|
|
| #include "TkzArchive.mqh"
|
|
|
| // Reference output of the C++ build of this exact code for the golden data
|
| // set below. A different value means the compiled codec does not produce the
|
| // reference byte stream (files would not be portable).
|
| #define TKZ_GOLDEN_SIZE 117563
|
| #define TKZ_GOLDEN_HASH 3252087341
|
|
|
| class CTkzSelfTest
|
| {
|
| private:
|
| CTkzCodec m_enc;
|
| CTkzCodec m_dec;
|
| ulong m_seed;
|
| int m_pass;
|
| int m_fail;
|
| int m_n;
|
| long m_t[];
|
| double m_b[];
|
| double m_a[];
|
| long m_ot[];
|
| double m_ob[];
|
| double m_oa[];
|
| uchar m_buf[];
|
|
|
| ulong Rnd(void)
|
| {
|
| m_seed^=m_seed<<13;
|
| m_seed^=m_seed>>7;
|
| m_seed^=m_seed<<17;
|
| return m_seed;
|
| }
|
| long RndInt(const long range) { return (long)(Rnd()%(ulong)range); }
|
| double FromBits(const long v)
|
| {
|
| UTkzBits u;
|
| u.l=v;
|
| return u.d;
|
| }
|
| long ToBits(const double v)
|
| {
|
| UTkzBits u;
|
| u.d=v;
|
| return u.l;
|
| }
|
| void Clear(void) { m_n=0; }
|
| void Push(const long t,const double b,const double a)
|
| {
|
| if(m_n>=ArraySize(m_t))
|
| {
|
| int cap=m_n*2+1024;
|
| ArrayResize(m_t,cap);
|
| ArrayResize(m_b,cap);
|
| ArrayResize(m_a,cap);
|
| }
|
| m_t[m_n]=t;
|
| m_b[m_n]=b;
|
| m_a[m_n]=a;
|
| m_n++;
|
| }
|
| void Result(const bool ok,const string name,const string details)
|
| {
|
| if(ok)
|
| m_pass++;
|
| else
|
| m_fail++;
|
| PrintFormat("TKZ self-test %s: %s: %s",ok ? "PASS" : "FAIL",name,details);
|
| }
|
| double Rate(const int n,const ulong us)
|
| {
|
| if(us==0)
|
| return 0.0;
|
| return (double)n*1e6/(double)us;
|
| }
|
|
|
| public:
|
| CTkzSelfTest(void)
|
| {
|
| m_seed=88172645463325252;
|
| m_pass=0;
|
| m_fail=0;
|
| m_n=0;
|
| }
|
| int Passed(void) { return m_pass; }
|
| int Failed(void) { return m_fail; }
|
|
|
| //--- encode, decode and compare bitwise; returns the chunk size or -1
|
| int RoundTrip(const long &t[],const double &b[],const double &a[],const int start,const int n,const string name)
|
| {
|
| ArrayResize(m_buf,0);
|
| ulong t0=GetMicrosecondCount();
|
| int sz=m_enc.Encode(t,b,a,start,n,m_buf);
|
| ulong t1=GetMicrosecondCount();
|
| if(sz<0)
|
| {
|
| Result(false,name,"encode error: "+m_enc.LastError());
|
| return -1;
|
| }
|
| int dn=m_dec.Decode(m_buf,0,m_ot,m_ob,m_oa);
|
| ulong t2=GetMicrosecondCount();
|
| if(dn!=n)
|
| {
|
| Result(false,name,"decode error: "+m_dec.LastError());
|
| return -1;
|
| }
|
| for(int i=0; i<n; i++)
|
| if(m_ot[i]!=t[start+i] || ToBits(m_ob[i])!=ToBits(b[start+i]) || ToBits(m_oa[i])!=ToBits(a[start+i]))
|
| {
|
| Result(false,name,"mismatch at tick "+IntegerToString(i));
|
| return -1;
|
| }
|
| Result(true,name,StringFormat("%d ticks, %d bytes, %.4f bytes/tick, encode %.0f ticks/s, decode %.0f ticks/s",
|
| n,sz,n>0 ? (double)sz/(double)n : 0.0,Rate(n,t1-t0),Rate(n,t2-t1)));
|
| return sz;
|
| }
|
|
|
| //--- FNV-1a of a byte range
|
| uint Fnv(const uchar &data[],const int n)
|
| {
|
| uint h=(uint)2166136261;
|
| for(int i=0; i<n; i++)
|
| h=(uint)((h^data[i])*(uint)16777619);
|
| return h;
|
| }
|
|
|
| //--- byte-exact comparison with the reference implementation: three chunks
|
| //--- that exercise negative operands of /, % and >>, the decimal sub-grid,
|
| //--- escapes, crossed quotes, a time step with negative steps and a price step
|
| void RunGolden(void)
|
| {
|
| ulong saved=m_seed;
|
| m_seed=0x2545F4914F6CDD1D;
|
| ArrayResize(m_buf,0);
|
| int i;
|
| //--- 1. forex-like stream
|
| Clear();
|
| long t=1700000000000;
|
| long p=108000;
|
| long sp=2;
|
| for(i=0; i<50000; i++)
|
| {
|
| long r=RndInt(1000);
|
| t+=(r<100) ? (long)0 : 1+RndInt(2000);
|
| if(r%3==0)
|
| p+=RndInt(5)-2;
|
| if(r%7==0)
|
| sp=1+RndInt(4);
|
| Push(t,(double)p/1e5,(double)(p+sp)/1e5);
|
| }
|
| int ok1=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
|
| //--- 2. negative prices on a decimal sub-grid, crossed quotes, escapes, time going back
|
| Clear();
|
| t=1700000000000;
|
| long q=-5000;
|
| for(i=0; i<20000; i++)
|
| {
|
| long r=RndInt(1000);
|
| t+=(r<50) ? -RndInt(300) : RndInt(700);
|
| if(r%2==0)
|
| q+=(RndInt(5)-2)*10;
|
| long qa=q+((r%13==0) ? -20 : 30);
|
| long rb=(r%17==0) ? 8 : 5;
|
| double b=(double)(q+rb)/1e3;
|
| double a=(double)(qa+5)/1e3;
|
| if(i%500==7)
|
| b=FromBits((long)Rnd());
|
| Push(t,b,a);
|
| }
|
| int ok2=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
|
| //--- 3. 250 ms time step with negative steps, tick size of 25 points
|
| Clear();
|
| t=1700000000000;
|
| long z=400000;
|
| for(i=0; i<10000; i++)
|
| {
|
| long r=RndInt(100);
|
| t+=250*((r<5) ? -RndInt(8) : RndInt(12));
|
| z+=25*(RndInt(7)-3);
|
| Push(t,(double)z/100.0,(double)(z+50)/100.0);
|
| }
|
| int ok3=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
|
| m_seed=saved;
|
| int sz=ArraySize(m_buf);
|
| uint h=Fnv(m_buf,sz);
|
| bool ok=(ok1>0 && ok2>0 && ok3>0 && sz==TKZ_GOLDEN_SIZE && h==(uint)TKZ_GOLDEN_HASH);
|
| //--- the chunks must have taken the intended paths
|
| STkzChunkInfo i1,i2,i3;
|
| if(ok)
|
| ok=m_enc.ReadInfo(m_buf,0,i1) && m_enc.ReadInfo(m_buf,ok1,i2) && m_enc.ReadInfo(m_buf,ok1+ok2,i3);
|
| if(ok)
|
| ok=(i1.modulus==1 && i2.modulus==10 && i2.flags==1 && i3.time_step==250 && i3.price_step==25);
|
| Result(ok,"golden output",StringFormat("size %d hash %u (reference %d / %u)",sz,h,TKZ_GOLDEN_SIZE,(uint)TKZ_GOLDEN_HASH));
|
| }
|
|
|
| //--- synthetic edge cases
|
| void RunSynthetic(void)
|
| {
|
| int i;
|
| double nan=FromBits(0x7FF8000000000000);
|
| double inf=FromBits(0x7FF0000000000000);
|
| double negzero=FromBits(((long)1)<<63);
|
| //--- empty and single tick
|
| Clear();
|
| RoundTrip(m_t,m_b,m_a,0,0,"empty chunk");
|
| Clear();
|
| Push(1700000000123,1.23456,1.23467);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"single tick");
|
| //--- constant prices and times
|
| Clear();
|
| for(i=0; i<1000; i++)
|
| Push(1700000000000,1.1,1.1);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"constant");
|
| //--- forex-like random walk, 5 digits
|
| Clear();
|
| long t=1700000000000;
|
| long p=108000;
|
| for(i=0; i<100000; i++)
|
| {
|
| t+=RndInt(3000);
|
| p+=RndInt(7)-3;
|
| long sp=1+RndInt(4);
|
| Push(t,(double)p/1e5,(double)(p+sp)/1e5);
|
| }
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"random walk");
|
| //--- time anomalies: negative steps, huge gaps, duplicates, extreme values
|
| Clear();
|
| t=1700000000000;
|
| for(i=0; i<30000; i++)
|
| {
|
| long r=RndInt(100);
|
| if(r==0)
|
| t-=RndInt(100000);
|
| else
|
| if(r==1)
|
| t+=RndInt(4000000000000);
|
| else
|
| if(r>=30)
|
| t+=RndInt(500);
|
| Push(t,2000.0+(double)RndInt(1000)/100.0,2001.0+(double)RndInt(1000)/100.0);
|
| }
|
| Push(-5000000000000,1.0,2.0);
|
| Push(9000000000000000000,1.0,2.0);
|
| Push(-9000000000000000000,1.0,2.0);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"time anomalies");
|
| //--- irregular doubles: NaN, Inf, -0.0, denormals, non-normalized values
|
| Clear();
|
| t=1700000000000;
|
| for(i=0; i<20000; i++)
|
| {
|
| t+=100;
|
| double b=1.2345+(double)(i%50)*1e-4;
|
| double a=b+0.0002;
|
| int r=i%97;
|
| if(r==3)
|
| b=nan;
|
| if(r==5)
|
| a=inf;
|
| if(r==7)
|
| b=-inf;
|
| if(r==11)
|
| a=negzero;
|
| if(r==13)
|
| b=FromBits(1);
|
| if(r==17)
|
| a=0.1+0.2;
|
| if(r==19)
|
| b=1e300;
|
| if(r==23)
|
| a=-1e300;
|
| if(r==29)
|
| b=FromBits(ToBits(1.2345)+1);
|
| Push(t,b,a);
|
| }
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"irregular doubles");
|
| //--- random bit patterns
|
| Clear();
|
| for(i=0; i<3000; i++)
|
| Push(1700000000000+(long)i*7,FromBits((long)Rnd()),FromBits((long)Rnd()));
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"random bit patterns");
|
| //--- 0.25 tick grid with an offset and whole-second timestamps
|
| Clear();
|
| t=1700000000000;
|
| long q=20000;
|
| for(i=0; i<50000; i++)
|
| {
|
| t+=1000*(1+RndInt(5));
|
| q+=RndInt(9)-4;
|
| double b=(double)q*0.25+0.01;
|
| Push(t,b,b+0.5);
|
| }
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"0.25 grid, 1 s timestamps");
|
| //--- negative prices and zero ask
|
| Clear();
|
| t=1700000000000;
|
| for(i=0; i<30000; i++)
|
| {
|
| t+=RndInt(200);
|
| double b=-37.63+(double)(RndInt(2000)-1000)/100.0;
|
| double a=(i%10==0) ? 0.0 : b+0.03;
|
| Push(t,b,a);
|
| }
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"negative prices");
|
| //--- 12 decimal digits and large values
|
| Clear();
|
| for(i=0; i<5000; i++)
|
| Push(1700000000000+i,0.000012345678+(double)i*1e-12,0.000012345699+(double)i*1e-12);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"12 digits");
|
| Clear();
|
| for(i=0; i<5000; i++)
|
| Push(1700000000000+i,3999999999.5+(double)i,3999999999.5+2.0*(double)i);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,"large values");
|
| //--- corruption must be detected, never crash
|
| Clear();
|
| t=1700000000000;
|
| for(i=0; i<20000; i++)
|
| {
|
| t+=RndInt(300);
|
| Push(t,1.1+(double)RndInt(100)/1e5,1.1002+(double)RndInt(100)/1e5);
|
| }
|
| ArrayResize(m_buf,0);
|
| int sz=m_enc.Encode(m_t,m_b,m_a,0,m_n,m_buf);
|
| uchar damaged[];
|
| int detected=0;
|
| int trials=100;
|
| for(int k=0; k<trials && sz>TKZ_HEADER_SIZE; k++)
|
| {
|
| ArrayResize(damaged,0);
|
| ArrayCopy(damaged,m_buf,0,0,sz);
|
| int pos=TKZ_HEADER_SIZE+(int)RndInt(sz-TKZ_HEADER_SIZE);
|
| damaged[pos]=(uchar)(damaged[pos]^(uchar)(1+RndInt(255)));
|
| if(m_dec.Decode(damaged,0,m_ot,m_ob,m_oa)<0)
|
| detected++;
|
| }
|
| Result(detected>=trials-2,"corruption detection",StringFormat("%d of %d corrupted chunks rejected",detected,trials));
|
| }
|
|
|
| //--- round trip of real ticks, as one chunk and as several chunks
|
| void RunTicks(const MqlTick &ticks[],const int n,const string name)
|
| {
|
| Clear();
|
| for(int i=0; i<n; i++)
|
| Push(ticks[i].time_msc,ticks[i].bid,ticks[i].ask);
|
| RoundTrip(m_t,m_b,m_a,0,m_n,name+" (one chunk)");
|
| int parts=8;
|
| int step=n/parts+1;
|
| long total=0;
|
| bool ok=true;
|
| for(int s=0; s<n; s+=step)
|
| {
|
| int c=(n-s<step) ? n-s : step;
|
| ArrayResize(m_buf,0);
|
| int sz=m_enc.Encode(m_t,m_b,m_a,s,c,m_buf);
|
| if(sz<0 || m_dec.Decode(m_buf,0,m_ot,m_ob,m_oa)!=c)
|
| {
|
| ok=false;
|
| break;
|
| }
|
| for(int i=0; i<c; i++)
|
| if(m_ot[i]!=m_t[s+i] || ToBits(m_ob[i])!=ToBits(m_b[s+i]) || ToBits(m_oa[i])!=ToBits(m_a[s+i]))
|
| {
|
| ok=false;
|
| break;
|
| }
|
| total+=sz;
|
| }
|
| Result(ok,name+" (8 chunks)",StringFormat("%I64d bytes, %.4f bytes/tick",total,n>0 ? (double)total/(double)n : 0.0));
|
| }
|
|
|
| //--- archive writer/reader with a temporary file
|
| void RunArchive(void)
|
| {
|
| string file="tkz_selftest.tkz";
|
| Clear();
|
| long t=1700000000000;
|
| long p=150000;
|
| for(int i=0; i<60000; i++)
|
| {
|
| t+=1+RndInt(800);
|
| p+=RndInt(5)-2;
|
| Push(t,(double)p/1e3,(double)(p+20+RndInt(3))/1e3);
|
| }
|
| CTkzWriter w;
|
| w.SetVerify(true);
|
| bool ok=w.Open(file,"SELFTEST",3,false);
|
| for(int s=0; ok && s<40000; s+=10000)
|
| ok=w.Write(m_t,m_b,m_a,s,10000);
|
| w.Close();
|
| CTkzWriter w2;
|
| if(ok)
|
| ok=w2.Open(file,"IGNORED",0,false,true);
|
| if(ok)
|
| ok=w2.Write(m_t,m_b,m_a,40000,20000);
|
| w2.Close();
|
| if(!ok)
|
| {
|
| Result(false,"archive","write failed: "+w.LastError()+" "+w2.LastError());
|
| return;
|
| }
|
| CTkzReader r;
|
| if(!r.Open(file,false))
|
| {
|
| Result(false,"archive","open failed: "+r.LastError());
|
| return;
|
| }
|
| MqlTick all[];
|
| int got=r.ReadAll(all);
|
| ok=(got==m_n && r.ChunkCount()==5 && r.ArchiveSymbol()=="SELFTEST" && r.ArchiveDigits()==3);
|
| for(int i=0; ok && i<got; i++)
|
| if(all[i].time_msc!=m_t[i] || ToBits(all[i].bid)!=ToBits(m_b[i]) || ToBits(all[i].ask)!=ToBits(m_a[i]))
|
| ok=false;
|
| long from=m_t[15000];
|
| long to=m_t[45000];
|
| MqlTick part[];
|
| int pn=r.ReadRange(from,to,part);
|
| int expect=0;
|
| for(int i=0; i<m_n; i++)
|
| if(m_t[i]>=from && m_t[i]<to)
|
| expect++;
|
| ok=ok && (pn==expect);
|
| r.Close();
|
| FileDelete(file);
|
| Result(ok,"archive",StringFormat("%d chunks, %d ticks, range query %d ticks",5,got,pn));
|
| }
|
| };
|
|
|
| #endif // TKZ_SELFTEST_MQH
|
| //+------------------------------------------------------------------+
|
852010e8ba