| //+------------------------------------------------------------------+
|
| //| TkzCodec.mqh |
|
| //| TKZ - context-mixing compression of tick history |
|
| //| (time_msc, bid, ask). Lossless, bit-exact round trip. |
|
| //| |
|
| //| One "chunk" = a self-contained compressed block of ticks: |
|
| //| 88-byte header + arithmetic-coded payload. See docs/FORMAT.md. |
|
| //+------------------------------------------------------------------+
|
| #ifndef TKZ_CODEC_MQH
|
| #define TKZ_CODEC_MQH
|
|
|
| #define TKZ_CHUNK_MAGIC 0x315A4B54 // bytes "TKZ1"
|
| #define TKZ_HEADER_SIZE 88
|
| #define TKZ_FORMAT_VERSION 1
|
| #define TKZ_MODEL_VERSION 1
|
|
|
| #define TKZ_MAXIN 12 // max contexts per decision
|
| #define TKZ_NW 13 // weights per mixer set (inputs + bias)
|
| #define TKZ_NCLASS 336 // decision classes
|
| #define TKZ_NSEL 5376 // TKZ_NCLASS * 16 mixer-1 weight sets
|
| #define TKZ_W2SETS 65536 // mixer-2 weight sets (hashed)
|
| #define TKZ_A2SETS 65536 // APM-2 contexts (hashed)
|
| #define TKZ_LIMIT 255 // adaptive counter limit
|
| #define TKZ_MAX_DIGITS 12
|
| #define TKZ_POW10_SIZE 13 // TKZ_MAX_DIGITS + 1
|
|
|
| // decision classes: groups of 80 classes for magnitudes
|
| #define TKZ_G_DT 0
|
| #define TKZ_G_BMAG 1
|
| #define TKZ_G_AMAG 2
|
| #define TKZ_G_SMAG 3
|
| #define TKZ_C_TYPE 320
|
| #define TKZ_C_BRES 321
|
| #define TKZ_C_ARES 322
|
| #define TKZ_C_BSGNZ 323
|
| #define TKZ_C_BSGN 324
|
| #define TKZ_C_ASGNZ 325
|
| #define TKZ_C_ASGN 326
|
| #define TKZ_C_SSGNZ 327
|
| #define TKZ_C_SSGN 328
|
| #define TKZ_C_ESC 329
|
| #define TKZ_C_RAW 330
|
|
|
| #define TKZ_ESC_BID 1
|
| #define TKZ_ESC_ASK 2
|
|
|
| //--- bit cast helper
|
| union UTkzBits
|
| {
|
| double d;
|
| long l;
|
| };
|
|
|
| //--- chunk header as seen by readers
|
| struct STkzChunkInfo
|
| {
|
| int tick_count;
|
| int payload_size;
|
| int total_size; // header + payload
|
| long first_msc;
|
| long last_msc;
|
| long min_msc;
|
| long max_msc;
|
| int digits;
|
| int modulus;
|
| int table_bits;
|
| int flags;
|
| uint checksum;
|
| long time_step;
|
| long price_step;
|
| long bid_base;
|
| long ask_base;
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Codec |
|
| //+------------------------------------------------------------------+
|
| class CTkzCodec
|
| {
|
| private:
|
| //--- model tables
|
| int m_tbits;
|
| uint m_tmask;
|
| uint m_t[]; // counters, blocks of 16 (slot 0 = check)
|
| int m_w1[]; // mixer 1 weights, selected by decision
|
| int m_w2[]; // mixer 2 weights, selected by hash
|
| int m_wf[]; // final mixer (2 inputs + bias)
|
| ushort m_a1[]; // APM 1
|
| ushort m_a2[]; // APM 2
|
| int m_rate[]; // 1024 counter adaptation rates
|
| int m_str[]; // 4096 entries: stretch(p) = ln(p/(1-p))
|
| int m_sq[33];
|
| int m_sqt[]; // 4095 entries: Squash(d) for d in [-2047, 2047]
|
| double m_pow10[TKZ_POW10_SIZE];
|
| //--- current decision contexts
|
| int m_nctx;
|
| uint m_ctx[TKZ_MAXIN];
|
| uint m_blk[TKZ_MAXIN];
|
| uint m_ix[TKZ_MAXIN];
|
| int m_x[TKZ_NW];
|
| uint m_sel2;
|
| //--- binary arithmetic coder
|
| bool m_enc;
|
| uint m_x1;
|
| uint m_x2;
|
| uint m_xv;
|
| uchar m_buf[];
|
| int m_bpos;
|
| int m_bend;
|
| int m_bcap;
|
| bool m_overrun; // decoder needed bytes past the payload
|
| //--- stream parameters
|
| long m_tstep;
|
| int m_mod;
|
| bool m_hasesc;
|
| //--- model state
|
| long m_pt;
|
| long m_pb;
|
| long m_pa;
|
| ulong m_dt1;
|
| ulong m_dt2;
|
| int m_k1;
|
| int m_k2;
|
| int m_ty1;
|
| int m_ty2;
|
| int m_ty3;
|
| long m_mb1;
|
| long m_mb2;
|
| long m_ma1;
|
| long m_ma2;
|
| long m_ms1;
|
| int m_emak;
|
| int m_emak2;
|
| int m_emab;
|
| long m_spavg16;
|
| //--- current tick (input when encoding, output when decoding)
|
| long m_ct;
|
| long m_cb;
|
| long m_ca;
|
| int m_cescb;
|
| int m_cesca;
|
| long m_crawb;
|
| long m_crawa;
|
| //--- work arrays
|
| long m_wb[];
|
| long m_wa[];
|
| uchar m_wesc[];
|
| long m_it[];
|
| double m_ib[];
|
| double m_ia[];
|
| //--- decoder output
|
| long m_ot[];
|
| double m_obid[];
|
| double m_oask[];
|
| uchar m_oflg[];
|
| int m_ocount;
|
| //--- statistics / diagnostics
|
| string m_error;
|
|
|
| //--- hashing
|
| uint H2(const uint a,const uint b)
|
| {
|
| uint h=(uint)(a*(uint)0x9E3779B1)^b;
|
| h^=h>>16;
|
| h=(uint)(h*(uint)0x85EBCA6B);
|
| h^=h>>13;
|
| h=(uint)(h*(uint)0xC2B2AE35);
|
| h^=h>>16;
|
| return h;
|
| }
|
| uint H3(const uint a,const uint b,const uint c) { return H2(H2(a,b),c); }
|
| uint H4(const uint a,const uint b,const uint c,const uint d) { return H2(H2(H2(a,b),c),d); }
|
|
|
| //--- small helpers
|
| int BitLen(ulong v)
|
| {
|
| int n=0;
|
| if((v>>32)!=0) { v>>=32; n+=32; }
|
| if((v>>16)!=0) { v>>=16; n+=16; }
|
| if((v>>8)!=0) { v>>=8; n+=8; }
|
| if((v>>4)!=0) { v>>=4; n+=4; }
|
| if((v>>2)!=0) { v>>=2; n+=2; }
|
| if((v>>1)!=0) { v>>=1; n+=1; }
|
| return n+(int)v;
|
| }
|
| long AbsL(const long v) { return v<0 ? -v : v; }
|
| long Gcd(long a,long b)
|
| {
|
| while(b!=0)
|
| {
|
| long r=a%b;
|
| a=b;
|
| b=r;
|
| }
|
| return a;
|
| }
|
| long FloorMod(const long a,const long m)
|
| {
|
| long r=a%m;
|
| if(r<0)
|
| r+=m;
|
| return r;
|
| }
|
| uint QLog(const long v)
|
| {
|
| if(v<0)
|
| return (uint)(64-BitLen((ulong)(-v)));
|
| return (uint)(64+BitLen((ulong)v));
|
| }
|
| int QSigned(const long v)
|
| {
|
| if(v==0)
|
| return 15;
|
| long m=v<0 ? -v : v;
|
| int q=7;
|
| if(m<4)
|
| q=(int)m;
|
| else
|
| if(m<8)
|
| q=4;
|
| else
|
| if(m<16)
|
| q=5;
|
| else
|
| if(m<32)
|
| q=6;
|
| else
|
| q=7;
|
| return v<0 ? 15-q : 15+q;
|
| }
|
| void Split(const long p,long &q,int &r)
|
| {
|
| if(m_mod<=1)
|
| {
|
| q=p;
|
| r=0;
|
| return;
|
| }
|
| long rr=p%m_mod;
|
| if(rr<0)
|
| rr+=m_mod;
|
| r=(int)rr;
|
| q=(p-rr)/m_mod;
|
| }
|
| int Squash(const int d)
|
| {
|
| if(d>2047)
|
| return 4095;
|
| if(d<-2047)
|
| return 1;
|
| int w=d&127;
|
| int i=(d>>7)+16;
|
| return (m_sq[i]*(128-w)+m_sq[i+1]*w+64)>>7;
|
| }
|
| int Clamp2047(const long v)
|
| {
|
| if(v>2047)
|
| return 2047;
|
| if(v<-2047)
|
| return -2047;
|
| return (int)v;
|
| }
|
| long DoubleBits(const double v)
|
| {
|
| UTkzBits u;
|
| u.d=v;
|
| return u.l;
|
| }
|
| double BitsDouble(const long v)
|
| {
|
| UTkzBits u;
|
| u.l=v;
|
| return u.d;
|
| }
|
| uint TickHash(uint h,const long t,const long b,const long a)
|
| {
|
| h=H2(h^(uint)t,(uint)(t>>32));
|
| h=H2(h^(uint)b,(uint)(b>>32));
|
| h=H2(h^(uint)a,(uint)(a>>32));
|
| return h;
|
| }
|
|
|
| //--- byte I/O (little endian)
|
| void PutU8(uchar &dst[],const int pos,const int v) { dst[pos]=(uchar)(v&255); }
|
| void PutU32(uchar &dst[],const int pos,const uint v)
|
| {
|
| dst[pos]=(uchar)(v&255);
|
| dst[pos+1]=(uchar)((v>>8)&255);
|
| dst[pos+2]=(uchar)((v>>16)&255);
|
| dst[pos+3]=(uchar)((v>>24)&255);
|
| }
|
| void PutI64(uchar &dst[],const int pos,const long v)
|
| {
|
| ulong u=(ulong)v;
|
| for(int i=0; i<8; i++)
|
| dst[pos+i]=(uchar)((u>>(8*i))&255);
|
| }
|
| uint GetU32(const uchar &src[],const int pos)
|
| {
|
| return (uint)src[pos]|((uint)src[pos+1]<<8)|((uint)src[pos+2]<<16)|((uint)src[pos+3]<<24);
|
| }
|
| long GetI64(const uchar &src[],const int pos)
|
| {
|
| ulong u=0;
|
| for(int i=7; i>=0; i--)
|
| u=(u<<8)|(ulong)src[pos+i];
|
| return (long)u;
|
| }
|
|
|
| //--- model initialisation
|
| bool InitModel(const int tbits)
|
| {
|
| int i,j;
|
| m_tbits=tbits;
|
| int tsize=1<<tbits;
|
| if(ArrayResize(m_t,tsize)!=tsize)
|
| {
|
| m_error="out of memory (counters)";
|
| return false;
|
| }
|
| ArrayInitialize(m_t,0);
|
| m_tmask=(uint)((tsize>>4)-1);
|
| if(ArrayResize(m_w1,TKZ_NSEL*TKZ_NW)!=TKZ_NSEL*TKZ_NW ||
|
| ArrayResize(m_w2,TKZ_W2SETS*TKZ_NW)!=TKZ_W2SETS*TKZ_NW ||
|
| ArrayResize(m_wf,TKZ_NCLASS*3)!=TKZ_NCLASS*3 ||
|
| ArrayResize(m_a1,TKZ_NSEL*33)!=TKZ_NSEL*33 ||
|
| ArrayResize(m_a2,TKZ_A2SETS*33)!=TKZ_A2SETS*33)
|
| {
|
| m_error="out of memory (mixers)";
|
| return false;
|
| }
|
| ArrayInitialize(m_w1,16384);
|
| ArrayInitialize(m_w2,16384);
|
| for(i=0; i<TKZ_NCLASS; i++)
|
| {
|
| m_wf[i*3]=32768;
|
| m_wf[i*3+1]=32768;
|
| m_wf[i*3+2]=0;
|
| }
|
| ushort row[33];
|
| for(j=0; j<33; j++)
|
| row[j]=(ushort)(Squash((j-16)*128)*16);
|
| for(i=0; i<TKZ_NSEL; i++)
|
| for(j=0; j<33; j++)
|
| m_a1[i*33+j]=row[j];
|
| for(i=0; i<TKZ_A2SETS; i++)
|
| for(j=0; j<33; j++)
|
| m_a2[i*33+j]=row[j];
|
| return true;
|
| }
|
| void ResetState(const long t0,const long b0,const long a0)
|
| {
|
| m_pt=t0;
|
| m_pb=b0;
|
| m_pa=a0;
|
| m_dt1=0;
|
| m_dt2=0;
|
| m_k1=0;
|
| m_k2=0;
|
| m_ty1=0;
|
| m_ty2=0;
|
| m_ty3=0;
|
| m_mb1=0;
|
| m_mb2=0;
|
| m_ma1=0;
|
| m_ma2=0;
|
| m_ms1=0;
|
| m_emak=0;
|
| m_emak2=0;
|
| m_emab=0;
|
| long qb=0,qa=0;
|
| int rb=0,ra=0;
|
| Split(b0,qb,rb);
|
| Split(a0,qa,ra);
|
| m_spavg16=(qa-qb)*16;
|
| m_nctx=0;
|
| m_sel2=0;
|
| }
|
|
|
| //--- arithmetic coder
|
| void PutByte(const uchar c)
|
| {
|
| if(m_bpos>=m_bcap)
|
| {
|
| m_bcap=m_bcap*2+65536;
|
| ArrayResize(m_buf,m_bcap);
|
| }
|
| m_buf[m_bpos]=c;
|
| m_bpos++;
|
| }
|
| uint GetByte(void)
|
| {
|
| if(m_bpos<m_bend)
|
| {
|
| uint c=m_buf[m_bpos];
|
| m_bpos++;
|
| return c;
|
| }
|
| m_overrun=true; // a valid stream is consumed exactly, never beyond its end
|
| return 0;
|
| }
|
|
|
| //--- context list
|
| void AddCtx(const uint h)
|
| {
|
| m_ctx[m_nctx]=h;
|
| m_nctx++;
|
| }
|
| void SetBlock(const int j,uint h)
|
| {
|
| h=H2(h,(uint)j);
|
| uint b=h&m_tmask;
|
| uint chk=(h>>8)|(uint)0x80000000;
|
| int b0=(int)(b<<4);
|
| int b1=(int)((b^1)<<4);
|
| int base;
|
| if(m_t[b0]==chk)
|
| base=b0;
|
| else
|
| if(m_t[b1]==chk)
|
| base=b1;
|
| else
|
| {
|
| uint n0=(m_t[b0]==0) ? (uint)0 : (m_t[b0+1]&1023);
|
| uint n1=(m_t[b1]==0) ? (uint)0 : (m_t[b1+1]&1023);
|
| base=(n0<=n1) ? b0 : b1;
|
| m_t[base]=chk;
|
| for(int i=1; i<16; i++)
|
| m_t[base+i]=(uint)0x80000000;
|
| }
|
| m_blk[j]=(uint)base;
|
| }
|
| void SetBlocks(const uint key)
|
| {
|
| for(int j=0; j<m_nctx; j++)
|
| SetBlock(j,H2(m_ctx[j],key));
|
| }
|
|
|
| //--- predict, code and update one binary decision
|
| int CodeBit(int bit,const uint node,const int sel,const uint apm2)
|
| {
|
| int nm=m_nctx;
|
| int j;
|
| for(j=0; j<nm; j++)
|
| {
|
| uint ix=m_blk[j]+node;
|
| m_ix[j]=ix;
|
| m_x[j]=m_str[(int)(m_t[(int)ix]>>20)];
|
| }
|
| m_x[nm]=256;
|
| int wb1=sel*TKZ_NW;
|
| int wb2=(int)(H2(m_sel2,(uint)sel)&0xFFFF)*TKZ_NW;
|
| long dot1=0;
|
| long dot2=0;
|
| for(j=0; j<=nm; j++)
|
| {
|
| dot1+=(long)m_x[j]*m_w1[wb1+j];
|
| dot2+=(long)m_x[j]*m_w2[wb2+j];
|
| }
|
| int d1=Clamp2047(dot1>>16);
|
| int d2=Clamp2047(dot2>>16);
|
| int wf=(sel>>4)*3;
|
| long df=(long)d1*m_wf[wf]+(long)d2*m_wf[wf+1]+(long)256*m_wf[wf+2];
|
| int d=Clamp2047(df>>16);
|
| int pfin=m_sqt[d+2047];
|
| //--- APM refinement
|
| int s=(d+2048)*32;
|
| int lo=s>>12;
|
| int wt=s&4095;
|
| int c1=sel*33+lo;
|
| int c2=(int)(apm2&0xFFFF)*33+lo;
|
| int p1=((int)m_a1[c1]*(4096-wt)+(int)m_a1[c1+1]*wt)>>16;
|
| int p2=((int)m_a2[c2]*(4096-wt)+(int)m_a2[c2+1]*wt)>>16;
|
| int pr=(pfin*2+p1+p2+2)>>2;
|
| if(pr<1)
|
| pr=1;
|
| if(pr>4095)
|
| pr=4095;
|
| //--- arithmetic coding, P(bit==1)=pr/4096
|
| uint xmid=m_x1+(uint)(((ulong)(m_x2-m_x1)*(ulong)pr)>>12);
|
| if(m_enc)
|
| {
|
| if(bit!=0)
|
| m_x2=xmid;
|
| else
|
| m_x1=xmid+1;
|
| while(((m_x1^m_x2)>>24)==0)
|
| {
|
| PutByte((uchar)(m_x2>>24));
|
| m_x1=m_x1<<8;
|
| m_x2=(m_x2<<8)|255;
|
| }
|
| }
|
| else
|
| {
|
| if(m_xv<=xmid)
|
| {
|
| bit=1;
|
| m_x2=xmid;
|
| }
|
| else
|
| {
|
| bit=0;
|
| m_x1=xmid+1;
|
| }
|
| while(((m_x1^m_x2)>>24)==0)
|
| {
|
| m_x1=m_x1<<8;
|
| m_x2=(m_x2<<8)|255;
|
| m_xv=(m_xv<<8)|GetByte();
|
| }
|
| }
|
| //--- update mixers
|
| int e1=((bit<<12)-m_sqt[d1+2047])*8;
|
| int e2=((bit<<12)-m_sqt[d2+2047])*8;
|
| for(j=0; j<=nm; j++)
|
| {
|
| m_w1[wb1+j]+=(m_x[j]*e1)>>14;
|
| m_w2[wb2+j]+=(m_x[j]*e2)>>14;
|
| }
|
| int ef=((bit<<12)-pfin)*6;
|
| m_wf[wf]+=(d1*ef)>>14;
|
| m_wf[wf+1]+=(d2*ef)>>14;
|
| m_wf[wf+2]+=(256*ef)>>14;
|
| //--- update counters
|
| int target=(bit!=0) ? 4194303 : 0;
|
| for(j=0; j<nm; j++)
|
| {
|
| int ix2=(int)m_ix[j];
|
| uint t=m_t[ix2];
|
| int n=(int)(t&1023);
|
| int p22=(int)(t>>10);
|
| p22+=(int)(((long)(target-p22)*m_rate[n])>>16);
|
| if(n<TKZ_LIMIT)
|
| n++;
|
| m_t[ix2]=((uint)p22<<10)|(uint)n;
|
| }
|
| //--- update APMs
|
| int g=(bit<<16)+(bit<<7)-bit-bit;
|
| int i1=c1+(wt>>11);
|
| int i2=c2+(wt>>11);
|
| m_a1[i1]=(ushort)((int)m_a1[i1]+((g-(int)m_a1[i1])>>7));
|
| m_a2[i2]=(ushort)((int)m_a2[i2]+((g-(int)m_a2[i2])>>7));
|
| return bit;
|
| }
|
|
|
| //--- code nb (1..4) bits of v as a binary tree segment
|
| uint CodeSeg(const uint v,const int nb,const uint key,const int cls,const uint apmsel)
|
| {
|
| SetBlocks(key);
|
| uint node=1;
|
| for(int i=nb-1; i>=0; i--)
|
| {
|
| int bit=(int)((v>>i)&1);
|
| bit=CodeBit(bit,node,cls*16+(int)node,H2(apmsel,node));
|
| node=node*2+(uint)bit;
|
| }
|
| return node-((uint)1<<nb);
|
| }
|
|
|
| //--- unsigned magnitude: bit length + mantissa; raw 64 bits for k>=30
|
| ulong CodeMag(const ulong v,const int grp)
|
| {
|
| int base=grp*80;
|
| int k=0;
|
| if(m_enc)
|
| k=BitLen(v);
|
| uint u=CodeSeg((uint)(k<15 ? k : 15),4,0x100,base,H2((uint)base,1));
|
| if(u==15)
|
| {
|
| uint u2=CodeSeg((uint)(k-15<15 ? k-15 : 15),4,0x200,base+1,H2((uint)base,2));
|
| if(u2==15)
|
| {
|
| ulong r=0;
|
| for(int i=15; i>=0; i--)
|
| {
|
| uint nib=CodeSeg((uint)((v>>(i*4))&15),4,(uint)(0x300+i),base+2,H2((uint)base,3));
|
| r=(r<<4)|(ulong)nib;
|
| }
|
| return r;
|
| }
|
| k=15+(int)u2;
|
| }
|
| else
|
| k=(int)u;
|
| if(k<=1)
|
| return (ulong)k;
|
| int nb=k-1;
|
| uint prefix=1;
|
| int si=0;
|
| int kq=k<15 ? k : 15;
|
| while(nb>0)
|
| {
|
| int take=nb>=4 ? 4 : nb;
|
| uint bits=(uint)((v>>(nb-take))&(ulong)((1<<take)-1));
|
| uint sk;
|
| if(si<3)
|
| sk=H3((uint)(0x400+k),(uint)si,prefix);
|
| else
|
| sk=H3((uint)(0x500+k),(uint)si,prefix&0xFFF);
|
| int cls=base+16+kq*4+(si<3 ? si : 3);
|
| bits=CodeSeg(bits,take,sk,cls,(uint)(sk*(uint)0x9E3779B1)+(uint)cls);
|
| prefix=(prefix<<take)|bits;
|
| nb-=take;
|
| si++;
|
| }
|
| return (ulong)prefix;
|
| }
|
|
|
| //--- signed move; zero allowed or excluded
|
| long CodeSigned(const long dq,const bool zero_ok,const int cls_z,const int cls_nz,const int grp)
|
| {
|
| int neg=0;
|
| if(zero_ok)
|
| {
|
| SetBlocks(0x21);
|
| uint apm=H2((uint)cls_z,9);
|
| int nz=(dq!=0) ? 1 : 0;
|
| nz=CodeBit(nz,1,cls_z*16+1,H2(apm,1));
|
| if(nz==0)
|
| return 0;
|
| neg=(dq<0) ? 1 : 0;
|
| neg=CodeBit(neg,3,cls_z*16+3,H2(apm,3));
|
| }
|
| else
|
| {
|
| SetBlocks(0x20);
|
| neg=(dq<0) ? 1 : 0;
|
| neg=CodeBit(neg,1,cls_nz*16+1,H2(H2((uint)cls_nz,10),1));
|
| }
|
| for(int j=0; j<m_nctx; j++)
|
| m_ctx[j]=H2(m_ctx[j],(uint)(neg!=0 ? 0x55 : 0xAA));
|
| ulong mag=0;
|
| if(m_enc)
|
| mag=(ulong)(dq<0 ? -dq : dq)-1;
|
| mag=CodeMag(mag,grp);
|
| long v=(long)mag+1;
|
| return neg!=0 ? -v : v;
|
| }
|
|
|
| int CodeFlag(const int flag,const int side)
|
| {
|
| m_nctx=0;
|
| AddCtx(H2(95,(uint)side));
|
| return (int)CodeSeg((uint)flag,1,0x40,TKZ_C_ESC,H2(96,(uint)side));
|
| }
|
| ulong CodeRaw64(const ulong v,const int side)
|
| {
|
| m_nctx=0;
|
| AddCtx(H2(97,(uint)side));
|
| ulong r=0;
|
| for(int i=15; i>=0; i--)
|
| {
|
| uint nib=CodeSeg((uint)((v>>(i*4))&15),4,(uint)(0x300+i),TKZ_C_RAW,H2(98,(uint)i));
|
| r=(r<<4)|(ulong)nib;
|
| }
|
| return r;
|
| }
|
|
|
| //--- code one tick (both directions share this path)
|
| void CodeTick(void)
|
| {
|
| //=== time
|
| long dtv=0;
|
| if(m_enc)
|
| dtv=(m_ct-m_pt)/m_tstep;
|
| int hour=(int)(((m_pt/3600000)%24+24)%24);
|
| uint dt1c=(m_dt1<(ulong)4096) ? (uint)m_dt1 : (uint)(4096+m_k1);
|
| uint dt2c=(m_dt2<(ulong)4096) ? (uint)m_dt2 : (uint)(4096+m_k2);
|
| long lastmv=((m_ty1&1)!=0) ? m_mb1 : m_ma1;
|
| m_sel2=H2((uint)(1000+m_k1),(uint)m_ty1);
|
| m_nctx=0;
|
| AddCtx(H2(1,0));
|
| AddCtx(H2(2,(uint)m_k1));
|
| AddCtx(H3(3,(uint)m_k1,(uint)m_k2));
|
| AddCtx(H2(4,dt1c));
|
| AddCtx(H3(5,(uint)m_ty1,(uint)m_k1));
|
| AddCtx(H2(6,(uint)(m_emak>>2)));
|
| AddCtx(H3(7,(uint)m_k1,dt2c));
|
| AddCtx(H3(8,(uint)(m_emak2>>3),(uint)hour));
|
| AddCtx(H4(91,QLog(lastmv),(uint)m_ty1,(uint)m_k1));
|
| dtv=(long)CodeMag((ulong)dtv,TKZ_G_DT);
|
| if(!m_enc)
|
| m_ct=m_pt+dtv*m_tstep;
|
| ulong m=(dtv<0) ? (ulong)0 : (ulong)dtv;
|
| int k=BitLen(m);
|
| //=== split current prices
|
| long qb=0,qa=0;
|
| int rb=0,ra=0;
|
| Split(m_pb,qb,rb);
|
| Split(m_pa,qa,ra);
|
| long sq=qa-qb;
|
| int qs=QSigned(sq*4-m_spavg16/4);
|
| int qsa=(sq<0) ? 0 : ((sq<30) ? (int)sq+1 : 31);
|
| uint mc=(m<(ulong)1024) ? (uint)m : (uint)(1024+k);
|
| //=== tick type: bid changed, ask changed, same move
|
| int cb=0,ca=0,same=0;
|
| if(m_enc)
|
| {
|
| cb=(m_cb!=m_pb) ? 1 : 0;
|
| ca=(m_ca!=m_pa) ? 1 : 0;
|
| if(cb==1 && ca==1 && (m_cb-m_pb)==(m_ca-m_pa))
|
| same=1;
|
| }
|
| m_sel2=H3(2000,(uint)k,(uint)qs);
|
| m_nctx=0;
|
| AddCtx(H2(10,0));
|
| AddCtx(H3(11,(uint)m_ty1,(uint)m_ty2));
|
| AddCtx(H4(12,(uint)m_ty1,(uint)m_ty2,(uint)m_ty3));
|
| AddCtx(H3(13,(uint)k,(uint)m_ty1));
|
| AddCtx(H3(14,(uint)qs,(uint)m_ty1));
|
| AddCtx(H3(15,mc,0));
|
| AddCtx(H4(16,(uint)rb,(uint)ra,(uint)m_ty1));
|
| AddCtx(H4(17,(uint)qsa,(uint)qs,(uint)k));
|
| AddCtx(H4(18,(uint)m_ty1,(uint)((m_mb1<0 ? 2 : 0)+(m_ma1<0 ? 1 : 0)),(uint)qsa));
|
| AddCtx(H4(19,(uint)m_ty1,(uint)m_ty2,(uint)qsa));
|
| SetBlocks(0x77);
|
| uint tapm=(uint)(m_ty1*64+qs);
|
| cb=CodeBit(cb,1,TKZ_C_TYPE*16+1,H3(41,1,tapm));
|
| uint node=(uint)(2+cb);
|
| ca=CodeBit(ca,node,TKZ_C_TYPE*16+(int)node,H3(41,node,tapm));
|
| if(cb==1 && ca==1)
|
| same=CodeBit(same,7,TKZ_C_TYPE*16+7,H3(41,7,tapm));
|
| else
|
| same=0;
|
| int ty=cb+ca*2+same;
|
| //=== escapes (only in chunks that contain irregular prices)
|
| int escb=0,esca=0;
|
| if(m_hasesc)
|
| {
|
| if(m_enc)
|
| {
|
| escb=m_cescb;
|
| esca=m_cesca;
|
| }
|
| escb=(cb==0) ? CodeFlag(escb,0) : 0;
|
| esca=(ca==0) ? CodeFlag(esca,1) : 0;
|
| if(escb!=0)
|
| m_crawb=(long)CodeRaw64((ulong)m_crawb,0);
|
| if(esca!=0)
|
| m_crawa=(long)CodeRaw64((ulong)m_crawa,1);
|
| }
|
| m_cescb=escb;
|
| m_cesca=esca;
|
| //=== new split states
|
| long nqb=qb,nqa=qa;
|
| int nrb=rb,nra=ra;
|
| long dqb=0,dqa=0;
|
| if(m_enc)
|
| {
|
| Split(m_cb,nqb,nrb);
|
| Split(m_ca,nqa,nra);
|
| dqb=nqb-qb;
|
| dqa=nqa-qa;
|
| }
|
| //=== bid move
|
| if(cb==1)
|
| {
|
| if(m_mod>1)
|
| {
|
| m_nctx=0;
|
| AddCtx(H3(20,(uint)rb,(uint)ty));
|
| AddCtx(H4(21,(uint)rb,(uint)ra,(uint)ty));
|
| AddCtx(H4(22,(uint)rb,(uint)ty,(uint)(m_mb1<0 ? 1 : 0)));
|
| AddCtx(H4(23,(uint)rb,(uint)m_ty1,(uint)ty));
|
| nrb=(int)CodeSeg((uint)nrb,4,0x10,TKZ_C_BRES,H2(50,(uint)rb));
|
| }
|
| m_sel2=H3((uint)(3000+ty),(uint)k,(uint)qs);
|
| uint rr=(uint)(nrb*16+rb);
|
| m_nctx=0;
|
| AddCtx(H3(30,(uint)ty,rr));
|
| AddCtx(H4(31,(uint)ty,rr,QLog(m_mb1)));
|
| AddCtx(H4(32,(uint)ty,QLog(m_mb1),QLog(m_ma1)));
|
| AddCtx(H4(33,(uint)ty,(uint)qs,rr));
|
| AddCtx(H4(34,(uint)ty,(uint)m_mb1,(uint)m_mb2));
|
| AddCtx(H4(35,(uint)ty,(uint)qs,(uint)k));
|
| AddCtx(H4(36,(uint)ty,(uint)(m_emab>>3),(uint)k));
|
| AddCtx(H4(37,(uint)ty,(uint)qsa,(uint)m_ty1));
|
| dqb=CodeSigned(dqb,(m_mod>1 && nrb!=rb),TKZ_C_BSGNZ,TKZ_C_BSGN,TKZ_G_BMAG);
|
| nqb=qb+dqb;
|
| if(!m_enc)
|
| m_cb=nqb*m_mod+nrb;
|
| }
|
| else
|
| {
|
| if(!m_enc)
|
| m_cb=m_pb;
|
| nqb=qb;
|
| nrb=rb;
|
| dqb=0;
|
| }
|
| //=== ask move
|
| if(ca==1 && same==0)
|
| {
|
| m_sel2=H3((uint)(4000+ty),(uint)k,(uint)qs);
|
| if(m_mod>1)
|
| {
|
| m_nctx=0;
|
| AddCtx(H3(60,(uint)ra,(uint)ty));
|
| AddCtx(H4(61,(uint)ra,(uint)nrb,(uint)ty));
|
| AddCtx(H4(62,(uint)ra,(uint)ty,(uint)(m_ma1<0 ? 1 : 0)));
|
| AddCtx(H4(63,(uint)ra,(uint)(rb*16+nrb),(uint)ty));
|
| nra=(int)CodeSeg((uint)nra,4,0x10,TKZ_C_ARES,H2(70,(uint)ra));
|
| }
|
| if(cb==0)
|
| {
|
| uint rr2=(uint)(nra*16+ra);
|
| m_nctx=0;
|
| AddCtx(H3(70,(uint)ty,rr2));
|
| AddCtx(H4(71,(uint)ty,rr2,QLog(m_ma1)));
|
| AddCtx(H4(72,(uint)ty,QLog(m_ma1),QLog(m_mb1)));
|
| AddCtx(H4(73,(uint)ty,(uint)qs,rr2));
|
| AddCtx(H4(74,(uint)ty,(uint)m_ma1,(uint)m_ma2));
|
| AddCtx(H4(75,(uint)ty,(uint)qs,(uint)k));
|
| AddCtx(H4(76,(uint)ty,(uint)(m_emab>>3),(uint)k));
|
| AddCtx(H4(77,(uint)ty,(uint)qsa,(uint)m_ty1));
|
| dqa=CodeSigned(dqa,(m_mod>1 && nra!=ra),TKZ_C_ASGNZ,TKZ_C_ASGN,TKZ_G_AMAG);
|
| }
|
| else
|
| {
|
| long dsq=dqa-dqb;
|
| bool zok=(m_mod>1 && (nra-ra)!=(nrb-rb));
|
| uint r3=(uint)((nra*16+ra)*16+nrb);
|
| m_nctx=0;
|
| AddCtx(H3(80,(uint)ty,r3));
|
| AddCtx(H4(81,(uint)ty,(uint)qs,QLog(dqb)));
|
| AddCtx(H4(82,(uint)ty,(uint)dqb,(uint)qs));
|
| AddCtx(H4(83,(uint)ty,(uint)qsa,QLog(dqb)));
|
| AddCtx(H4(84,(uint)ty,(uint)m_ms1,(uint)qs));
|
| AddCtx(H4(85,(uint)ty,(uint)qs,(uint)k));
|
| AddCtx(H4(86,(uint)ty,(uint)dqb,r3));
|
| AddCtx(H4(87,(uint)ty,(uint)qsa,(uint)m_ty1));
|
| dsq=CodeSigned(dsq,zok,TKZ_C_SSGNZ,TKZ_C_SSGN,TKZ_G_SMAG);
|
| dqa=dqb+dsq;
|
| m_ms1=dsq;
|
| }
|
| nqa=qa+dqa;
|
| if(!m_enc)
|
| m_ca=nqa*m_mod+nra;
|
| }
|
| else
|
| if(same==1)
|
| {
|
| if(!m_enc)
|
| m_ca=m_pa+(m_cb-m_pb);
|
| Split(m_ca,nqa,nra);
|
| dqa=nqa-qa;
|
| }
|
| else
|
| {
|
| if(!m_enc)
|
| m_ca=m_pa;
|
| nqa=qa;
|
| nra=ra;
|
| dqa=0;
|
| }
|
| //=== state update
|
| m_dt2=m_dt1;
|
| m_dt1=m;
|
| m_k2=m_k1;
|
| m_k1=k;
|
| m_ty3=m_ty2;
|
| m_ty2=m_ty1;
|
| m_ty1=ty;
|
| if(cb==1)
|
| {
|
| m_mb2=m_mb1;
|
| m_mb1=dqb;
|
| }
|
| if(ca==1)
|
| {
|
| m_ma2=m_ma1;
|
| m_ma1=dqa;
|
| }
|
| m_emak+=(k*16-m_emak)>>3;
|
| m_emak2+=(k*16-m_emak2)>>6;
|
| if(cb==1)
|
| m_emab+=(BitLen((ulong)(dqb<0 ? -dqb : dqb))*16-m_emab)>>4;
|
| m_pt=m_ct;
|
| m_pb=m_cb;
|
| m_pa=m_ca;
|
| m_spavg16+=((nqa-nqb)*16-m_spavg16)/16;
|
| }
|
|
|
| //--- price -> integer in 10^-digits units, exact round trip required
|
| bool ToInt(const double p,const double scale,long &v)
|
| {
|
| if(!MathIsValidNumber(p))
|
| return false;
|
| double x=p*scale;
|
| if(x>4.0e15 || x<-4.0e15)
|
| return false;
|
| v=(long)MathRound(x);
|
| double back=(double)v/scale;
|
| return DoubleBits(back)==DoubleBits(p);
|
| }
|
| int DetectDigits(const double &bid[],const double &ask[],const int start,const int n)
|
| {
|
| int best=-1;
|
| long bestfails=-1;
|
| for(int d=0; d<=TKZ_MAX_DIGITS; d++)
|
| {
|
| double scale=m_pow10[d];
|
| long fails=0;
|
| long v=0;
|
| for(int i=0; i<n; i++)
|
| {
|
| if(!ToInt(bid[start+i],scale,v))
|
| fails++;
|
| if(!ToInt(ask[start+i],scale,v))
|
| fails++;
|
| if(bestfails>=0 && fails>=bestfails)
|
| break;
|
| }
|
| if(bestfails<0 || fails<bestfails)
|
| {
|
| best=d;
|
| bestfails=fails;
|
| }
|
| if(fails==0)
|
| break;
|
| }
|
| return best<0 ? 0 : best;
|
| }
|
| int ChooseModulus(const int n)
|
| {
|
| if(n<256)
|
| return 1;
|
| long cnt[10];
|
| ArrayInitialize(cnt,0);
|
| long total=0;
|
| for(int i=0; i<n; i++)
|
| {
|
| if((m_wesc[i]&TKZ_ESC_BID)==0)
|
| {
|
| cnt[(int)FloorMod(m_wb[i],10)]++;
|
| total++;
|
| }
|
| if((m_wesc[i]&TKZ_ESC_ASK)==0)
|
| {
|
| cnt[(int)FloorMod(m_wa[i],10)]++;
|
| total++;
|
| }
|
| }
|
| if(total==0)
|
| return 1;
|
| double h=0.0;
|
| for(int r=0; r<10; r++)
|
| if(cnt[r]>0)
|
| {
|
| double p=(double)cnt[r]/(double)total;
|
| h-=p*MathLog(p)/MathLog(2.0);
|
| }
|
| return (h<2.0) ? 10 : 1;
|
| }
|
|
|
| public:
|
| CTkzCodec(void)
|
| {
|
| int i;
|
| m_tbits=0;
|
| m_tmask=0;
|
| m_nctx=0;
|
| m_sel2=0;
|
| m_enc=true;
|
| m_x1=0;
|
| m_x2=0;
|
| m_xv=0;
|
| m_bpos=0;
|
| m_bend=0;
|
| m_bcap=0;
|
| m_overrun=false;
|
| m_tstep=1;
|
| m_mod=1;
|
| m_hasesc=false;
|
| m_ocount=0;
|
| m_error="";
|
| m_ct=0;
|
| m_cb=0;
|
| m_ca=0;
|
| m_cescb=0;
|
| m_cesca=0;
|
| m_crawb=0;
|
| m_crawa=0;
|
| //--- logistic tables (integer only, platform independent)
|
| m_sq[0]=1; m_sq[1]=2; m_sq[2]=3; m_sq[3]=6; m_sq[4]=10;
|
| m_sq[5]=16; m_sq[6]=27; m_sq[7]=45; m_sq[8]=73; m_sq[9]=120;
|
| m_sq[10]=194; m_sq[11]=310; m_sq[12]=488; m_sq[13]=747; m_sq[14]=1101;
|
| m_sq[15]=1546; m_sq[16]=2047; m_sq[17]=2549; m_sq[18]=2994; m_sq[19]=3348;
|
| m_sq[20]=3607; m_sq[21]=3785; m_sq[22]=3901; m_sq[23]=3975; m_sq[24]=4022;
|
| m_sq[25]=4050; m_sq[26]=4068; m_sq[27]=4079; m_sq[28]=4085; m_sq[29]=4089;
|
| m_sq[30]=4092; m_sq[31]=4093; m_sq[32]=4094;
|
| ArrayResize(m_str,4096);
|
| ArrayResize(m_sqt,4095);
|
| ArrayResize(m_rate,1024);
|
| int pi=0;
|
| for(int x=-2047; x<=2047; x++)
|
| {
|
| int v=Squash(x);
|
| for(int jj=pi; jj<=v; jj++)
|
| m_str[jj]=x;
|
| pi=v+1;
|
| }
|
| for(i=pi; i<4096; i++)
|
| m_str[i]=2047;
|
| for(i=0; i<4095; i++)
|
| m_sqt[i]=Squash(i-2047);
|
| for(i=0; i<1024; i++)
|
| m_rate[i]=131072/(2*i+3);
|
| m_pow10[0]=1.0;
|
| for(i=1; i<=TKZ_MAX_DIGITS; i++)
|
| m_pow10[i]=m_pow10[i-1]*10.0;
|
| }
|
| ~CTkzCodec(void)
|
| {
|
| Release();
|
| }
|
|
|
| //--- free model memory (tables are re-allocated on the next call)
|
| void Release(void)
|
| {
|
| ArrayFree(m_t);
|
| ArrayFree(m_w1);
|
| ArrayFree(m_w2);
|
| ArrayFree(m_wf);
|
| ArrayFree(m_a1);
|
| ArrayFree(m_a2);
|
| ArrayFree(m_buf);
|
| m_bcap=0;
|
| ArrayFree(m_wb);
|
| ArrayFree(m_wa);
|
| ArrayFree(m_wesc);
|
| ArrayFree(m_it);
|
| ArrayFree(m_ib);
|
| ArrayFree(m_ia);
|
| ArrayFree(m_ot);
|
| ArrayFree(m_obid);
|
| ArrayFree(m_oask);
|
| ArrayFree(m_oflg);
|
| }
|
|
|
| string LastError(void) { return m_error; }
|
|
|
| //--- bitwise equality of two doubles (distinguishes -0.0, NaN payloads)
|
| bool SameBits(const double x,const double y) { return DoubleBits(x)==DoubleBits(y); }
|
|
|
| //+---------------------------------------------------------------+
|
| //| Encode ticks [start, start+count) from parallel arrays. |
|
| //| The chunk (header + payload) is appended to out[]. |
|
| //| Returns the chunk size in bytes or -1 on error. |
|
| //+---------------------------------------------------------------+
|
| int Encode(const long &time_msc[],const double &bid[],const double &ask[],
|
| const int start,const int count,uchar &out[])
|
| {
|
| m_error="";
|
| int n=count;
|
| if(n<0 || start<0 || start+n>ArraySize(time_msc) || start+n>ArraySize(bid) || start+n>ArraySize(ask))
|
| {
|
| m_error="bad arguments";
|
| return -1;
|
| }
|
| int i;
|
| //--- 1. decimal digits that make prices exact integers
|
| int digits=DetectDigits(bid,ask,start,n);
|
| double scale=m_pow10[digits];
|
| //--- 2. integer prices, irregular prices are escaped
|
| if(ArrayResize(m_wb,n+1)<0 || ArrayResize(m_wa,n+1)<0 || ArrayResize(m_wesc,n+1)<0)
|
| {
|
| m_error="out of memory";
|
| return -1;
|
| }
|
| bool hasesc=false;
|
| long bbase=0,abase=0;
|
| bool bfound=false,afound=false;
|
| long v=0;
|
| for(i=0; i<n; i++)
|
| {
|
| m_wesc[i]=0;
|
| m_wb[i]=0;
|
| m_wa[i]=0;
|
| if(ToInt(bid[start+i],scale,v))
|
| {
|
| m_wb[i]=v;
|
| if(!bfound)
|
| {
|
| bbase=v;
|
| bfound=true;
|
| }
|
| }
|
| else
|
| {
|
| m_wesc[i]=(uchar)(m_wesc[i]|TKZ_ESC_BID);
|
| hasesc=true;
|
| }
|
| if(ToInt(ask[start+i],scale,v))
|
| {
|
| m_wa[i]=v;
|
| if(!afound)
|
| {
|
| abase=v;
|
| afound=true;
|
| }
|
| }
|
| else
|
| {
|
| m_wesc[i]=(uchar)(m_wesc[i]|TKZ_ESC_ASK);
|
| hasesc=true;
|
| }
|
| }
|
| //--- 3. carry states over escapes, common price step
|
| long g=0;
|
| long pb=bbase,pa=abase;
|
| for(i=0; i<n; i++)
|
| {
|
| if((m_wesc[i]&TKZ_ESC_BID)!=0)
|
| m_wb[i]=pb;
|
| else
|
| {
|
| g=Gcd(g,AbsL(m_wb[i]-pb));
|
| pb=m_wb[i];
|
| }
|
| if((m_wesc[i]&TKZ_ESC_ASK)!=0)
|
| m_wa[i]=pa;
|
| else
|
| {
|
| g=Gcd(g,AbsL(m_wa[i]-pa));
|
| pa=m_wa[i];
|
| }
|
| }
|
| if(g<=0)
|
| g=1;
|
| long offb=FloorMod(bbase,g);
|
| long offa=FloorMod(abase,g);
|
| for(i=0; i<n; i++)
|
| {
|
| m_wb[i]=(m_wb[i]-offb)/g;
|
| m_wa[i]=(m_wa[i]-offa)/g;
|
| }
|
| //--- 4. common time step
|
| long tg=0;
|
| for(i=1; i<n; i++)
|
| tg=Gcd(tg,AbsL(time_msc[start+i]-time_msc[start+i-1]));
|
| if(tg<=0)
|
| tg=1;
|
| //--- 5. decimal sub-grid modulus
|
| int mod=ChooseModulus(n);
|
| //--- 6. model
|
| int tbits=22;
|
| if(n<=8192)
|
| tbits=18;
|
| else
|
| if(n<=65536)
|
| tbits=20;
|
| if(!InitModel(tbits))
|
| return -1;
|
| m_enc=true;
|
| m_tstep=tg;
|
| m_mod=mod;
|
| m_hasesc=hasesc;
|
| m_x1=0;
|
| m_x2=(uint)0xFFFFFFFF;
|
| m_bpos=0;
|
| long t0=(n>0) ? time_msc[start] : (long)0;
|
| ResetState(t0,(bbase-offb)/g,(abase-offa)/g);
|
| uint crc=0;
|
| long tmin=t0,tmax=t0;
|
| for(i=0; i<n; i++)
|
| {
|
| m_ct=time_msc[start+i];
|
| if(m_ct<tmin)
|
| tmin=m_ct;
|
| if(m_ct>tmax)
|
| tmax=m_ct;
|
| m_cb=m_wb[i];
|
| m_ca=m_wa[i];
|
| m_cescb=((m_wesc[i]&TKZ_ESC_BID)!=0) ? 1 : 0;
|
| m_cesca=((m_wesc[i]&TKZ_ESC_ASK)!=0) ? 1 : 0;
|
| long bbits=DoubleBits(bid[start+i]);
|
| long abits=DoubleBits(ask[start+i]);
|
| m_crawb=bbits;
|
| m_crawa=abits;
|
| crc=TickHash(crc,m_ct,bbits,abits);
|
| CodeTick();
|
| }
|
| //--- flush coder
|
| PutByte((uchar)(m_x1>>24));
|
| PutByte((uchar)((m_x1>>16)&255));
|
| PutByte((uchar)((m_x1>>8)&255));
|
| PutByte((uchar)(m_x1&255));
|
| int payload=m_bpos;
|
| //--- 7. header + payload
|
| int pos=ArraySize(out);
|
| int total=TKZ_HEADER_SIZE+payload;
|
| if(ArrayResize(out,pos+total)!=pos+total)
|
| {
|
| m_error="out of memory (output)";
|
| return -1;
|
| }
|
| PutU32(out,pos,(uint)TKZ_CHUNK_MAGIC);
|
| PutU8(out,pos+4,TKZ_FORMAT_VERSION);
|
| PutU8(out,pos+5,TKZ_MODEL_VERSION);
|
| PutU8(out,pos+6,tbits);
|
| PutU8(out,pos+7,digits);
|
| PutU8(out,pos+8,mod);
|
| PutU8(out,pos+9,hasesc ? 1 : 0);
|
| PutU8(out,pos+10,0);
|
| PutU8(out,pos+11,0);
|
| PutU32(out,pos+12,(uint)n);
|
| PutU32(out,pos+16,(uint)payload);
|
| PutU32(out,pos+20,crc);
|
| PutI64(out,pos+24,t0);
|
| PutI64(out,pos+32,(n>0) ? time_msc[start+n-1] : (long)0);
|
| PutI64(out,pos+40,tg);
|
| PutI64(out,pos+48,g);
|
| PutI64(out,pos+56,bbase);
|
| PutI64(out,pos+64,abase);
|
| PutI64(out,pos+72,tmin);
|
| PutI64(out,pos+80,tmax);
|
| ArrayCopy(out,m_buf,pos+TKZ_HEADER_SIZE,0,payload);
|
| return total;
|
| }
|
|
|
| //--- encode from MqlTick array (only time_msc, bid and ask are stored)
|
| int EncodeTicks(const MqlTick &ticks[],const int start,const int count,uchar &out[])
|
| {
|
| if(count<0 || start<0 || start+count>ArraySize(ticks))
|
| {
|
| m_error="bad arguments";
|
| return -1;
|
| }
|
| if(ArrayResize(m_it,count+1)<0 || ArrayResize(m_ib,count+1)<0 || ArrayResize(m_ia,count+1)<0)
|
| {
|
| m_error="out of memory";
|
| return -1;
|
| }
|
| for(int i=0; i<count; i++)
|
| {
|
| m_it[i]=ticks[start+i].time_msc;
|
| m_ib[i]=ticks[start+i].bid;
|
| m_ia[i]=ticks[start+i].ask;
|
| }
|
| return Encode(m_it,m_ib,m_ia,0,count,out);
|
| }
|
|
|
| //+---------------------------------------------------------------+
|
| //| Parse a chunk header at src[pos]. |
|
| //+---------------------------------------------------------------+
|
| bool ReadInfo(const uchar &src[],const int pos,STkzChunkInfo &info)
|
| {
|
| m_error="";
|
| if(pos<0 || pos+TKZ_HEADER_SIZE>ArraySize(src))
|
| {
|
| m_error="truncated header";
|
| return false;
|
| }
|
| if(GetU32(src,pos)!=(uint)TKZ_CHUNK_MAGIC)
|
| {
|
| m_error="bad chunk magic";
|
| return false;
|
| }
|
| if(src[pos+4]!=TKZ_FORMAT_VERSION || src[pos+5]!=TKZ_MODEL_VERSION)
|
| {
|
| m_error="unsupported chunk version";
|
| return false;
|
| }
|
| info.table_bits=src[pos+6];
|
| info.digits=src[pos+7];
|
| info.modulus=src[pos+8];
|
| info.flags=src[pos+9];
|
| uint cnt=GetU32(src,pos+12);
|
| uint psz=GetU32(src,pos+16);
|
| info.checksum=GetU32(src,pos+20);
|
| info.first_msc=GetI64(src,pos+24);
|
| info.last_msc=GetI64(src,pos+32);
|
| info.time_step=GetI64(src,pos+40);
|
| info.price_step=GetI64(src,pos+48);
|
| info.bid_base=GetI64(src,pos+56);
|
| info.ask_base=GetI64(src,pos+64);
|
| info.min_msc=GetI64(src,pos+72);
|
| info.max_msc=GetI64(src,pos+80);
|
| //--- every tick costs at least 6 decisions of >= 1/2840 bit, so a payload
|
| //--- byte can never hold more than ~3800 ticks: bound with a 2x margin
|
| if(cnt>(uint)0x7FFFFFF0 || psz>(uint)0x7FFFFFF0 || info.table_bits<16 || info.table_bits>24 ||
|
| info.digits>TKZ_MAX_DIGITS || info.modulus<1 || info.modulus>16 ||
|
| info.time_step<1 || info.price_step<1 || (long)cnt>((long)psz+16)*8192)
|
| {
|
| m_error="corrupt chunk header";
|
| return false;
|
| }
|
| info.tick_count=(int)cnt;
|
| info.payload_size=(int)psz;
|
| info.total_size=TKZ_HEADER_SIZE+(int)psz;
|
| return true;
|
| }
|
|
|
| //+---------------------------------------------------------------+
|
| //| Decode the chunk at src[pos] into the internal buffer. |
|
| //| Returns the tick count or -1 on error. |
|
| //+---------------------------------------------------------------+
|
| int DecodeChunk(const uchar &src[],const int pos)
|
| {
|
| STkzChunkInfo info;
|
| m_ocount=0;
|
| if(!ReadInfo(src,pos,info))
|
| return -1;
|
| if(pos+info.total_size>ArraySize(src))
|
| {
|
| m_error="truncated chunk";
|
| return -1;
|
| }
|
| int n=info.tick_count;
|
| if(ArrayResize(m_ot,n+1)<0 || ArrayResize(m_obid,n+1)<0 || ArrayResize(m_oask,n+1)<0 ||
|
| ArrayResize(m_oflg,n+1)<0)
|
| {
|
| m_error="out of memory";
|
| return -1;
|
| }
|
| if(!InitModel(info.table_bits))
|
| return -1;
|
| //--- payload
|
| if(m_bcap<info.payload_size)
|
| {
|
| m_bcap=info.payload_size;
|
| ArrayResize(m_buf,m_bcap);
|
| }
|
| if(info.payload_size>0)
|
| ArrayCopy(m_buf,src,0,pos+TKZ_HEADER_SIZE,info.payload_size);
|
| m_bpos=0;
|
| m_bend=info.payload_size;
|
| m_overrun=false;
|
| m_enc=false;
|
| m_x1=0;
|
| m_x2=(uint)0xFFFFFFFF;
|
| m_xv=0;
|
| for(int b=0; b<4; b++)
|
| m_xv=(m_xv<<8)|GetByte();
|
| //--- parameters
|
| m_tstep=info.time_step;
|
| m_mod=info.modulus;
|
| m_hasesc=(info.flags&1)!=0;
|
| long g=info.price_step;
|
| long offb=FloorMod(info.bid_base,g);
|
| long offa=FloorMod(info.ask_base,g);
|
| double scale=m_pow10[info.digits];
|
| ResetState(info.first_msc,(info.bid_base-offb)/g,(info.ask_base-offa)/g);
|
| uint crc=0;
|
| double pbid=0.0,pask=0.0;
|
| for(int i=0; i<n; i++)
|
| {
|
| CodeTick();
|
| if(m_overrun)
|
| {
|
| m_error="corrupt payload (overrun)";
|
| return -1;
|
| }
|
| double bv=0.0,av=0.0;
|
| if(m_cescb!=0)
|
| bv=BitsDouble(m_crawb);
|
| else
|
| bv=(double)(m_cb*g+offb)/scale;
|
| if(m_cesca!=0)
|
| av=BitsDouble(m_crawa);
|
| else
|
| av=(double)(m_ca*g+offa)/scale;
|
| long bbits=DoubleBits(bv);
|
| long abits=DoubleBits(av);
|
| crc=TickHash(crc,m_ct,bbits,abits);
|
| uchar fl=0;
|
| if(i==0 || bbits!=DoubleBits(pbid))
|
| fl=(uchar)(fl|TKZ_ESC_BID);
|
| if(i==0 || abits!=DoubleBits(pask))
|
| fl=(uchar)(fl|TKZ_ESC_ASK);
|
| m_ot[i]=m_ct;
|
| m_obid[i]=bv;
|
| m_oask[i]=av;
|
| m_oflg[i]=fl;
|
| pbid=bv;
|
| pask=av;
|
| }
|
| if(m_bpos!=m_bend)
|
| {
|
| m_error="corrupt payload (length)";
|
| return -1;
|
| }
|
| if(crc!=info.checksum)
|
| {
|
| m_error="checksum mismatch";
|
| return -1;
|
| }
|
| m_ocount=n;
|
| return n;
|
| }
|
|
|
| //--- decode into parallel arrays (resized to the tick count)
|
| int Decode(const uchar &src[],const int pos,long &time_msc[],double &bid[],double &ask[])
|
| {
|
| int n=DecodeChunk(src,pos);
|
| if(n<0)
|
| return -1;
|
| ArrayResize(time_msc,n);
|
| ArrayResize(bid,n);
|
| ArrayResize(ask,n);
|
| if(n>0)
|
| {
|
| ArrayCopy(time_msc,m_ot,0,0,n);
|
| ArrayCopy(bid,m_obid,0,0,n);
|
| ArrayCopy(ask,m_oask,0,0,n);
|
| }
|
| return n;
|
| }
|
|
|
| //--- decode into MqlTick array; ticks are appended when append=true
|
| int DecodeTicks(const uchar &src[],const int pos,MqlTick &ticks[],const bool append=false)
|
| {
|
| int n=DecodeChunk(src,pos);
|
| if(n<0)
|
| return -1;
|
| int base=append ? ArraySize(ticks) : 0;
|
| if(ArrayResize(ticks,base+n)!=base+n)
|
| {
|
| m_error="out of memory";
|
| return -1;
|
| }
|
| for(int i=0; i<n; i++)
|
| {
|
| long t=m_ot[i];
|
| ticks[base+i].time=(datetime)(t/1000);
|
| ticks[base+i].time_msc=t;
|
| ticks[base+i].bid=m_obid[i];
|
| ticks[base+i].ask=m_oask[i];
|
| ticks[base+i].last=0.0;
|
| ticks[base+i].volume=0;
|
| ticks[base+i].volume_real=0.0;
|
| uint fl=0;
|
| if((m_oflg[i]&TKZ_ESC_BID)!=0)
|
| fl=fl|(uint)TICK_FLAG_BID;
|
| if((m_oflg[i]&TKZ_ESC_ASK)!=0)
|
| fl=fl|(uint)TICK_FLAG_ASK;
|
| ticks[base+i].flags=fl;
|
| }
|
| return n;
|
| }
|
| };
|
|
|
| #endif // TKZ_CODEC_MQH
|
| //+------------------------------------------------------------------+
|
852010e8ba