| //+------------------------------------------------------------------+
|
| //| TkzArchive.mqh |
|
| //| .tkz archive files: a 64-byte file header followed by TKZ1 |
|
| //| chunks (see TkzCodec.mqh and docs/FORMAT.md). |
|
| //+------------------------------------------------------------------+
|
| #ifndef TKZ_ARCHIVE_MQH
|
| #define TKZ_ARCHIVE_MQH
|
|
|
| #include "TkzCodec.mqh"
|
|
|
| #define TKZ_FILE_MAGIC 0x465A4B54 // bytes "TKZF"
|
| #define TKZ_FILE_VERSION 1
|
| #define TKZ_FILE_HEADER_SIZE 64
|
| #define TKZ_SYMBOL_BYTES 32
|
|
|
| //+------------------------------------------------------------------+
|
| //| Little-endian helpers shared by writer and reader |
|
| //+------------------------------------------------------------------+
|
| class CTkzBytes
|
| {
|
| public:
|
| void PutU16(uchar &dst[],const int pos,const int v)
|
| {
|
| dst[pos]=(uchar)(v&255);
|
| dst[pos+1]=(uchar)((v>>8)&255);
|
| }
|
| void PutU32(uchar &dst[],const int pos,const uint v)
|
| {
|
| dst[pos]=(uchar)(v&255);
|
| dst[pos+1]=(uchar)((v>>8)&255);
|
| dst[pos+2]=(uchar)((v>>16)&255);
|
| dst[pos+3]=(uchar)((v>>24)&255);
|
| }
|
| int GetU16(const uchar &src[],const int pos)
|
| {
|
| return (int)src[pos]|((int)src[pos+1]<<8);
|
| }
|
| uint GetU32(const uchar &src[],const int pos)
|
| {
|
| return (uint)src[pos]|((uint)src[pos+1]<<8)|((uint)src[pos+2]<<16)|((uint)src[pos+3]<<24);
|
| }
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Writer: creates (or appends to) a .tkz file |
|
| //+------------------------------------------------------------------+
|
| class CTkzWriter
|
| {
|
| private:
|
| int m_h;
|
| CTkzCodec m_codec;
|
| CTkzCodec m_check;
|
| CTkzBytes m_io;
|
| uchar m_buf[];
|
| string m_error;
|
| long m_ticks;
|
| long m_bytes;
|
| bool m_verify;
|
| ulong m_enc_us;
|
| ulong m_ver_us;
|
| long m_vt[];
|
| double m_vb[];
|
| double m_va[];
|
|
|
| public:
|
| CTkzWriter(void)
|
| {
|
| m_h=INVALID_HANDLE;
|
| m_error="";
|
| m_ticks=0;
|
| m_bytes=0;
|
| m_verify=false;
|
| m_enc_us=0;
|
| m_ver_us=0;
|
| }
|
| ~CTkzWriter(void)
|
| {
|
| Close();
|
| }
|
|
|
| string LastError(void) { return m_error; }
|
| long TicksWritten(void) { return m_ticks; }
|
| long BytesWritten(void) { return m_bytes; }
|
| ulong EncodeMicroseconds(void) { return m_enc_us; }
|
| ulong VerifyMicroseconds(void) { return m_ver_us; }
|
| //--- decode every written chunk again and compare it with the input
|
| void SetVerify(const bool verify) { m_verify=verify; }
|
|
|
| //--- create a new file (or append chunks to an existing one when append=true)
|
| bool Open(const string filename,const string symbol,const int digits,const bool common,const bool append=false)
|
| {
|
| Close();
|
| m_error="";
|
| m_ticks=0;
|
| m_bytes=0;
|
| int flags=FILE_BIN|FILE_WRITE;
|
| if(common)
|
| flags|=FILE_COMMON;
|
| if(append && FileIsExist(filename,common ? FILE_COMMON : 0))
|
| {
|
| m_h=FileOpen(filename,flags|FILE_READ);
|
| if(m_h==INVALID_HANDLE)
|
| {
|
| m_error="cannot open file for append";
|
| return false;
|
| }
|
| uchar old[];
|
| ArrayResize(old,TKZ_FILE_HEADER_SIZE);
|
| if((int)FileReadArray(m_h,old,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE ||
|
| m_io.GetU32(old,0)!=(uint)TKZ_FILE_MAGIC)
|
| {
|
| m_error="existing file is not a TKZ archive";
|
| FileClose(m_h);
|
| m_h=INVALID_HANDLE;
|
| return false;
|
| }
|
| FileSeek(m_h,0,SEEK_END);
|
| m_bytes=(long)FileTell(m_h);
|
| return true;
|
| }
|
| m_h=FileOpen(filename,flags);
|
| if(m_h==INVALID_HANDLE)
|
| {
|
| m_error="cannot create file";
|
| return false;
|
| }
|
| uchar hdr[];
|
| ArrayResize(hdr,TKZ_FILE_HEADER_SIZE);
|
| ArrayInitialize(hdr,0);
|
| m_io.PutU32(hdr,0,(uint)TKZ_FILE_MAGIC);
|
| m_io.PutU16(hdr,4,TKZ_FILE_VERSION);
|
| m_io.PutU16(hdr,6,TKZ_FILE_HEADER_SIZE);
|
| uchar name[];
|
| int len=StringToCharArray(symbol,name,0,-1,CP_UTF8);
|
| for(int i=0; i<len && i<TKZ_SYMBOL_BYTES-1; i++)
|
| hdr[8+i]=name[i];
|
| m_io.PutU32(hdr,40,(uint)digits);
|
| if((int)FileWriteArray(m_h,hdr,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE)
|
| {
|
| m_error="cannot write file header";
|
| Close();
|
| return false;
|
| }
|
| m_bytes=TKZ_FILE_HEADER_SIZE;
|
| return true;
|
| }
|
|
|
| //--- compress ticks[start..start+count) into one chunk and append it
|
| bool WriteTicks(const MqlTick &ticks[],const int start,const int count)
|
| {
|
| if(m_h==INVALID_HANDLE)
|
| {
|
| m_error="file is not open";
|
| return false;
|
| }
|
| ArrayResize(m_buf,0);
|
| ulong t0=GetMicrosecondCount();
|
| int sz=m_codec.EncodeTicks(ticks,start,count,m_buf);
|
| m_enc_us+=GetMicrosecondCount()-t0;
|
| if(sz<0)
|
| {
|
| m_error=m_codec.LastError();
|
| return false;
|
| }
|
| if(m_verify)
|
| {
|
| t0=GetMicrosecondCount();
|
| int n=m_check.Decode(m_buf,0,m_vt,m_vb,m_va);
|
| m_ver_us+=GetMicrosecondCount()-t0;
|
| if(n!=count)
|
| {
|
| m_error="verification failed: "+m_check.LastError();
|
| return false;
|
| }
|
| for(int i=0; i<n; i++)
|
| if(m_vt[i]!=ticks[start+i].time_msc || !m_check.SameBits(m_vb[i],ticks[start+i].bid) ||
|
| !m_check.SameBits(m_va[i],ticks[start+i].ask))
|
| {
|
| m_error="verification failed at tick "+IntegerToString(start+i);
|
| return false;
|
| }
|
| }
|
| return Flush(sz,count);
|
| }
|
| bool Write(const long &time_msc[],const double &bid[],const double &ask[],const int start,const int count)
|
| {
|
| if(m_h==INVALID_HANDLE)
|
| {
|
| m_error="file is not open";
|
| return false;
|
| }
|
| ArrayResize(m_buf,0);
|
| ulong t0=GetMicrosecondCount();
|
| int sz=m_codec.Encode(time_msc,bid,ask,start,count,m_buf);
|
| m_enc_us+=GetMicrosecondCount()-t0;
|
| if(sz<0)
|
| {
|
| m_error=m_codec.LastError();
|
| return false;
|
| }
|
| if(m_verify)
|
| {
|
| t0=GetMicrosecondCount();
|
| int n=m_check.Decode(m_buf,0,m_vt,m_vb,m_va);
|
| m_ver_us+=GetMicrosecondCount()-t0;
|
| if(n!=count)
|
| {
|
| m_error="verification failed: "+m_check.LastError();
|
| return false;
|
| }
|
| for(int i=0; i<n; i++)
|
| if(m_vt[i]!=time_msc[start+i] || !m_check.SameBits(m_vb[i],bid[start+i]) ||
|
| !m_check.SameBits(m_va[i],ask[start+i]))
|
| {
|
| m_error="verification failed at tick "+IntegerToString(start+i);
|
| return false;
|
| }
|
| }
|
| return Flush(sz,count);
|
| }
|
| void Close(void)
|
| {
|
| if(m_h!=INVALID_HANDLE)
|
| {
|
| FileClose(m_h);
|
| m_h=INVALID_HANDLE;
|
| }
|
| m_codec.Release();
|
| m_check.Release();
|
| ArrayFree(m_buf);
|
| ArrayFree(m_vt);
|
| ArrayFree(m_vb);
|
| ArrayFree(m_va);
|
| }
|
|
|
| private:
|
| bool Flush(const int sz,const int count)
|
| {
|
| if((int)FileWriteArray(m_h,m_buf,0,sz)!=sz)
|
| {
|
| m_error="file write failed";
|
| return false;
|
| }
|
| FileFlush(m_h);
|
| m_ticks+=count;
|
| m_bytes+=sz;
|
| return true;
|
| }
|
| };
|
|
|
| //+------------------------------------------------------------------+
|
| //| Reader: chunk index, random access by chunk or by time range |
|
| //+------------------------------------------------------------------+
|
| class CTkzReader
|
| {
|
| private:
|
| int m_h;
|
| CTkzCodec m_codec;
|
| CTkzBytes m_io;
|
| uchar m_buf[];
|
| string m_error;
|
| string m_symbol;
|
| int m_digits;
|
| int m_count;
|
| long m_offset[];
|
| STkzChunkInfo m_info[];
|
|
|
| public:
|
| CTkzReader(void)
|
| {
|
| m_h=INVALID_HANDLE;
|
| m_error="";
|
| m_symbol="";
|
| m_digits=-1;
|
| m_count=0;
|
| }
|
| ~CTkzReader(void)
|
| {
|
| Close();
|
| }
|
|
|
| string LastError(void) { return m_error; }
|
| string ArchiveSymbol(void) { return m_symbol; }
|
| int ArchiveDigits(void) { return m_digits; }
|
| int ChunkCount(void) { return m_count; }
|
|
|
| bool Open(const string filename,const bool common)
|
| {
|
| Close();
|
| m_error="";
|
| int flags=FILE_BIN|FILE_READ|FILE_SHARE_READ;
|
| if(common)
|
| flags|=FILE_COMMON;
|
| m_h=FileOpen(filename,flags);
|
| if(m_h==INVALID_HANDLE)
|
| {
|
| m_error="cannot open file";
|
| return false;
|
| }
|
| long fsize=(long)FileSize(m_h);
|
| uchar hdr[];
|
| ArrayResize(hdr,TKZ_FILE_HEADER_SIZE);
|
| if(fsize<TKZ_FILE_HEADER_SIZE || (int)FileReadArray(m_h,hdr,0,TKZ_FILE_HEADER_SIZE)!=TKZ_FILE_HEADER_SIZE ||
|
| m_io.GetU32(hdr,0)!=(uint)TKZ_FILE_MAGIC)
|
| {
|
| m_error="not a TKZ archive";
|
| Close();
|
| return false;
|
| }
|
| if(m_io.GetU16(hdr,4)!=TKZ_FILE_VERSION)
|
| {
|
| m_error="unsupported archive version";
|
| Close();
|
| return false;
|
| }
|
| int hsize=m_io.GetU16(hdr,6);
|
| m_symbol=CharArrayToString(hdr,8,TKZ_SYMBOL_BYTES,CP_UTF8);
|
| m_digits=(int)m_io.GetU32(hdr,40);
|
| //--- scan chunk headers
|
| long pos=hsize;
|
| uchar ch[];
|
| ArrayResize(ch,TKZ_HEADER_SIZE);
|
| while(pos+TKZ_HEADER_SIZE<=fsize)
|
| {
|
| if(!FileSeek(m_h,pos,SEEK_SET) || (int)FileReadArray(m_h,ch,0,TKZ_HEADER_SIZE)!=TKZ_HEADER_SIZE)
|
| {
|
| m_error="read error";
|
| break;
|
| }
|
| STkzChunkInfo info;
|
| if(!m_codec.ReadInfo(ch,0,info))
|
| {
|
| m_error="bad chunk at offset "+IntegerToString(pos)+": "+m_codec.LastError();
|
| break;
|
| }
|
| if(pos+info.total_size>fsize)
|
| {
|
| m_error="truncated last chunk";
|
| break;
|
| }
|
| ArrayResize(m_offset,m_count+1,256);
|
| ArrayResize(m_info,m_count+1,256);
|
| m_offset[m_count]=pos;
|
| m_info[m_count]=info;
|
| m_count++;
|
| pos+=info.total_size;
|
| }
|
| return true;
|
| }
|
|
|
| void Close(void)
|
| {
|
| if(m_h!=INVALID_HANDLE)
|
| {
|
| FileClose(m_h);
|
| m_h=INVALID_HANDLE;
|
| }
|
| m_count=0;
|
| ArrayFree(m_offset);
|
| ArrayFree(m_info);
|
| ArrayFree(m_buf);
|
| m_codec.Release();
|
| }
|
|
|
| bool GetChunkInfo(const int i,STkzChunkInfo &info)
|
| {
|
| if(i<0 || i>=m_count)
|
| return false;
|
| info=m_info[i];
|
| return true;
|
| }
|
|
|
| long TickCount(void)
|
| {
|
| long n=0;
|
| for(int i=0; i<m_count; i++)
|
| n+=m_info[i].tick_count;
|
| return n;
|
| }
|
|
|
| //--- decode one chunk; ticks are appended when append=true
|
| int ReadChunk(const int i,MqlTick &ticks[],const bool append=false)
|
| {
|
| if(i<0 || i>=m_count || m_h==INVALID_HANDLE)
|
| {
|
| m_error="bad chunk index";
|
| return -1;
|
| }
|
| int sz=m_info[i].total_size;
|
| ArrayResize(m_buf,sz);
|
| if(!FileSeek(m_h,m_offset[i],SEEK_SET) || (int)FileReadArray(m_h,m_buf,0,sz)!=sz)
|
| {
|
| m_error="read error";
|
| return -1;
|
| }
|
| int n=m_codec.DecodeTicks(m_buf,0,ticks,append);
|
| if(n<0)
|
| m_error=m_codec.LastError();
|
| return n;
|
| }
|
|
|
| //--- all ticks of the archive
|
| int ReadAll(MqlTick &ticks[])
|
| {
|
| ArrayResize(ticks,0);
|
| for(int i=0; i<m_count; i++)
|
| if(ReadChunk(i,ticks,true)<0)
|
| return -1;
|
| return ArraySize(ticks);
|
| }
|
|
|
| //--- ticks with from_msc <= time_msc < to_msc
|
| int ReadRange(const long from_msc,const long to_msc,MqlTick &ticks[])
|
| {
|
| ArrayResize(ticks,0);
|
| MqlTick tmp[];
|
| int total=0;
|
| for(int i=0; i<m_count; i++)
|
| {
|
| if(m_info[i].tick_count==0 || m_info[i].max_msc<from_msc || m_info[i].min_msc>=to_msc)
|
| continue;
|
| int n=ReadChunk(i,tmp,false);
|
| if(n<0)
|
| return -1;
|
| ArrayResize(ticks,total+n,n);
|
| for(int k=0; k<n; k++)
|
| {
|
| if(tmp[k].time_msc>=from_msc && tmp[k].time_msc<to_msc)
|
| {
|
| ticks[total]=tmp[k];
|
| total++;
|
| }
|
| }
|
| ArrayResize(ticks,total);
|
| }
|
| return total;
|
| }
|
| };
|
|
|
| #endif // TKZ_ARCHIVE_MQH
|
| //+------------------------------------------------------------------+
|
852010e8ba