mql5-tick-compression/tests/test_codec.cpp
Claude Claude 7b1a9c4271 Reject corrupt chunks early, add fuzz test
A valid payload is consumed exactly, so reading past its end or leaving
bytes unread now fails the decode immediately instead of producing
garbage until the checksum check. Headers with an impossible tick count
for their payload size or a table larger than 2^24 are rejected before
any allocation. The fuzz test mutates header and payload bytes, sizes
and lengths and requires every accepted chunk to decode to the original
ticks. The format and encoder output are unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 16:40:14 -08:00
490 lines
18 KiB
C++
// Test driver for the TKZ codec: the MQL5 source is translated to C++ by
// mql2cpp.py and compiled against mql5_shim.h, so this exercises the exact
// code that runs in MetaTrader 5.
//
// usage: test_codec [-chunk=N] [-quiet] file.ticks ...
// *.ticks files: little-endian records {int64 time_msc, double bid, double ask}
// Without files only the synthetic tests run.
#include "mql5_shim.h"
#include "TkzCodec.h"
#include "TkzArchive.h"
#include "TkzSelfTest.h"
#include <random>
static int g_failures = 0;
#define CHECK(cond, ...) \
do { \
if (!(cond)) { \
printf(" FAIL %s:%d: ", __FILE__, __LINE__); \
printf(__VA_ARGS__); \
printf("\n"); \
g_failures++; \
} \
} while (0)
static long bits_of(double d) {
long l;
memcpy(&l, &d, 8);
return l;
}
static double double_of(long l) {
double d;
memcpy(&d, &l, 8);
return d;
}
struct Series {
MqlArray<long> t;
MqlArray<double> b, a;
int size() const { return ArraySize(t); }
void push(long tt, double bb, double aa) {
int n = size();
ArrayResize(t, n + 1);
ArrayResize(b, n + 1);
ArrayResize(a, n + 1);
t[n] = tt;
b[n] = bb;
a[n] = aa;
}
};
static bool load_ticks(const char *path, Series &s) {
FILE *f = fopen(path, "rb");
if (!f) return false;
fseek(f, 0, SEEK_END);
long n = ftell(f) / 24;
fseek(f, 0, SEEK_SET);
ArrayResize(s.t, (int)n);
ArrayResize(s.b, (int)n);
ArrayResize(s.a, (int)n);
for (long i = 0; i < n; ++i) {
long t;
double b, a;
if (fread(&t, 8, 1, f) != 1 || fread(&b, 8, 1, f) != 1 || fread(&a, 8, 1, f) != 1) {
fclose(f);
return false;
}
s.t[i] = t;
s.b[i] = b;
s.a[i] = a;
}
fclose(f);
return true;
}
// encode s[start, start+count) into one chunk, decode it back and compare bitwise
static int roundtrip(CTkzCodec &enc, CTkzCodec &dec, const Series &s, int start, int count, const char *name,
MqlArray<uchar> *keep = nullptr) {
MqlArray<uchar> buf;
int sz = enc.Encode(s.t, s.b, s.a, start, count, buf);
CHECK(sz > 0, "%s: encode failed: %s", name, enc.LastError().c_str());
if (sz <= 0) return -1;
CHECK(sz == ArraySize(buf), "%s: size mismatch", name);
MqlArray<long> t;
MqlArray<double> b, a;
int n = dec.Decode(buf, 0, t, b, a);
CHECK(n == count, "%s: decode returned %d (expected %d): %s", name, n, count, dec.LastError().c_str());
if (n != count) return -1;
int bad = 0;
for (int i = 0; i < n && bad < 5; ++i) {
if (t[i] != s.t[start + i] || bits_of(b[i]) != bits_of(s.b[start + i]) || bits_of(a[i]) != bits_of(s.a[start + i])) {
printf(" mismatch %s at %d: got (%ld, %.17g, %.17g) expected (%ld, %.17g, %.17g)\n", name, i, t[i], b[i], a[i],
s.t[start + i], s.b[start + i], s.a[start + i]);
bad++;
}
}
CHECK(bad == 0, "%s: %d mismatching ticks", name, bad);
// MqlTick API
MqlArray<MqlTick> ticks;
int n2 = dec.DecodeTicks(buf, 0, ticks);
CHECK(n2 == count, "%s: DecodeTicks failed", name);
for (int i = 0; i < n2 && i < count; ++i) {
if (ticks[i].time_msc != s.t[start + i] || bits_of(ticks[i].bid) != bits_of(s.b[start + i]) ||
bits_of(ticks[i].ask) != bits_of(s.a[start + i]) || ticks[i].time != s.t[start + i] / 1000) {
CHECK(false, "%s: DecodeTicks mismatch at %d", name, i);
break;
}
}
if (keep) *keep = buf;
return sz;
}
// ------------------------------------------------------------------ synthetic data
static void synthetic_tests() {
printf("== synthetic edge cases\n");
CTkzCodec enc, dec;
std::mt19937_64 rng(12345);
const double nan = std::nan(""), inf = INFINITY;
{
Series s;
roundtrip(enc, dec, s, 0, 0, "empty");
}
{
Series s;
s.push(1700000000123L, 1.23456, 1.23467);
roundtrip(enc, dec, s, 0, 1, "single");
}
{ // constant prices, constant times
Series s;
for (int i = 0; i < 1000; ++i) s.push(1700000000000L, 1.1, 1.1);
roundtrip(enc, dec, s, 0, s.size(), "constant");
}
{ // random walk forex-like
Series s;
long t = 1700000000000L, b = 108000, sp = 2;
for (int i = 0; i < 200000; ++i) {
t += (long)(rng() % 3000);
b += (long)(rng() % 7) - 3;
sp = 1 + (long)(rng() % 4);
s.push(t, b / 1e5, (b + sp) / 1e5);
}
int sz = roundtrip(enc, dec, s, 0, s.size(), "randomwalk5");
printf(" randomwalk5: %.3f bytes/tick\n", (double)sz / s.size());
}
{ // negative time steps, huge gaps, duplicates
Series s;
long t = 1700000000000L;
for (int i = 0; i < 50000; ++i) {
int r = (int)(rng() % 100);
if (r == 0) t -= (long)(rng() % 100000);
else if (r == 1) t += (long)(rng() % 4000000000000L);
else if (r < 30) t += 0;
else t += (long)(rng() % 500);
s.push(t, 2000.0 + (double)(rng() % 1000) / 100.0, 2001.0 + (double)(rng() % 1000) / 100.0);
}
s.push(-5000000000000L, 1, 2);
s.push(9000000000000000000L, 1, 2);
s.push(-9000000000000000000L, 1, 2);
roundtrip(enc, dec, s, 0, s.size(), "time-anomalies");
}
{ // irregular doubles: NaN, Inf, -0, denormals, non-normalized, huge
Series s;
long t = 1700000000000L;
for (int i = 0; i < 20000; ++i) {
t += 100;
double b = 1.2345 + (double)(i % 50) * 1e-4;
double a = b + 0.0002;
int r = i % 97;
if (r == 3) b = nan;
if (r == 5) a = inf;
if (r == 7) b = -inf;
if (r == 11) a = -0.0;
if (r == 13) b = 4.9e-324;
if (r == 17) a = 0.1 + 0.2; // 0.30000000000000004
if (r == 19) b = 1e300;
if (r == 23) a = -1e300;
if (r == 29) b = double_of(bits_of(1.2345) + 1);
s.push(t, b, a);
}
int sz = roundtrip(enc, dec, s, 0, s.size(), "irregular-doubles");
printf(" irregular-doubles: %.3f bytes/tick\n", (double)sz / s.size());
}
{ // all prices irregular
Series s;
for (int i = 0; i < 3000; ++i) s.push(1700000000000L + i * 7, double_of((long)rng()), double_of((long)rng()));
roundtrip(enc, dec, s, 0, s.size(), "all-random-bits");
}
{ // tick size 0.25 at 2 digits with offset, second timestamps
Series s;
long t = 1700000000000L;
long q = 20000;
for (int i = 0; i < 100000; ++i) {
t += 1000 * (1 + (long)(rng() % 5));
q += (long)(rng() % 9) - 4;
double b = q * 0.25 + 0.01;
s.push(t, b, b + 0.5);
}
int sz = roundtrip(enc, dec, s, 0, s.size(), "grid-0.25");
printf(" grid-0.25 + 1s timestamps: %.3f bytes/tick\n", (double)sz / s.size());
}
{ // negative prices (oil 2020 style), zero ask, 12 digits, large values
Series s;
long t = 1700000000000L;
for (int i = 0; i < 30000; ++i) {
t += (long)(rng() % 200);
double b = -37.63 + (double)((long)(rng() % 2000) - 1000) / 100.0;
double a = (i % 10 == 0) ? 0.0 : b + 0.03;
s.push(t, b, a);
}
roundtrip(enc, dec, s, 0, s.size(), "negative-prices");
Series s2;
for (int i = 0; i < 5000; ++i) s2.push(1700000000000L + i, 0.000012345678 + i * 1e-12, 0.000012345699 + i * 1e-12);
roundtrip(enc, dec, s2, 0, s2.size(), "12-digits");
Series s3;
for (int i = 0; i < 5000; ++i) s3.push(1700000000000L + i, 3999999999.5 + i, 3999999999.5 + 2 * i);
roundtrip(enc, dec, s3, 0, s3.size(), "large-values");
Series s4;
for (int i = 0; i < 5000; ++i) s4.push(1700000000000L + i, (double)(rng() % 1000000000000L), (double)(rng() % 3));
roundtrip(enc, dec, s4, 0, s4.size(), "random-integers");
}
{ // determinism: same input -> identical bytes
Series s;
long t = 1700000000000L;
for (int i = 0; i < 30000; ++i) {
t += (long)(rng() % 300);
s.push(t, 1.1 + (double)(rng() % 100) / 1e5, 1.1002 + (double)(rng() % 100) / 1e5);
}
MqlArray<uchar> b1, b2;
roundtrip(enc, dec, s, 0, s.size(), "det-1", &b1);
CTkzCodec fresh;
fresh.Encode(s.t, s.b, s.a, 0, s.size(), b2);
CHECK(ArraySize(b1) == ArraySize(b2) && memcmp(b1.v.data(), b2.v.data(), (size_t)ArraySize(b1)) == 0,
"determinism: reused vs fresh codec output differs");
// corruption: flip bytes in the payload, decoder must fail cleanly (no crash)
int detected = 0, trials = 200;
for (int k = 0; k < trials; ++k) {
MqlArray<uchar> c = b1;
int pos = TKZ_HEADER_SIZE + (int)(rng() % (ulong)(ArraySize(c) - TKZ_HEADER_SIZE));
c[pos] = (uchar)(c[pos] ^ (1 + rng() % 255));
MqlArray<long> t2;
MqlArray<double> bb, aa;
if (dec.Decode(c, 0, t2, bb, aa) < 0) detected++;
}
printf(" corruption: %d/%d payload corruptions detected\n", detected, trials);
CHECK(detected >= trials - 2, "corruption detection too weak");
// truncated / bad header
MqlArray<uchar> c = b1;
ArrayResize(c, ArraySize(c) - 10);
MqlArray<long> t2;
MqlArray<double> bb, aa;
CHECK(dec.Decode(c, 0, t2, bb, aa) < 0, "truncated chunk accepted");
c = b1;
c[0] = 0;
CHECK(dec.Decode(c, 0, t2, bb, aa) < 0, "bad magic accepted");
}
}
// ------------------------------------------------------------------ fuzzing
// Random mutations of valid chunks (header and payload bytes, truncation,
// extension, random garbage) must be rejected or decoded, never crash.
static void fuzz_tests(int iterations) {
printf("== fuzzing (%d mutated chunks)\n", iterations);
std::mt19937_64 rng(777);
CTkzCodec enc, dec;
MqlArray<uchar> chunks[3];
Series orig[3];
{
Series s;
long t = 1700000000000L, b = 108000;
for (int i = 0; i < 3000; ++i) {
t += (long)(rng() % 900);
b += (long)(rng() % 5) - 2;
s.push(t, b / 1e5, (b + 1 + (long)(rng() % 3)) / 1e5);
}
enc.Encode(s.t, s.b, s.a, 0, s.size(), chunks[0]);
orig[0] = s;
Series s2; // with escapes and decimal sub-grid
for (int i = 0; i < 3000; ++i) {
double bb = 1900.005 + 0.01 * (double)(rng() % 50);
if (i % 50 == 7) bb = double_of((long)rng());
s2.push(t + i * 37, bb, bb + 0.3);
}
enc.Encode(s2.t, s2.b, s2.a, 0, s2.size(), chunks[1]);
orig[1] = s2;
Series s3;
s3.push(t, 1.0, 1.0);
enc.Encode(s3.t, s3.b, s3.a, 0, 1, chunks[2]);
orig[2] = s3;
}
int accepted = 0, rejected = 0, silent = 0;
ulong t0 = GetMicrosecondCount();
for (int it = 0; it < iterations; ++it) {
MqlArray<uchar> c = chunks[it % 3];
int n = ArraySize(c);
int kind = (int)(rng() % 6);
if (kind == 0) { // header byte
int pos = (int)(rng() % TKZ_HEADER_SIZE);
c[pos] = (uchar)(c[pos] ^ (1 + rng() % 255));
} else if (kind == 1) { // several payload bytes
for (int k = 0; k < 1 + (int)(rng() % 4); ++k) {
int pos = TKZ_HEADER_SIZE + (int)(rng() % (ulong)(n - TKZ_HEADER_SIZE));
c[pos] = (uchar)rng();
}
} else if (kind == 2) { // truncate
ArrayResize(c, (int)(rng() % (ulong)n));
} else if (kind == 3) { // extend with garbage
int extra = 1 + (int)(rng() % 64);
ArrayResize(c, n + extra);
for (int k = 0; k < extra; ++k) c[n + k] = (uchar)rng();
} else if (kind == 4) { // payload size field changed
uint psz = (uint)c[16] | ((uint)c[17] << 8) | ((uint)c[18] << 16) | ((uint)c[19] << 24);
psz = (uint)((long)psz + (long)(rng() % 9) - 4);
c[16] = (uchar)psz; c[17] = (uchar)(psz >> 8); c[18] = (uchar)(psz >> 16); c[19] = (uchar)(psz >> 24);
} else { // tick count field changed
uint cnt = (uint)rng() % 100000u;
c[12] = (uchar)cnt; c[13] = (uchar)(cnt >> 8); c[14] = (uchar)(cnt >> 16); c[15] = (uchar)(cnt >> 24);
}
MqlArray<long> t;
MqlArray<double> b, a;
int got = dec.Decode(c, 0, t, b, a);
if (got < 0) {
rejected++;
continue;
}
// an accepted chunk must decode to exactly the original ticks
accepted++;
const Series &o = orig[it % 3];
bool same = got == o.size();
for (int i = 0; same && i < got; ++i)
same = t[i] == o.t[i] && bits_of(b[i]) == bits_of(o.b[i]) && bits_of(a[i]) == bits_of(o.a[i]);
if (!same) silent++;
}
printf(" %d rejected, %d accepted with identical output (mutations of unused bytes), %.1f s\n", rejected,
accepted - silent, (double)(GetMicrosecondCount() - t0) / 1e6);
CHECK(silent == 0, "%d mutated chunks decoded to different ticks without an error", silent);
CHECK(rejected > iterations / 2, "too few mutations rejected");
}
// ------------------------------------------------------------------ archive
static void archive_tests(const Series &s) {
printf("== archive\n");
CTkzWriter w;
string fname = "test_archive.tkz";
CHECK(w.Open(fname, "TESTSYM", 5, false), "writer open: %s", w.LastError().c_str());
w.SetVerify(true);
int n = s.size();
int chunk = n / 7 + 1;
MqlArray<MqlTick> ticks;
ArrayResize(ticks, n);
for (int i = 0; i < n; ++i) {
memset(&ticks[i], 0, sizeof(MqlTick));
ticks[i].time_msc = s.t[i];
ticks[i].time = s.t[i] / 1000;
ticks[i].bid = s.b[i];
ticks[i].ask = s.a[i];
}
for (int st = 0; st < n; st += chunk) {
int c = n - st < chunk ? n - st : chunk;
CHECK(w.WriteTicks(ticks, st, c), "write chunk: %s", w.LastError().c_str());
}
w.Close();
{
CTkzWriter w2;
CHECK(w2.Open(fname, "IGNORED", 0, false, true), "append open: %s", w2.LastError().c_str());
w2.SetVerify(true);
CHECK(w2.Write(s.t, s.b, s.a, 0, 1000), "append write: %s", w2.LastError().c_str());
w2.Close();
}
CTkzReader r;
CHECK(r.Open(fname, false), "reader open: %s", r.LastError().c_str());
CHECK(r.ArchiveSymbol() == "TESTSYM", "symbol name %s", r.ArchiveSymbol().c_str());
CHECK(r.ArchiveDigits() == 5, "digits");
CHECK(r.ChunkCount() == (n + chunk - 1) / chunk + 1, "chunk count %d", r.ChunkCount());
CHECK(r.TickCount() == (long)n + 1000, "tick count %ld", r.TickCount());
MqlArray<MqlTick> all;
int got = r.ReadAll(all);
CHECK(got == n + 1000, "ReadAll %d", got);
int bad = 0;
for (int i = 0; i < got && i < n; ++i)
if (all[i].time_msc != s.t[i] || bits_of(all[i].bid) != bits_of(s.b[i]) || bits_of(all[i].ask) != bits_of(s.a[i])) bad++;
CHECK(bad == 0, "archive mismatches: %d", bad);
// range query
long from = s.t[n / 3], to = s.t[n / 2];
MqlArray<MqlTick> part;
int pn = r.ReadRange(from, to, part);
int expect = 0;
for (int i = 0; i < n; ++i)
if (s.t[i] >= from && s.t[i] < to) expect++;
for (int i = 0; i < 1000; ++i)
if (s.t[i] >= from && s.t[i] < to) expect++;
CHECK(pn == expect, "ReadRange returned %d expected %d", pn, expect);
r.Close();
FileDelete(fname);
}
int main(int argc, char **argv) {
int chunk = 0;
bool quiet = false;
int ai = 1;
for (; ai < argc && argv[ai][0] == '-'; ++ai) {
if (!strncmp(argv[ai], "-chunk=", 7)) chunk = atoi(argv[ai] + 7);
else if (!strcmp(argv[ai], "-quiet")) quiet = true;
}
synthetic_tests();
fuzz_tests(3000);
CTkzCodec enc, dec;
double tot_bytes = 0, tot_ticks = 0;
Series first;
bool have_first = false;
for (; ai < argc; ++ai) {
Series s;
if (!load_ticks(argv[ai], s)) {
printf("cannot load %s\n", argv[ai]);
g_failures++;
continue;
}
const char *nm = strrchr(argv[ai], '/');
nm = nm ? nm + 1 : argv[ai];
int n = s.size();
int csize = chunk > 0 ? chunk : n;
double bytes = 0, tenc = 0, tdec = 0;
for (int st = 0; st < n; st += csize) {
int c = n - st < csize ? n - st : csize;
MqlArray<uchar> buf;
ulong t0 = GetMicrosecondCount();
int sz = enc.Encode(s.t, s.b, s.a, st, c, buf);
ulong t1 = GetMicrosecondCount();
CHECK(sz > 0, "%s: encode failed", nm);
MqlArray<long> t;
MqlArray<double> b, a;
int dn = dec.Decode(buf, 0, t, b, a);
ulong t2 = GetMicrosecondCount();
CHECK(dn == c, "%s: decode failed: %s", nm, dec.LastError().c_str());
int bad = 0;
for (int i = 0; i < dn; ++i)
if (t[i] != s.t[st + i] || bits_of(b[i]) != bits_of(s.b[st + i]) || bits_of(a[i]) != bits_of(s.a[st + i])) bad++;
CHECK(bad == 0, "%s: %d mismatches in chunk at %d", nm, bad, st);
bytes += sz;
tenc += (double)(t1 - t0);
tdec += (double)(t2 - t1);
if (!quiet && st == 0) {
STkzChunkInfo info;
enc.ReadInfo(buf, 0, info);
printf(" %s: digits=%d mod=%d price_step=%ld time_step=%ld flags=%d\n", nm, info.digits, info.modulus,
info.price_step, info.time_step, info.flags);
}
}
printf("%-28s ticks=%8d %.4f bytes/tick ratio %.1fx (vs 24 B) enc %.0f t/s dec %.0f t/s\n", nm, n, bytes / n,
24.0 * n / bytes, n / (tenc / 1e6), n / (tdec / 1e6));
tot_bytes += bytes;
tot_ticks += n;
if (!have_first) {
Series cut;
int m = n < 300000 ? n : 300000;
for (int i = 0; i < m; ++i) cut.push(s.t[i], s.b[i], s.a[i]);
first = cut;
have_first = true;
}
}
if (tot_ticks > 0) printf("TOTAL %.0f ticks %.4f bytes/tick\n", tot_ticks, tot_bytes / tot_ticks);
if (have_first) archive_tests(first);
// the self-test shipped for MetaTrader (TkzSelfTest.mqh) must pass as well
{
printf("== MQL5 self-test (TkzSelfTest.mqh)\n");
CTkzSelfTest st;
st.RunGolden();
st.RunSynthetic();
if (have_first) {
MqlArray<MqlTick> ticks;
ArrayResize(ticks, first.size());
for (int i = 0; i < first.size(); ++i) {
memset(&ticks[i], 0, sizeof(MqlTick));
ticks[i].time_msc = first.t[i];
ticks[i].bid = first.b[i];
ticks[i].ask = first.a[i];
}
st.RunTicks(ticks, first.size(), "history sample");
}
st.RunArchive();
CHECK(st.Failed() == 0, "self-test reported %d failures", st.Failed());
CHECK(st.Passed() >= (have_first ? 16 : 14), "self-test passed only %d checks", st.Passed());
}
if (g_failures) {
printf("FAILURES: %d\n", g_failures);
return 1;
}
printf("ALL TESTS PASSED\n");
return 0;
}