mql5-tick-compression/tests/test_codec.cpp

// Test driver for the TKZ codec: the MQL5 source is translated to C++ by
// mql2cpp.py and compiled against mql5_shim.h, so this exercises the exact
// code that runs in MetaTrader 5.
//
// usage: test_codec [-chunk=N] [-quiet] file.ticks ...
//   *.ticks files: little-endian records {int64 time_msc, double bid, double ask}
// Without files only the synthetic tests run.
#include "mql5_shim.h"
#include "TkzCodec.h"
#include "TkzArchive.h"
#include "TkzSelfTest.h"

#include <random>

static int g_failures = 0;

#define CHECK(cond, ...)                                  \
  do {                                                    \
    if (!(cond)) {                                        \
      printf("  FAIL %s:%d: ", __FILE__, __LINE__);       \
      printf(__VA_ARGS__);                                \
      printf("\n");                                       \
      g_failures++;                                       \
    }                                                     \
  } while (0)

static long bits_of(double d) {
  long l;
  memcpy(&l, &d, 8);
  return l;
}
static double double_of(long l) {
  double d;
  memcpy(&d, &l, 8);
  return d;
}

struct Series {
  MqlArray<long> t;
  MqlArray<double> b, a;
  int size() const { return ArraySize(t); }
  void push(long tt, double bb, double aa) {
    int n = size();
    ArrayResize(t, n + 1);
    ArrayResize(b, n + 1);
    ArrayResize(a, n + 1);
    t[n] = tt;
    b[n] = bb;
    a[n] = aa;
  }
};

static bool load_ticks(const char *path, Series &s) {
  FILE *f = fopen(path, "rb");
  if (!f) return false;
  fseek(f, 0, SEEK_END);
  long n = ftell(f) / 24;
  fseek(f, 0, SEEK_SET);
  ArrayResize(s.t, (int)n);
  ArrayResize(s.b, (int)n);
  ArrayResize(s.a, (int)n);
  for (long i = 0; i < n; ++i) {
    long t;
    double b, a;
    if (fread(&t, 8, 1, f) != 1 || fread(&b, 8, 1, f) != 1 || fread(&a, 8, 1, f) != 1) {
      fclose(f);
      return false;
    }
    s.t[i] = t;
    s.b[i] = b;
    s.a[i] = a;
  }
  fclose(f);
  return true;
}

// encode s[start, start+count) into one chunk, decode it back and compare bitwise
static int roundtrip(CTkzCodec &enc, CTkzCodec &dec, const Series &s, int start, int count, const char *name,
                     MqlArray<uchar> *keep = nullptr) {
  MqlArray<uchar> buf;
  int sz = enc.Encode(s.t, s.b, s.a, start, count, buf);
  CHECK(sz > 0, "%s: encode failed: %s", name, enc.LastError().c_str());
  if (sz <= 0) return -1;
  CHECK(sz == ArraySize(buf), "%s: size mismatch", name);
  MqlArray<long> t;
  MqlArray<double> b, a;
  int n = dec.Decode(buf, 0, t, b, a);
  CHECK(n == count, "%s: decode returned %d (expected %d): %s", name, n, count, dec.LastError().c_str());
  if (n != count) return -1;
  int bad = 0;
  for (int i = 0; i < n && bad < 5; ++i) {
    if (t[i] != s.t[start + i] || bits_of(b[i]) != bits_of(s.b[start + i]) || bits_of(a[i]) != bits_of(s.a[start + i])) {
      printf("  mismatch %s at %d: got (%ld, %.17g, %.17g) expected (%ld, %.17g, %.17g)\n", name, i, t[i], b[i], a[i],
             s.t[start + i], s.b[start + i], s.a[start + i]);
      bad++;
    }
  }
  CHECK(bad == 0, "%s: %d mismatching ticks", name, bad);
  // MqlTick API
  MqlArray<MqlTick> ticks;
  int n2 = dec.DecodeTicks(buf, 0, ticks);
  CHECK(n2 == count, "%s: DecodeTicks failed", name);
  for (int i = 0; i < n2 && i < count; ++i) {
    if (ticks[i].time_msc != s.t[start + i] || bits_of(ticks[i].bid) != bits_of(s.b[start + i]) ||
        bits_of(ticks[i].ask) != bits_of(s.a[start + i]) || ticks[i].time != s.t[start + i] / 1000) {
      CHECK(false, "%s: DecodeTicks mismatch at %d", name, i);
      break;
    }
  }
  if (keep) *keep = buf;
  return sz;
}

// ------------------------------------------------------------------ synthetic data
static void synthetic_tests() {
  printf("== synthetic edge cases\n");
  CTkzCodec enc, dec;
  std::mt19937_64 rng(12345);
  const double nan = std::nan(""), inf = INFINITY;
  {
    Series s;
    roundtrip(enc, dec, s, 0, 0, "empty");
  }
  {
    Series s;
    s.push(1700000000123L, 1.23456, 1.23467);
    roundtrip(enc, dec, s, 0, 1, "single");
  }
  {  // constant prices, constant times
    Series s;
    for (int i = 0; i < 1000; ++i) s.push(1700000000000L, 1.1, 1.1);
    roundtrip(enc, dec, s, 0, s.size(), "constant");
  }
  {  // random walk forex-like
    Series s;
    long t = 1700000000000L, b = 108000, sp = 2;
    for (int i = 0; i < 200000; ++i) {
      t += (long)(rng() % 3000);
      b += (long)(rng() % 7) - 3;
      sp = 1 + (long)(rng() % 4);
      s.push(t, b / 1e5, (b + sp) / 1e5);
    }
    int sz = roundtrip(enc, dec, s, 0, s.size(), "randomwalk5");
    printf("  randomwalk5: %.3f bytes/tick\n", (double)sz / s.size());
  }
  {  // negative time steps, huge gaps, duplicates
    Series s;
    long t = 1700000000000L;
    for (int i = 0; i < 50000; ++i) {
      int r = (int)(rng() % 100);
      if (r == 0) t -= (long)(rng() % 100000);
      else if (r == 1) t += (long)(rng() % 4000000000000L);
      else if (r < 30) t += 0;
      else t += (long)(rng() % 500);
      s.push(t, 2000.0 + (double)(rng() % 1000) / 100.0, 2001.0 + (double)(rng() % 1000) / 100.0);
    }
    s.push(-5000000000000L, 1, 2);
    s.push(9000000000000000000L, 1, 2);
    s.push(-9000000000000000000L, 1, 2);
    roundtrip(enc, dec, s, 0, s.size(), "time-anomalies");
  }
  {  // irregular doubles: NaN, Inf, -0, denormals, non-normalized, huge
    Series s;
    long t = 1700000000000L;
    for (int i = 0; i < 20000; ++i) {
      t += 100;
      double b = 1.2345 + (double)(i % 50) * 1e-4;
      double a = b + 0.0002;
      int r = i % 97;
      if (r == 3) b = nan;
      if (r == 5) a = inf;
      if (r == 7) b = -inf;
      if (r == 11) a = -0.0;
      if (r == 13) b = 4.9e-324;
      if (r == 17) a = 0.1 + 0.2;  // 0.30000000000000004
      if (r == 19) b = 1e300;
      if (r == 23) a = -1e300;
      if (r == 29) b = double_of(bits_of(1.2345) + 1);
      s.push(t, b, a);
    }
    int sz = roundtrip(enc, dec, s, 0, s.size(), "irregular-doubles");
    printf("  irregular-doubles: %.3f bytes/tick\n", (double)sz / s.size());
  }
  {  // all prices irregular
    Series s;
    for (int i = 0; i < 3000; ++i) s.push(1700000000000L + i * 7, double_of((long)rng()), double_of((long)rng()));
    roundtrip(enc, dec, s, 0, s.size(), "all-random-bits");
  }
  {  // tick size 0.25 at 2 digits with offset, second timestamps
    Series s;
    long t = 1700000000000L;
    long q = 20000;
    for (int i = 0; i < 100000; ++i) {
      t += 1000 * (1 + (long)(rng() % 5));
      q += (long)(rng() % 9) - 4;
      double b = q * 0.25 + 0.01;
      s.push(t, b, b + 0.5);
    }
    int sz = roundtrip(enc, dec, s, 0, s.size(), "grid-0.25");
    printf("  grid-0.25 + 1s timestamps: %.3f bytes/tick\n", (double)sz / s.size());
  }
  {  // negative prices (oil 2020 style), zero ask, 12 digits, large values
    Series s;
    long t = 1700000000000L;
    for (int i = 0; i < 30000; ++i) {
      t += (long)(rng() % 200);
      double b = -37.63 + (double)((long)(rng() % 2000) - 1000) / 100.0;
      double a = (i % 10 == 0) ? 0.0 : b + 0.03;
      s.push(t, b, a);
    }
    roundtrip(enc, dec, s, 0, s.size(), "negative-prices");
    Series s2;
    for (int i = 0; i < 5000; ++i) s2.push(1700000000000L + i, 0.000012345678 + i * 1e-12, 0.000012345699 + i * 1e-12);
    roundtrip(enc, dec, s2, 0, s2.size(), "12-digits");
    Series s3;
    for (int i = 0; i < 5000; ++i) s3.push(1700000000000L + i, 3999999999.5 + i, 3999999999.5 + 2 * i);
    roundtrip(enc, dec, s3, 0, s3.size(), "large-values");
    Series s4;
    for (int i = 0; i < 5000; ++i) s4.push(1700000000000L + i, (double)(rng() % 1000000000000L), (double)(rng() % 3));
    roundtrip(enc, dec, s4, 0, s4.size(), "random-integers");
  }
  {  // determinism: same input -> identical bytes
    Series s;
    long t = 1700000000000L;
    for (int i = 0; i < 30000; ++i) {
      t += (long)(rng() % 300);
      s.push(t, 1.1 + (double)(rng() % 100) / 1e5, 1.1002 + (double)(rng() % 100) / 1e5);
    }
    MqlArray<uchar> b1, b2;
    roundtrip(enc, dec, s, 0, s.size(), "det-1", &b1);
    CTkzCodec fresh;
    fresh.Encode(s.t, s.b, s.a, 0, s.size(), b2);
    CHECK(ArraySize(b1) == ArraySize(b2) && memcmp(b1.v.data(), b2.v.data(), (size_t)ArraySize(b1)) == 0,
          "determinism: reused vs fresh codec output differs");
    // corruption: flip bytes in the payload, decoder must fail cleanly (no crash)
    int detected = 0, trials = 200;
    for (int k = 0; k < trials; ++k) {
      MqlArray<uchar> c = b1;
      int pos = TKZ_HEADER_SIZE + (int)(rng() % (ulong)(ArraySize(c) - TKZ_HEADER_SIZE));
      c[pos] = (uchar)(c[pos] ^ (1 + rng() % 255));
      MqlArray<long> t2;
      MqlArray<double> bb, aa;
      if (dec.Decode(c, 0, t2, bb, aa) < 0) detected++;
    }
    printf("  corruption: %d/%d payload corruptions detected\n", detected, trials);
    CHECK(detected >= trials - 2, "corruption detection too weak");
    // truncated / bad header
    MqlArray<uchar> c = b1;
    ArrayResize(c, ArraySize(c) - 10);
    MqlArray<long> t2;
    MqlArray<double> bb, aa;
    CHECK(dec.Decode(c, 0, t2, bb, aa) < 0, "truncated chunk accepted");
    c = b1;
    c[0] = 0;
    CHECK(dec.Decode(c, 0, t2, bb, aa) < 0, "bad magic accepted");
  }
}

// ------------------------------------------------------------------ fuzzing
// Random mutations of valid chunks (header and payload bytes, truncation,
// extension, random garbage) must be rejected or decoded, never crash.
static void fuzz_tests(int iterations) {
  printf("== fuzzing (%d mutated chunks)\n", iterations);
  std::mt19937_64 rng(777);
  CTkzCodec enc, dec;
  MqlArray<uchar> chunks[3];
  Series orig[3];
  {
    Series s;
    long t = 1700000000000L, b = 108000;
    for (int i = 0; i < 3000; ++i) {
      t += (long)(rng() % 900);
      b += (long)(rng() % 5) - 2;
      s.push(t, b / 1e5, (b + 1 + (long)(rng() % 3)) / 1e5);
    }
    enc.Encode(s.t, s.b, s.a, 0, s.size(), chunks[0]);
    orig[0] = s;
    Series s2;  // with escapes and decimal sub-grid
    for (int i = 0; i < 3000; ++i) {
      double bb = 1900.005 + 0.01 * (double)(rng() % 50);
      if (i % 50 == 7) bb = double_of((long)rng());
      s2.push(t + i * 37, bb, bb + 0.3);
    }
    enc.Encode(s2.t, s2.b, s2.a, 0, s2.size(), chunks[1]);
    orig[1] = s2;
    Series s3;
    s3.push(t, 1.0, 1.0);
    enc.Encode(s3.t, s3.b, s3.a, 0, 1, chunks[2]);
    orig[2] = s3;
  }
  int accepted = 0, rejected = 0, silent = 0;
  ulong t0 = GetMicrosecondCount();
  for (int it = 0; it < iterations; ++it) {
    MqlArray<uchar> c = chunks[it % 3];
    int n = ArraySize(c);
    int kind = (int)(rng() % 6);
    if (kind == 0) {  // header byte
      int pos = (int)(rng() % TKZ_HEADER_SIZE);
      c[pos] = (uchar)(c[pos] ^ (1 + rng() % 255));
    } else if (kind == 1) {  // several payload bytes
      for (int k = 0; k < 1 + (int)(rng() % 4); ++k) {
        int pos = TKZ_HEADER_SIZE + (int)(rng() % (ulong)(n - TKZ_HEADER_SIZE));
        c[pos] = (uchar)rng();
      }
    } else if (kind == 2) {  // truncate
      ArrayResize(c, (int)(rng() % (ulong)n));
    } else if (kind == 3) {  // extend with garbage
      int extra = 1 + (int)(rng() % 64);
      ArrayResize(c, n + extra);
      for (int k = 0; k < extra; ++k) c[n + k] = (uchar)rng();
    } else if (kind == 4) {  // payload size field changed
      uint psz = (uint)c[16] | ((uint)c[17] << 8) | ((uint)c[18] << 16) | ((uint)c[19] << 24);
      psz = (uint)((long)psz + (long)(rng() % 9) - 4);
      c[16] = (uchar)psz; c[17] = (uchar)(psz >> 8); c[18] = (uchar)(psz >> 16); c[19] = (uchar)(psz >> 24);
    } else {  // tick count field changed
      uint cnt = (uint)rng() % 100000u;
      c[12] = (uchar)cnt; c[13] = (uchar)(cnt >> 8); c[14] = (uchar)(cnt >> 16); c[15] = (uchar)(cnt >> 24);
    }
    MqlArray<long> t;
    MqlArray<double> b, a;
    int got = dec.Decode(c, 0, t, b, a);
    if (got < 0) {
      rejected++;
      continue;
    }
    // an accepted chunk must decode to exactly the original ticks
    accepted++;
    const Series &o = orig[it % 3];
    bool same = got == o.size();
    for (int i = 0; same && i < got; ++i)
      same = t[i] == o.t[i] && bits_of(b[i]) == bits_of(o.b[i]) && bits_of(a[i]) == bits_of(o.a[i]);
    if (!same) silent++;
  }
  printf("  %d rejected, %d accepted with identical output (mutations of unused bytes), %.1f s\n", rejected,
         accepted - silent, (double)(GetMicrosecondCount() - t0) / 1e6);
  CHECK(silent == 0, "%d mutated chunks decoded to different ticks without an error", silent);
  CHECK(rejected > iterations / 2, "too few mutations rejected");
}

// ------------------------------------------------------------------ archive
static void archive_tests(const Series &s) {
  printf("== archive\n");
  CTkzWriter w;
  string fname = "test_archive.tkz";
  CHECK(w.Open(fname, "TESTSYM", 5, false), "writer open: %s", w.LastError().c_str());
  w.SetVerify(true);
  int n = s.size();
  int chunk = n / 7 + 1;
  MqlArray<MqlTick> ticks;
  ArrayResize(ticks, n);
  for (int i = 0; i < n; ++i) {
    memset(&ticks[i], 0, sizeof(MqlTick));
    ticks[i].time_msc = s.t[i];
    ticks[i].time = s.t[i] / 1000;
    ticks[i].bid = s.b[i];
    ticks[i].ask = s.a[i];
  }
  for (int st = 0; st < n; st += chunk) {
    int c = n - st < chunk ? n - st : chunk;
    CHECK(w.WriteTicks(ticks, st, c), "write chunk: %s", w.LastError().c_str());
  }
  w.Close();
  {
    CTkzWriter w2;
    CHECK(w2.Open(fname, "IGNORED", 0, false, true), "append open: %s", w2.LastError().c_str());
    w2.SetVerify(true);
    CHECK(w2.Write(s.t, s.b, s.a, 0, 1000), "append write: %s", w2.LastError().c_str());
    w2.Close();
  }
  CTkzReader r;
  CHECK(r.Open(fname, false), "reader open: %s", r.LastError().c_str());
  CHECK(r.ArchiveSymbol() == "TESTSYM", "symbol name %s", r.ArchiveSymbol().c_str());
  CHECK(r.ArchiveDigits() == 5, "digits");
  CHECK(r.ChunkCount() == (n + chunk - 1) / chunk + 1, "chunk count %d", r.ChunkCount());
  CHECK(r.TickCount() == (long)n + 1000, "tick count %ld", r.TickCount());
  MqlArray<MqlTick> all;
  int got = r.ReadAll(all);
  CHECK(got == n + 1000, "ReadAll %d", got);
  int bad = 0;
  for (int i = 0; i < got && i < n; ++i)
    if (all[i].time_msc != s.t[i] || bits_of(all[i].bid) != bits_of(s.b[i]) || bits_of(all[i].ask) != bits_of(s.a[i])) bad++;
  CHECK(bad == 0, "archive mismatches: %d", bad);
  // range query
  long from = s.t[n / 3], to = s.t[n / 2];
  MqlArray<MqlTick> part;
  int pn = r.ReadRange(from, to, part);
  int expect = 0;
  for (int i = 0; i < n; ++i)
    if (s.t[i] >= from && s.t[i] < to) expect++;
  for (int i = 0; i < 1000; ++i)
    if (s.t[i] >= from && s.t[i] < to) expect++;
  CHECK(pn == expect, "ReadRange returned %d expected %d", pn, expect);
  r.Close();
  FileDelete(fname);
}

int main(int argc, char **argv) {
  int chunk = 0;
  bool quiet = false;
  int ai = 1;
  for (; ai < argc && argv[ai][0] == '-'; ++ai) {
    if (!strncmp(argv[ai], "-chunk=", 7)) chunk = atoi(argv[ai] + 7);
    else if (!strcmp(argv[ai], "-quiet")) quiet = true;
  }
  synthetic_tests();
  fuzz_tests(3000);
  CTkzCodec enc, dec;
  double tot_bytes = 0, tot_ticks = 0;
  Series first;
  bool have_first = false;
  for (; ai < argc; ++ai) {
    Series s;
    if (!load_ticks(argv[ai], s)) {
      printf("cannot load %s\n", argv[ai]);
      g_failures++;
      continue;
    }
    const char *nm = strrchr(argv[ai], '/');
    nm = nm ? nm + 1 : argv[ai];
    int n = s.size();
    int csize = chunk > 0 ? chunk : n;
    double bytes = 0, tenc = 0, tdec = 0;
    for (int st = 0; st < n; st += csize) {
      int c = n - st < csize ? n - st : csize;
      MqlArray<uchar> buf;
      ulong t0 = GetMicrosecondCount();
      int sz = enc.Encode(s.t, s.b, s.a, st, c, buf);
      ulong t1 = GetMicrosecondCount();
      CHECK(sz > 0, "%s: encode failed", nm);
      MqlArray<long> t;
      MqlArray<double> b, a;
      int dn = dec.Decode(buf, 0, t, b, a);
      ulong t2 = GetMicrosecondCount();
      CHECK(dn == c, "%s: decode failed: %s", nm, dec.LastError().c_str());
      int bad = 0;
      for (int i = 0; i < dn; ++i)
        if (t[i] != s.t[st + i] || bits_of(b[i]) != bits_of(s.b[st + i]) || bits_of(a[i]) != bits_of(s.a[st + i])) bad++;
      CHECK(bad == 0, "%s: %d mismatches in chunk at %d", nm, bad, st);
      bytes += sz;
      tenc += (double)(t1 - t0);
      tdec += (double)(t2 - t1);
      if (!quiet && st == 0) {
        STkzChunkInfo info;
        enc.ReadInfo(buf, 0, info);
        printf("  %s: digits=%d mod=%d price_step=%ld time_step=%ld flags=%d\n", nm, info.digits, info.modulus,
               info.price_step, info.time_step, info.flags);
      }
    }
    printf("%-28s ticks=%8d  %.4f bytes/tick  ratio %.1fx (vs 24 B)  enc %.0f t/s  dec %.0f t/s\n", nm, n, bytes / n,
           24.0 * n / bytes, n / (tenc / 1e6), n / (tdec / 1e6));
    tot_bytes += bytes;
    tot_ticks += n;
    if (!have_first) {
      Series cut;
      int m = n < 300000 ? n : 300000;
      for (int i = 0; i < m; ++i) cut.push(s.t[i], s.b[i], s.a[i]);
      first = cut;
      have_first = true;
    }
  }
  if (tot_ticks > 0) printf("TOTAL %.0f ticks  %.4f bytes/tick\n", tot_ticks, tot_bytes / tot_ticks);
  if (have_first) archive_tests(first);
  // the self-test shipped for MetaTrader (TkzSelfTest.mqh) must pass as well
  {
    printf("== MQL5 self-test (TkzSelfTest.mqh)\n");
    CTkzSelfTest st;
    st.RunGolden();
    st.RunSynthetic();
    if (have_first) {
      MqlArray<MqlTick> ticks;
      ArrayResize(ticks, first.size());
      for (int i = 0; i < first.size(); ++i) {
        memset(&ticks[i], 0, sizeof(MqlTick));
        ticks[i].time_msc = first.t[i];
        ticks[i].bid = first.b[i];
        ticks[i].ask = first.a[i];
      }
      st.RunTicks(ticks, first.size(), "history sample");
    }
    st.RunArchive();
    CHECK(st.Failed() == 0, "self-test reported %d failures", st.Failed());
    CHECK(st.Passed() >= (have_first ? 16 : 14), "self-test passed only %d checks", st.Passed());
  }
  if (g_failures) {
    printf("FAILURES: %d\n", g_failures);
    return 1;
  }
  printf("ALL TESTS PASSED\n");
  return 0;
}