mql5-tick-compression/tests/test_scripts.cpp

// End-to-end test of the MetaTrader scripts (scripts/*.mq5) against an
// emulated terminal whose tick history is loaded from a .ticks file.
//
// usage: test_scripts file.ticks [symbol digits]
#define MQL5_TERMINAL_SIM
#include "mql5_shim.h"
#include "TkzCodec.h"
#include "TkzArchive.h"
#include "TkzSelfTest.h"

namespace s_export {
#include "script_TkzExport.h"
}
namespace s_verify {
#include "script_TkzVerify.h"
}
namespace s_custom {
#include "script_TkzToCustom.h"
}
namespace s_selftest {
#include "script_TkzSelfTest.h"
}

static int g_failures = 0;
#define CHECK(cond, ...)                            \
  do {                                              \
    if (!(cond)) {                                  \
      printf("  FAIL %s:%d: ", __FILE__, __LINE__); \
      printf(__VA_ARGS__);                          \
      printf("\n");                                 \
      g_failures++;                                 \
    }                                               \
  } while (0)

static long bits_of(double d) {
  long l;
  memcpy(&l, &d, 8);
  return l;
}
static bool same_tick(const MqlTick &a, const MqlTick &b) {
  return a.time_msc == b.time_msc && bits_of(a.bid) == bits_of(b.bid) && bits_of(a.ask) == bits_of(b.ask);
}
static bool log_has(const char *text) { return MqlLog().find(text) != std::string::npos; }

// ticks of the emulated history with from <= time < to (seconds)
static std::vector<MqlTick> history(datetime from, datetime to) {
  std::vector<MqlTick> r;
  for (auto &t : MqlSim().ticks)
    if (t.time_msc >= from * 1000 && t.time_msc < to * 1000) r.push_back(t);
  return r;
}

// calendar period of a tick time: 0 = day, 1 = week starting Sunday, 2 = month (UTC/server time)
static long period_of(long msc, int mode) {
  long day = (msc / 1000) / 86400;
  if (mode == 0) return day;
  if (mode == 1) return (day + 4) / 7;  // 1970-01-01 was a Thursday
  time_t tt = (time_t)(msc / 1000);
  struct tm g;
  gmtime_r(&tt, &g);
  return (long)g.tm_year * 12 + g.tm_mon;
}

static void check_archive(const char *file, const std::vector<MqlTick> &expect, int expect_chunks, const char *what,
                          int mode) {
  CTkzReader r;
  CHECK(r.Open(file, false), "%s: cannot open archive", what);
  MqlArray<MqlTick> all;
  int n = r.ReadAll(all);
  CHECK(n == (int)expect.size(), "%s: %d ticks in archive, expected %zu", what, n, expect.size());
  int bad = 0;
  for (int i = 0; i < n && i < (int)expect.size(); ++i)
    if (!same_tick(all[i], expect[(size_t)i])) bad++;
  CHECK(bad == 0, "%s: %d ticks differ", what, bad);
  if (expect_chunks >= 0) CHECK(r.ChunkCount() == expect_chunks, "%s: %d chunks, expected %d", what, r.ChunkCount(), expect_chunks);
  // every chunk must stay inside one calendar period, consecutive chunks in different ones
  long prev = -1;
  for (int i = 0; i < r.ChunkCount(); ++i) {
    STkzChunkInfo info;
    r.GetChunkInfo(i, info);
    CHECK(info.min_msc <= info.max_msc, "%s: chunk %d time range", what, i);
    long p0 = period_of(info.min_msc, mode), p1 = period_of(info.max_msc, mode);
    CHECK(p0 == p1, "%s: chunk %d spans two calendar periods", what, i);
    CHECK(p0 > prev, "%s: chunk %d is not in a new period", what, i);
    prev = p1;
  }
  printf("  %s: %d ticks, %d chunks, archive OK\n", what, n, r.ChunkCount());
  r.Close();
}

int main(int argc, char **argv) {
  if (argc < 2) {
    printf("usage: test_scripts file.ticks [symbol digits]\n");
    return 2;
  }
  auto &sim = MqlSim();
  if (argc >= 4) {
    sim.symbol = argv[2];
    sim.digits = atoi(argv[3]);
  }
  FILE *f = fopen(argv[1], "rb");
  if (!f) {
    printf("cannot open %s\n", argv[1]);
    return 2;
  }
  long t;
  double b, a;
  while (fread(&t, 8, 1, f) == 1 && fread(&b, 8, 1, f) == 1 && fread(&a, 8, 1, f) == 1) {
    MqlTick k;
    memset(&k, 0, sizeof(k));
    k.time_msc = t;
    k.time = t / 1000;
    k.bid = b;
    k.ask = a;
    k.flags = TICK_FLAG_BID | TICK_FLAG_ASK;
    sim.ticks.push_back(k);
  }
  fclose(f);
  printf("== scripts on %s (%zu ticks, %s)\n", argv[1], sim.ticks.size(), sim.symbol.c_str());
  MqlQuiet() = true;
  datetime first = sim.ticks.front().time, last = sim.ticks.back().time;
  // a range that starts and ends in the middle of days (and of weeks for long histories)
  datetime span = last - first;
  datetime from = span > 10 * 86400 ? first + 3 * 86400 + 5 * 3600 + 17 : first + span / 4 + 17;
  datetime to = span > 10 * 86400 ? last - 4 * 86400 - 3 * 3600 : last - span / 4;

  // --- export, weekly chunks, verification on
  s_export::InpSymbol = "";
  s_export::InpFrom = from;
  s_export::InpTo = to;
  s_export::InpChunk = s_export::TKZ_CHUNK_WEEK;
  s_export::InpFile = "";
  s_export::InpVerify = true;
  MqlLog().clear();
  s_export::OnStart();
  CHECK(log_has("TKZ: exported"), "export did not finish: %s", MqlLog().c_str());
  string file = sim.symbol + ".tkz";
  // weeks touched by [from, to): boundaries are Sundays 00:00
  int weeks = 0;
  for (datetime c = from; c < to;) {
    datetime nb = s_export::NextBoundary(c, s_export::TKZ_CHUNK_WEEK);
    if (!history(c, nb < to ? nb : to).empty()) weeks++;
    c = nb;
  }
  check_archive(file.c_str(), history(from, to), weeks, "export week", 1);
  printf("  export log: %s", MqlLog().c_str());

  // --- verify script on that archive
  s_verify::InpFile = "";
  s_verify::InpCompare = true;
  MqlLog().clear();
  s_verify::OnStart();
  CHECK(log_has(StringFormat("TKZ: %d chunks identical to the history, 0 chunks with problems", weeks).c_str()),
        "verify: %s", MqlLog().c_str());

  // --- monthly and daily chunks, whole history, verification off
  s_export::InpFrom = first;
  s_export::InpTo = last + 1;
  s_export::InpChunk = s_export::TKZ_CHUNK_MONTH;
  s_export::InpVerify = false;
  s_export::InpFile = "month.tkz";
  MqlLog().clear();
  s_export::OnStart();
  check_archive("month.tkz", history(first, last + 1), -1, "export month", 2);
  s_export::InpChunk = s_export::TKZ_CHUNK_DAY;
  s_export::InpFile = "day.tkz";
  MqlLog().clear();
  s_export::OnStart();
  int days = 0;
  for (datetime c = first - first % 86400; c <= last; c += 86400)
    if (!history(c, c + 86400).empty()) days++;
  check_archive("day.tkz", history(first, last + 1), days, "export day", 0);

  // --- custom symbol import
  s_custom::InpFile = "day.tkz";
  s_custom::InpCustom = "";
  MqlLog().clear();
  s_custom::OnStart();
  string custom = sim.symbol + ".tkz";
  CHECK(sim.custom.count(custom) == 1, "custom symbol not created: %s", MqlLog().c_str());
  {
    auto &ct = sim.custom[custom];
    auto ref = history(first, last + 1);
    bool ok = ct.size() == ref.size();
    for (size_t i = 0; ok && i < ct.size(); ++i) ok = same_tick(ct[i], ref[i]);
    CHECK(ok, "custom symbol ticks differ (%zu vs %zu)", ct.size(), ref.size());
    CHECK(sim.custom_digits[custom] == sim.digits, "custom digits %d", sim.custom_digits[custom]);
    printf("  custom symbol %s: %zu ticks OK\n", custom.c_str(), ct.size());
  }
  // importing again replaces, it must not duplicate
  s_custom::OnStart();
  CHECK(sim.custom[custom].size() == history(first, last + 1).size(), "re-import duplicated ticks");

  // --- self-test script
  s_selftest::InpHistoryTicks = 300000;
  MqlLog().clear();
  s_selftest::OnStart();
  CHECK(log_has("passed, 0 failed"), "self-test: %s", MqlLog().c_str());
  CHECK(!log_has("FAIL"), "self-test reported a failure");
  {
    size_t p = MqlLog().find("TKZ self-test finished");
    printf("  %s", p == std::string::npos ? "self-test summary missing\n" : MqlLog().substr(p).c_str());
  }

  for (const char *fn : {"EURUSD.tkz", "month.tkz", "day.tkz"}) FileDelete(fn);
  FileDelete(file);
  if (g_failures) {
    printf("SCRIPT TEST FAILURES: %d\n", g_failures);
    return 1;
  }
  printf("ALL SCRIPT TESTS PASSED\n");
  return 0;
}