Tick_Forge/TF_Engine.mqh

//+------------------------------------------------------------------+
//| TF_Engine.mqh - Tick Forge data generator                         |
//| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE).     |
//|                                                                   |
//| Reads the broker's own history for a source symbol, shifts the    |
//| raw timestamps, maps every price through a user function, and     |
//| writes the result into a custom symbol.                           |
//+------------------------------------------------------------------+
#property strict
#include "TF_Expr.mqh"

void TF_Status(const string s);          // implemented by the host program (logs + updates the GUI)

enum ENUM_TF_MODE
  {
   TF_MODE_TICKS = 0,    // Real ticks (transformed) + M1 bars rebuilt from them
   TF_MODE_BARS  = 1     // M1 bars only (tester synthesizes the ticks)
  };

struct CTFJob
  {
   string            src;            // source symbol (broker data)
   string            name;           // custom symbol to write
   datetime          from;           // first day (00:00)
   datetime          to;             // last day (inclusive)
   int               mode;           // ENUM_TF_MODE
   long              offset_sec;     // raw timestamp shift, seconds (may be negative)
   string            expr;           // price function
   bool              scale_spread;   // spread follows the price factor
   bool              auto_digits;    // add digits when the price level shrinks
  };

struct CTFResult
  {
   bool              ok;
   string            err;
   long              ticks;
   long              bars;
   int               digits;
   double            p0;
  };

//--- helpers -------------------------------------------------------
// Sleep() is not allowed in indicators; a short busy wait is enough to let a history download answer.
void TF_Wait(const uint ms)
  {
   const uint t = GetTickCount();
   while(GetTickCount() - t < ms) { }
  }
bool TF_IsFinite(const double v) { return MathIsValidNumber(v); }

bool TF_SymbolExists(const string name, bool &custom)
  {
   custom = false;
   for(int i = 0; i < SymbolsTotal(false); i++)
      if(SymbolName(i, false) == name)
        {
         custom = (bool)SymbolInfoInteger(name, SYMBOL_CUSTOM);
         return true;
        }
   return false;
  }

string TF_DataPath(const string name)
  {
   return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\history\\" + name;
  }
string TF_TickPath(const string name)
  {
   return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\ticks\\" + name;
  }

//--- the engine ----------------------------------------------------
class CTFEngine
  {
private:
   CTFJob            m_job;
   CTFExpr           m_ex;
   double            m_p0;
   double            m_span;         // seconds in the range
   datetime          m_from;
   int               m_digits_src, m_digits_dst;
   double            m_point_dst;
   long              m_bad;
   int               m_i, m_days;    // stepwise run: next day index, number of days
   uint              m_ready_at;     // GetTickCount() before which no write is attempted

   double            Map(const double price, const datetime t);
   double            PointDst() const { return m_point_dst; }
   bool              Prepare(CTFResult &res);
   bool              WriteTicksDay(const datetime day, CTFResult &res);
   bool              WriteBarsChunk(const datetime a, const datetime b, CTFResult &res);
   void              FlushMinute(MqlRates &cur, MqlRates &out[], int &n);
   bool              SetupSymbol(CTFResult &res);

public:
                     CTFEngine() : m_bad(0), m_i(0), m_days(0), m_ready_at(0), m_p0(0), m_span(0), m_from(0), m_digits_src(0), m_digits_dst(0), m_point_dst(0) {}
   bool              Preview(const CTFJob &job, double &first, double &last, double &mid, string &err);
   bool              Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err);
   bool              Begin(const CTFJob &job, CTFResult &res);   // validate, create the symbol; then call Step() until Done()
   bool              Step(CTFResult &res);                        // write one day (returns false on error)
   bool              Done() const { return m_i >= m_days; }
   int               DayIndex() const { return m_i; }
   int               Days() const { return m_days; }
   bool              Run(const CTFJob &job, CTFResult &res);      // all steps at once (for tools; blocks)
  };

double CTFEngine::Map(const double price, const datetime t)
  {
   double tt = (m_span > 0.0 ? (double)(t - m_from) / m_span : 0.0);
   if(tt < 0.0) tt = 0.0;
   if(tt > 1.0) tt = 1.0;
   double v = m_ex.Eval(price, m_p0, tt, (double)(t - m_from) / 86400.0);
   if(!TF_IsFinite(v) || v <= 0.0) { m_bad++; return 0.0; }
   return v;
  }

//--- first M1 open on/after the range start ---------------------------
bool TF_FirstOpen(const string sym, const datetime from, const datetime to, double &p0, datetime &t0)
  {
   MqlRates r[];
   for(int attempt = 0; attempt < 4; attempt++)
     {
      int n = CopyRates(sym, PERIOD_M1, from, MathMin(from + 7 * 86400, to + 86400), r);
      if(n > 0) { p0 = r[0].open; t0 = r[0].time; return true; }
      TF_Wait(250);
     }
   return false;
  }

bool CTFEngine::Preview(const CTFJob &job, double &first, double &last, double &mid, string &err)
  {
   m_job = job; m_bad = 0; err = "";
   if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; }
   datetime t0;
   if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; }
   m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
   MqlRates r[];
   int n = CopyRates(job.src, PERIOD_M1, job.to, 1, r);       // 1 bar at/ before range end
   double pl = (n > 0 ? r[0].close : m_p0);
   first = Map(m_p0, job.from);
   mid   = Map((m_p0 + pl) / 2.0, job.from + (datetime)(m_span / 2));
   last  = Map(pl, job.to + 86400);
   if(m_bad > 0) { err = "function gives zero/negative/invalid prices"; return false; }
   return true;
  }

//--- overview series: whole range, source vs result, timeframe chosen from the span ---
bool CTFEngine::Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err)
  {
   m_job = job; m_bad = 0; err = "";
   if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; }
   datetime t0;
   if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; }
   m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
   double days = m_span / 86400.0;
   ENUM_TIMEFRAMES tf = (days <= 3 ? PERIOD_M5 : days <= 14 ? PERIOD_M30 : days <= 120 ? PERIOD_H1 : days <= 700 ? PERIOD_H4 : PERIOD_D1);
   tfname = StringSubstr(EnumToString(tf), 7);
   MqlRates r[];
   int n = -1;
   for(int a = 0; a < 3 && n <= 0; a++)
     {
      n = CopyRates(job.src, tf, job.from, job.to + 86400 - 1, r);
      if(n <= 0) TF_Wait(250);
     }
   if(n <= 1) { err = "no " + tfname + " bars in that range"; return false; }
   int stride = (n > 600 ? (n + 599) / 600 : 1), m = 0;
   ArrayResize(tm, n / stride + 1); ArrayResize(orig, n / stride + 1); ArrayResize(neu, n / stride + 1);
   for(int i = 0; i < n; i += stride)
     {
      double v = Map(r[i].close, r[i].time);
      if(v <= 0.0) { err = "function gives zero/negative/invalid prices"; return false; }
      tm[m] = r[i].time; orig[m] = r[i].close; neu[m] = v; m++;
     }
   ArrayResize(tm, m); ArrayResize(orig, m); ArrayResize(neu, m);
   return true;
  }

//--- digits / contract properties --------------------------------------
bool CTFEngine::SetupSymbol(CTFResult &res)
  {
   bool custom;
   bool exists = TF_SymbolExists(m_job.name, custom);
   if(exists && !custom) { res.err = "'" + m_job.name + "' is a real broker symbol - choose another name"; return false; }
   if(exists)
     {
      CustomRatesDelete(m_job.name, 0, LONG_MAX);
      CustomTicksDelete(m_job.name, 0, LONG_MAX);
     }
   else if(!CustomSymbolCreate(m_job.name, "Tick Forge", m_job.src))
     { res.err = StringFormat("CustomSymbolCreate failed (err %d)", GetLastError()); return false; }
   SymbolSelect(m_job.name, true);

   // trading permissions follow the source, so an Expert's orders are accepted exactly as on the real symbol
   const ENUM_SYMBOL_INFO_INTEGER props[] = {SYMBOL_FILLING_MODE, SYMBOL_ORDER_MODE, SYMBOL_EXPIRATION_MODE, SYMBOL_TRADE_EXEMODE, SYMBOL_TRADE_MODE};
   for(int k = 0; k < ArraySize(props); k++)
      CustomSymbolSetInteger(m_job.name, props[k], SymbolInfoInteger(m_job.src, props[k]));

   // sample the price factor across the range to size the digits
   m_digits_src = (int)SymbolInfoInteger(m_job.src, SYMBOL_DIGITS);
   double psrc = SymbolInfoDouble(m_job.src, SYMBOL_POINT);
   double sumlog = 0.0; int cnt = 0;
   MqlRates r[];
   for(int k = 0; k <= 12; k++)
     {
      datetime tk = m_job.from + (datetime)((m_job.to + 86400 - m_job.from) * k / 12);
      if(tk >= m_job.to + 86400) tk = m_job.to;
      if(CopyRates(m_job.src, PERIOD_M1, tk, 1, r) > 0 && r[0].close > 0.0)
        {
         double f = Map(r[0].close, tk);
         if(f > 0.0) { sumlog += MathLog10(f / r[0].close); cnt++; }
        }
     }
   if(m_bad > 0) { res.err = "function gives zero/negative/invalid prices on sampled data"; return false; }
   double lg = (cnt > 0 ? sumlog / cnt : 0.0);
   m_digits_dst = m_digits_src;
   if(m_job.auto_digits && lg < 0.0) m_digits_dst = (int)MathMin(8, m_digits_src + (int)MathRound(-lg));
   m_point_dst = MathPow(10.0, -m_digits_dst);
   res.digits = m_digits_dst;

   if(m_digits_dst != m_digits_src)
     {
      CustomSymbolSetInteger(m_job.name, SYMBOL_DIGITS, m_digits_dst);
      CustomSymbolSetDouble(m_job.name, SYMBOL_POINT, m_point_dst);
      double ts = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_SIZE);
      double tv = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE);
      double tvp = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_PROFIT);
      double tvl = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_LOSS);
      double nts = m_point_dst, ratio = (ts > 0.0 ? nts / ts : 1.0);
      CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_SIZE, nts);
      CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE, tv * ratio);
      CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_PROFIT, tvp * ratio);
      CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_LOSS, tvl * ratio);
     }
   m_ready_at = GetTickCount() + 1500;            // a fresh custom symbol rejects its first write (err 4301): Step() waits
   return true;
  }

//--- one UTC-day of ticks -----------------------------------------------
void CTFEngine::FlushMinute(MqlRates &cur, MqlRates &out[], int &n)
  {
   if(cur.time == 0) return;
   if(ArraySize(out) <= n) ArrayResize(out, n + 1024);
   out[n++] = cur;
  }

bool CTFEngine::WriteTicksDay(const datetime day, CTFResult &res)
  {
   MqlTick src[];
   ulong a = (ulong)day * 1000, b = ((ulong)day + 86400) * 1000 - 1;
   int got = -1;
   for(int attempt = 0; attempt < 3 && got < 0; attempt++)
     {
      ResetLastError();
      got = CopyTicksRange(m_job.src, src, COPY_TICKS_ALL, a, b);
      if(got < 0) TF_Wait(200);
     }
   if(got <= 0) return true;                      // weekend / no data: nothing to write

   MqlTick dst[]; ArrayResize(dst, got);
   MqlRates bars[]; ArrayResize(bars, 1500);
   int nb = 0;
   MqlRates cur; ZeroMemory(cur);
   long off_msc = m_job.offset_sec * 1000;
   double pt = m_point_dst;
   int w = 0;
   for(int i = 0; i < got; i++)
     {
      double bid = src[i].bid, ask = src[i].ask;
      if(bid <= 0.0) continue;
      datetime ot = (datetime)(src[i].time_msc / 1000);
      double nb_ = Map(bid, ot);
      if(nb_ <= 0.0) { res.err = "function produced an invalid price"; return false; }
      double k = nb_ / bid;
      double na = 0.0;
      if(ask > 0.0)
        {
         double sp = ask - bid;
         na = nb_ + (m_job.scale_spread ? sp * k : sp);
        }
      nb_ = NormalizeDouble(nb_, m_digits_dst);
      na  = (na > 0.0 ? NormalizeDouble(na, m_digits_dst) : 0.0);
      dst[w].time_msc = src[i].time_msc + off_msc;
      dst[w].time = (datetime)(dst[w].time_msc / 1000);
      dst[w].bid = nb_; dst[w].ask = na; dst[w].last = 0.0; dst[w].volume = 0; dst[w].volume_real = 0.0;
      dst[w].flags = src[i].flags & (TICK_FLAG_BID | TICK_FLAG_ASK);
      if(dst[w].flags == 0) dst[w].flags = TICK_FLAG_BID | TICK_FLAG_ASK;

      // M1 aggregation on the shifted clock
      datetime mt = (datetime)((dst[w].time / 60) * 60);
      int spr = (na > 0.0 ? (int)MathRound((na - nb_) / pt) : 0);
      if(cur.time != mt)
        {
         FlushMinute(cur, bars, nb);
         cur.time = mt; cur.open = cur.high = cur.low = cur.close = nb_;
         cur.tick_volume = 1; cur.spread = spr; cur.real_volume = 0;
        }
      else
        {
         if(nb_ > cur.high) cur.high = nb_;
         if(nb_ < cur.low)  cur.low = nb_;
         cur.close = nb_; cur.tick_volume++; cur.spread = spr;
        }
      w++;
     }
   FlushMinute(cur, bars, nb);
   if(w == 0) return true;
   ArrayResize(dst, w);
   ArrayResize(bars, nb);

   ResetLastError();
   int dt = CustomTicksReplace(m_job.name, dst[0].time_msc, dst[w - 1].time_msc, dst);
   if(dt < 0) { res.err = StringFormat("CustomTicksReplace failed (err %d)", GetLastError()); return false; }
   res.ticks += dt;
   if(nb > 0)
     {
      ResetLastError();
      int db = CustomRatesReplace(m_job.name, bars[0].time, bars[nb - 1].time + 60, bars);
      if(db < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; }
      res.bars += db;
     }
   return true;
  }

//--- bars-only chunk -----------------------------------------------------
bool CTFEngine::WriteBarsChunk(const datetime a, const datetime b, CTFResult &res)
  {
   MqlRates src[];
   int got = -1;
   for(int attempt = 0; attempt < 3 && got < 0; attempt++)
     {
      ResetLastError();
      got = CopyRates(m_job.src, PERIOD_M1, a, b, src);
      if(got < 0) TF_Wait(200);
     }
   if(got <= 0) return true;
   MqlRates dst[]; ArrayResize(dst, got);
   for(int i = 0; i < got; i++)
     {
      double c = src[i].close;
      double o = Map(src[i].open, src[i].time), h = Map(src[i].high, src[i].time);
      double l = Map(src[i].low, src[i].time),  cl = Map(c, src[i].time);
      if(o <= 0.0 || h <= 0.0 || l <= 0.0 || cl <= 0.0) { res.err = "function produced an invalid price"; return false; }
      double hi = MathMax(MathMax(o, h), MathMax(l, cl)), lo = MathMin(MathMin(o, h), MathMin(l, cl));
      double k = (c > 0.0 ? cl / c : 1.0);
      dst[i].time  = (datetime)(src[i].time + m_job.offset_sec);
      dst[i].open  = NormalizeDouble(o, m_digits_dst);
      dst[i].high  = NormalizeDouble(hi, m_digits_dst);
      dst[i].low   = NormalizeDouble(lo, m_digits_dst);
      dst[i].close = NormalizeDouble(cl, m_digits_dst);
      dst[i].tick_volume = src[i].tick_volume;
      dst[i].real_volume = 0;
      // source spread is in source points; express it in destination points
      double sp_price = src[i].spread * SymbolInfoDouble(m_job.src, SYMBOL_POINT) * (m_job.scale_spread ? k : 1.0);
      dst[i].spread = (int)MathRound(sp_price / m_point_dst);
     }
   ResetLastError();
   int d = CustomRatesReplace(m_job.name, dst[0].time, dst[got - 1].time + 60, dst);
   if(d < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; }
   res.bars += d;
   return true;
  }

bool CTFEngine::Begin(const CTFJob &job, CTFResult &res)
  {
   m_job = job; m_bad = 0; m_i = 0; m_days = 0; m_ready_at = 0;
   res.ok = false; res.err = ""; res.ticks = 0; res.bars = 0; res.digits = 0; res.p0 = 0.0;
   if(!m_ex.Compile(job.expr)) { res.err = "function: " + m_ex.Error(); return false; }
   if(job.to < job.from) { res.err = "end date is before start date"; return false; }
   datetime t0;
   if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { res.err = "no M1 bars for " + job.src + " in that range"; return false; }
   res.p0 = m_p0;
   m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
   if(!SetupSymbol(res)) return false;
   m_days = (int)((job.to - job.from) / 86400) + 1;
   return true;
  }

bool CTFEngine::Step(CTFResult &res)
  {
   if(Done()) { res.ok = true; return true; }
   if((int)(GetTickCount() - m_ready_at) < 0) return true;      // symbol still settling
   datetime day = m_job.from + (datetime)m_i * 86400;
   bool ok = (m_job.mode == TF_MODE_TICKS ? WriteTicksDay(day, res) : WriteBarsChunk(day, day + 86400 - 60, res));
   if(!ok) return false;
   if(m_bad > 0) { res.err = "function produced an invalid price"; return false; }
   TF_Status(StringFormat("%s  day %d/%d   ticks %I64d   M1 bars %I64d", TimeToString(day, TIME_DATE), m_i + 1, m_days, res.ticks, res.bars));
   m_i++;
   if(Done()) res.ok = true;
   return true;
  }

bool CTFEngine::Run(const CTFJob &job, CTFResult &res)
  {
   if(!Begin(job, res)) return false;
   while(!Done())
      if(!Step(res)) return false;
   return true;
  }
//+------------------------------------------------------------------+