//+------------------------------------------------------------------+ //| TF_Engine.mqh - Tick Forge data generator | //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). | //| | //| Reads the broker's own history for a source symbol, shifts the | //| raw timestamps, maps every price through a user function, and | //| writes the result into a custom symbol. | //+------------------------------------------------------------------+ #property strict #include "TF_Expr.mqh" void TF_Status(const string s); // implemented by the host program (logs + updates the GUI) enum ENUM_TF_MODE { TF_MODE_TICKS = 0, // Real ticks (transformed) + M1 bars rebuilt from them TF_MODE_BARS = 1 // M1 bars only (tester synthesizes the ticks) }; struct CTFJob { string src; // source symbol (broker data) string name; // custom symbol to write datetime from; // first day (00:00) datetime to; // last day (inclusive) int mode; // ENUM_TF_MODE long offset_sec; // raw timestamp shift, seconds (may be negative) string expr; // price function bool scale_spread; // spread follows the price factor bool auto_digits; // add digits when the price level shrinks }; struct CTFResult { bool ok; string err; long ticks; long bars; int digits; double p0; }; //--- helpers ------------------------------------------------------- // Sleep() is not allowed in indicators; a short busy wait is enough to let a history download answer. void TF_Wait(const uint ms) { const uint t = GetTickCount(); while(GetTickCount() - t < ms) { } } bool TF_IsFinite(const double v) { return MathIsValidNumber(v); } bool TF_SymbolExists(const string name, bool &custom) { custom = false; for(int i = 0; i < SymbolsTotal(false); i++) if(SymbolName(i, false) == name) { custom = (bool)SymbolInfoInteger(name, SYMBOL_CUSTOM); return true; } return false; } string TF_DataPath(const string name) { return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\history\\" + name; } string TF_TickPath(const string name) { return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\ticks\\" + name; } //--- the engine ---------------------------------------------------- class CTFEngine { private: CTFJob m_job; CTFExpr m_ex; double m_p0; double m_span; // seconds in the range datetime m_from; int m_digits_src, m_digits_dst; double m_point_dst; long m_bad; int m_i, m_days; // stepwise run: next day index, number of days uint m_ready_at; // GetTickCount() before which no write is attempted double Map(const double price, const datetime t); double PointDst() const { return m_point_dst; } bool Prepare(CTFResult &res); bool WriteTicksDay(const datetime day, CTFResult &res); bool WriteBarsChunk(const datetime a, const datetime b, CTFResult &res); void FlushMinute(MqlRates &cur, MqlRates &out[], int &n); bool SetupSymbol(CTFResult &res); public: CTFEngine() : m_bad(0), m_i(0), m_days(0), m_ready_at(0), m_p0(0), m_span(0), m_from(0), m_digits_src(0), m_digits_dst(0), m_point_dst(0) {} bool Preview(const CTFJob &job, double &first, double &last, double &mid, string &err); bool Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err); bool Begin(const CTFJob &job, CTFResult &res); // validate, create the symbol; then call Step() until Done() bool Step(CTFResult &res); // write one day (returns false on error) bool Done() const { return m_i >= m_days; } int DayIndex() const { return m_i; } int Days() const { return m_days; } bool Run(const CTFJob &job, CTFResult &res); // all steps at once (for tools; blocks) }; double CTFEngine::Map(const double price, const datetime t) { double tt = (m_span > 0.0 ? (double)(t - m_from) / m_span : 0.0); if(tt < 0.0) tt = 0.0; if(tt > 1.0) tt = 1.0; double v = m_ex.Eval(price, m_p0, tt, (double)(t - m_from) / 86400.0); if(!TF_IsFinite(v) || v <= 0.0) { m_bad++; return 0.0; } return v; } //--- first M1 open on/after the range start --------------------------- bool TF_FirstOpen(const string sym, const datetime from, const datetime to, double &p0, datetime &t0) { MqlRates r[]; for(int attempt = 0; attempt < 4; attempt++) { int n = CopyRates(sym, PERIOD_M1, from, MathMin(from + 7 * 86400, to + 86400), r); if(n > 0) { p0 = r[0].open; t0 = r[0].time; return true; } TF_Wait(250); } return false; } bool CTFEngine::Preview(const CTFJob &job, double &first, double &last, double &mid, string &err) { m_job = job; m_bad = 0; err = ""; if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; } m_from = job.from; m_span = (double)(job.to + 86400 - job.from); MqlRates r[]; int n = CopyRates(job.src, PERIOD_M1, job.to, 1, r); // 1 bar at/ before range end double pl = (n > 0 ? r[0].close : m_p0); first = Map(m_p0, job.from); mid = Map((m_p0 + pl) / 2.0, job.from + (datetime)(m_span / 2)); last = Map(pl, job.to + 86400); if(m_bad > 0) { err = "function gives zero/negative/invalid prices"; return false; } return true; } //--- overview series: whole range, source vs result, timeframe chosen from the span --- bool CTFEngine::Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err) { m_job = job; m_bad = 0; err = ""; if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; } m_from = job.from; m_span = (double)(job.to + 86400 - job.from); double days = m_span / 86400.0; ENUM_TIMEFRAMES tf = (days <= 3 ? PERIOD_M5 : days <= 14 ? PERIOD_M30 : days <= 120 ? PERIOD_H1 : days <= 700 ? PERIOD_H4 : PERIOD_D1); tfname = StringSubstr(EnumToString(tf), 7); MqlRates r[]; int n = -1; for(int a = 0; a < 3 && n <= 0; a++) { n = CopyRates(job.src, tf, job.from, job.to + 86400 - 1, r); if(n <= 0) TF_Wait(250); } if(n <= 1) { err = "no " + tfname + " bars in that range"; return false; } int stride = (n > 600 ? (n + 599) / 600 : 1), m = 0; ArrayResize(tm, n / stride + 1); ArrayResize(orig, n / stride + 1); ArrayResize(neu, n / stride + 1); for(int i = 0; i < n; i += stride) { double v = Map(r[i].close, r[i].time); if(v <= 0.0) { err = "function gives zero/negative/invalid prices"; return false; } tm[m] = r[i].time; orig[m] = r[i].close; neu[m] = v; m++; } ArrayResize(tm, m); ArrayResize(orig, m); ArrayResize(neu, m); return true; } //--- digits / contract properties -------------------------------------- bool CTFEngine::SetupSymbol(CTFResult &res) { bool custom; bool exists = TF_SymbolExists(m_job.name, custom); if(exists && !custom) { res.err = "'" + m_job.name + "' is a real broker symbol - choose another name"; return false; } if(exists) { CustomRatesDelete(m_job.name, 0, LONG_MAX); CustomTicksDelete(m_job.name, 0, LONG_MAX); } else if(!CustomSymbolCreate(m_job.name, "Tick Forge", m_job.src)) { res.err = StringFormat("CustomSymbolCreate failed (err %d)", GetLastError()); return false; } SymbolSelect(m_job.name, true); // trading permissions follow the source, so an Expert's orders are accepted exactly as on the real symbol const ENUM_SYMBOL_INFO_INTEGER props[] = {SYMBOL_FILLING_MODE, SYMBOL_ORDER_MODE, SYMBOL_EXPIRATION_MODE, SYMBOL_TRADE_EXEMODE, SYMBOL_TRADE_MODE}; for(int k = 0; k < ArraySize(props); k++) CustomSymbolSetInteger(m_job.name, props[k], SymbolInfoInteger(m_job.src, props[k])); // sample the price factor across the range to size the digits m_digits_src = (int)SymbolInfoInteger(m_job.src, SYMBOL_DIGITS); double psrc = SymbolInfoDouble(m_job.src, SYMBOL_POINT); double sumlog = 0.0; int cnt = 0; MqlRates r[]; for(int k = 0; k <= 12; k++) { datetime tk = m_job.from + (datetime)((m_job.to + 86400 - m_job.from) * k / 12); if(tk >= m_job.to + 86400) tk = m_job.to; if(CopyRates(m_job.src, PERIOD_M1, tk, 1, r) > 0 && r[0].close > 0.0) { double f = Map(r[0].close, tk); if(f > 0.0) { sumlog += MathLog10(f / r[0].close); cnt++; } } } if(m_bad > 0) { res.err = "function gives zero/negative/invalid prices on sampled data"; return false; } double lg = (cnt > 0 ? sumlog / cnt : 0.0); m_digits_dst = m_digits_src; if(m_job.auto_digits && lg < 0.0) m_digits_dst = (int)MathMin(8, m_digits_src + (int)MathRound(-lg)); m_point_dst = MathPow(10.0, -m_digits_dst); res.digits = m_digits_dst; if(m_digits_dst != m_digits_src) { CustomSymbolSetInteger(m_job.name, SYMBOL_DIGITS, m_digits_dst); CustomSymbolSetDouble(m_job.name, SYMBOL_POINT, m_point_dst); double ts = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_SIZE); double tv = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE); double tvp = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_PROFIT); double tvl = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_LOSS); double nts = m_point_dst, ratio = (ts > 0.0 ? nts / ts : 1.0); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_SIZE, nts); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE, tv * ratio); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_PROFIT, tvp * ratio); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_LOSS, tvl * ratio); } m_ready_at = GetTickCount() + 1500; // a fresh custom symbol rejects its first write (err 4301): Step() waits return true; } //--- one UTC-day of ticks ----------------------------------------------- void CTFEngine::FlushMinute(MqlRates &cur, MqlRates &out[], int &n) { if(cur.time == 0) return; if(ArraySize(out) <= n) ArrayResize(out, n + 1024); out[n++] = cur; } bool CTFEngine::WriteTicksDay(const datetime day, CTFResult &res) { MqlTick src[]; ulong a = (ulong)day * 1000, b = ((ulong)day + 86400) * 1000 - 1; int got = -1; for(int attempt = 0; attempt < 3 && got < 0; attempt++) { ResetLastError(); got = CopyTicksRange(m_job.src, src, COPY_TICKS_ALL, a, b); if(got < 0) TF_Wait(200); } if(got <= 0) return true; // weekend / no data: nothing to write MqlTick dst[]; ArrayResize(dst, got); MqlRates bars[]; ArrayResize(bars, 1500); int nb = 0; MqlRates cur; ZeroMemory(cur); long off_msc = m_job.offset_sec * 1000; double pt = m_point_dst; int w = 0; for(int i = 0; i < got; i++) { double bid = src[i].bid, ask = src[i].ask; if(bid <= 0.0) continue; datetime ot = (datetime)(src[i].time_msc / 1000); double nb_ = Map(bid, ot); if(nb_ <= 0.0) { res.err = "function produced an invalid price"; return false; } double k = nb_ / bid; double na = 0.0; if(ask > 0.0) { double sp = ask - bid; na = nb_ + (m_job.scale_spread ? sp * k : sp); } nb_ = NormalizeDouble(nb_, m_digits_dst); na = (na > 0.0 ? NormalizeDouble(na, m_digits_dst) : 0.0); dst[w].time_msc = src[i].time_msc + off_msc; dst[w].time = (datetime)(dst[w].time_msc / 1000); dst[w].bid = nb_; dst[w].ask = na; dst[w].last = 0.0; dst[w].volume = 0; dst[w].volume_real = 0.0; dst[w].flags = src[i].flags & (TICK_FLAG_BID | TICK_FLAG_ASK); if(dst[w].flags == 0) dst[w].flags = TICK_FLAG_BID | TICK_FLAG_ASK; // M1 aggregation on the shifted clock datetime mt = (datetime)((dst[w].time / 60) * 60); int spr = (na > 0.0 ? (int)MathRound((na - nb_) / pt) : 0); if(cur.time != mt) { FlushMinute(cur, bars, nb); cur.time = mt; cur.open = cur.high = cur.low = cur.close = nb_; cur.tick_volume = 1; cur.spread = spr; cur.real_volume = 0; } else { if(nb_ > cur.high) cur.high = nb_; if(nb_ < cur.low) cur.low = nb_; cur.close = nb_; cur.tick_volume++; cur.spread = spr; } w++; } FlushMinute(cur, bars, nb); if(w == 0) return true; ArrayResize(dst, w); ArrayResize(bars, nb); ResetLastError(); int dt = CustomTicksReplace(m_job.name, dst[0].time_msc, dst[w - 1].time_msc, dst); if(dt < 0) { res.err = StringFormat("CustomTicksReplace failed (err %d)", GetLastError()); return false; } res.ticks += dt; if(nb > 0) { ResetLastError(); int db = CustomRatesReplace(m_job.name, bars[0].time, bars[nb - 1].time + 60, bars); if(db < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; } res.bars += db; } return true; } //--- bars-only chunk ----------------------------------------------------- bool CTFEngine::WriteBarsChunk(const datetime a, const datetime b, CTFResult &res) { MqlRates src[]; int got = -1; for(int attempt = 0; attempt < 3 && got < 0; attempt++) { ResetLastError(); got = CopyRates(m_job.src, PERIOD_M1, a, b, src); if(got < 0) TF_Wait(200); } if(got <= 0) return true; MqlRates dst[]; ArrayResize(dst, got); for(int i = 0; i < got; i++) { double c = src[i].close; double o = Map(src[i].open, src[i].time), h = Map(src[i].high, src[i].time); double l = Map(src[i].low, src[i].time), cl = Map(c, src[i].time); if(o <= 0.0 || h <= 0.0 || l <= 0.0 || cl <= 0.0) { res.err = "function produced an invalid price"; return false; } double hi = MathMax(MathMax(o, h), MathMax(l, cl)), lo = MathMin(MathMin(o, h), MathMin(l, cl)); double k = (c > 0.0 ? cl / c : 1.0); dst[i].time = (datetime)(src[i].time + m_job.offset_sec); dst[i].open = NormalizeDouble(o, m_digits_dst); dst[i].high = NormalizeDouble(hi, m_digits_dst); dst[i].low = NormalizeDouble(lo, m_digits_dst); dst[i].close = NormalizeDouble(cl, m_digits_dst); dst[i].tick_volume = src[i].tick_volume; dst[i].real_volume = 0; // source spread is in source points; express it in destination points double sp_price = src[i].spread * SymbolInfoDouble(m_job.src, SYMBOL_POINT) * (m_job.scale_spread ? k : 1.0); dst[i].spread = (int)MathRound(sp_price / m_point_dst); } ResetLastError(); int d = CustomRatesReplace(m_job.name, dst[0].time, dst[got - 1].time + 60, dst); if(d < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; } res.bars += d; return true; } bool CTFEngine::Begin(const CTFJob &job, CTFResult &res) { m_job = job; m_bad = 0; m_i = 0; m_days = 0; m_ready_at = 0; res.ok = false; res.err = ""; res.ticks = 0; res.bars = 0; res.digits = 0; res.p0 = 0.0; if(!m_ex.Compile(job.expr)) { res.err = "function: " + m_ex.Error(); return false; } if(job.to < job.from) { res.err = "end date is before start date"; return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { res.err = "no M1 bars for " + job.src + " in that range"; return false; } res.p0 = m_p0; m_from = job.from; m_span = (double)(job.to + 86400 - job.from); if(!SetupSymbol(res)) return false; m_days = (int)((job.to - job.from) / 86400) + 1; return true; } bool CTFEngine::Step(CTFResult &res) { if(Done()) { res.ok = true; return true; } if((int)(GetTickCount() - m_ready_at) < 0) return true; // symbol still settling datetime day = m_job.from + (datetime)m_i * 86400; bool ok = (m_job.mode == TF_MODE_TICKS ? WriteTicksDay(day, res) : WriteBarsChunk(day, day + 86400 - 60, res)); if(!ok) return false; if(m_bad > 0) { res.err = "function produced an invalid price"; return false; } TF_Status(StringFormat("%s day %d/%d ticks %I64d M1 bars %I64d", TimeToString(day, TIME_DATE), m_i + 1, m_days, res.ticks, res.bars)); m_i++; if(Done()) res.ok = true; return true; } bool CTFEngine::Run(const CTFJob &job, CTFResult &res) { if(!Begin(job, res)) return false; while(!Done()) if(!Step(res)) return false; return true; } //+------------------------------------------------------------------+