| //+------------------------------------------------------------------+
|
| //| TF_Engine.mqh - Tick Forge data generator |
|
| //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). |
|
| //| |
|
| //| Reads the broker's own history for a source symbol, shifts the |
|
| //| raw timestamps, maps every price through a user function, and |
|
| //| writes the result into a custom symbol. |
|
| //+------------------------------------------------------------------+
|
| #property strict
|
| #include "TF_Expr.mqh"
|
|
|
| void TF_Status(const string s); // implemented by the host program (logs + updates the GUI)
|
|
|
| enum ENUM_TF_MODE
|
| {
|
| TF_MODE_TICKS = 0, // Real ticks (transformed) + M1 bars rebuilt from them
|
| TF_MODE_BARS = 1 // M1 bars only (tester synthesizes the ticks)
|
| };
|
|
|
| struct CTFJob
|
| {
|
| string src; // source symbol (broker data)
|
| string name; // custom symbol to write
|
| datetime from; // first day (00:00)
|
| datetime to; // last day (inclusive)
|
| int mode; // ENUM_TF_MODE
|
| long offset_sec; // raw timestamp shift, seconds (may be negative)
|
| string expr; // price function
|
| bool scale_spread; // spread follows the price factor
|
| bool auto_digits; // add digits when the price level shrinks
|
| };
|
|
|
| struct CTFResult
|
| {
|
| bool ok;
|
| string err;
|
| long ticks;
|
| long bars;
|
| int digits;
|
| double p0;
|
| };
|
|
|
| //--- helpers -------------------------------------------------------
|
| // Sleep() is not allowed in indicators; a short busy wait is enough to let a history download answer.
|
| void TF_Wait(const uint ms)
|
| {
|
| const uint t = GetTickCount();
|
| while(GetTickCount() - t < ms) { }
|
| }
|
| bool TF_IsFinite(const double v) { return MathIsValidNumber(v); }
|
|
|
| bool TF_SymbolExists(const string name, bool &custom)
|
| {
|
| custom = false;
|
| for(int i = 0; i < SymbolsTotal(false); i++)
|
| if(SymbolName(i, false) == name)
|
| {
|
| custom = (bool)SymbolInfoInteger(name, SYMBOL_CUSTOM);
|
| return true;
|
| }
|
| return false;
|
| }
|
|
|
| string TF_DataPath(const string name)
|
| {
|
| return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\history\\" + name;
|
| }
|
| string TF_TickPath(const string name)
|
| {
|
| return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\ticks\\" + name;
|
| }
|
|
|
| //--- the engine ----------------------------------------------------
|
| class CTFEngine
|
| {
|
| private:
|
| CTFJob m_job;
|
| CTFExpr m_ex;
|
| double m_p0;
|
| double m_span; // seconds in the range
|
| datetime m_from;
|
| int m_digits_src, m_digits_dst;
|
| double m_point_dst;
|
| long m_bad;
|
| int m_i, m_days; // stepwise run: next day index, number of days
|
| uint m_ready_at; // GetTickCount() before which no write is attempted
|
|
|
| double Map(const double price, const datetime t);
|
| double PointDst() const { return m_point_dst; }
|
| bool Prepare(CTFResult &res);
|
| bool WriteTicksDay(const datetime day, CTFResult &res);
|
| bool WriteBarsChunk(const datetime a, const datetime b, CTFResult &res);
|
| void FlushMinute(MqlRates &cur, MqlRates &out[], int &n);
|
| bool SetupSymbol(CTFResult &res);
|
|
|
| public:
|
| CTFEngine() : m_bad(0), m_i(0), m_days(0), m_ready_at(0), m_p0(0), m_span(0), m_from(0), m_digits_src(0), m_digits_dst(0), m_point_dst(0) {}
|
| bool Preview(const CTFJob &job, double &first, double &last, double &mid, string &err);
|
| bool Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err);
|
| bool Begin(const CTFJob &job, CTFResult &res); // validate, create the symbol; then call Step() until Done()
|
| bool Step(CTFResult &res); // write one day (returns false on error)
|
| bool Done() const { return m_i >= m_days; }
|
| int DayIndex() const { return m_i; }
|
| int Days() const { return m_days; }
|
| bool Run(const CTFJob &job, CTFResult &res); // all steps at once (for tools; blocks)
|
| };
|
|
|
| double CTFEngine::Map(const double price, const datetime t)
|
| {
|
| double tt = (m_span > 0.0 ? (double)(t - m_from) / m_span : 0.0);
|
| if(tt < 0.0) tt = 0.0;
|
| if(tt > 1.0) tt = 1.0;
|
| double v = m_ex.Eval(price, m_p0, tt, (double)(t - m_from) / 86400.0);
|
| if(!TF_IsFinite(v) || v <= 0.0) { m_bad++; return 0.0; }
|
| return v;
|
| }
|
|
|
| //--- first M1 open on/after the range start ---------------------------
|
| bool TF_FirstOpen(const string sym, const datetime from, const datetime to, double &p0, datetime &t0)
|
| {
|
| MqlRates r[];
|
| for(int attempt = 0; attempt < 4; attempt++)
|
| {
|
| int n = CopyRates(sym, PERIOD_M1, from, MathMin(from + 7 * 86400, to + 86400), r);
|
| if(n > 0) { p0 = r[0].open; t0 = r[0].time; return true; }
|
| TF_Wait(250);
|
| }
|
| return false;
|
| }
|
|
|
| bool CTFEngine::Preview(const CTFJob &job, double &first, double &last, double &mid, string &err)
|
| {
|
| m_job = job; m_bad = 0; err = "";
|
| if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; }
|
| datetime t0;
|
| if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; }
|
| m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
|
| MqlRates r[];
|
| int n = CopyRates(job.src, PERIOD_M1, job.to, 1, r); // 1 bar at/ before range end
|
| double pl = (n > 0 ? r[0].close : m_p0);
|
| first = Map(m_p0, job.from);
|
| mid = Map((m_p0 + pl) / 2.0, job.from + (datetime)(m_span / 2));
|
| last = Map(pl, job.to + 86400);
|
| if(m_bad > 0) { err = "function gives zero/negative/invalid prices"; return false; }
|
| return true;
|
| }
|
|
|
| //--- overview series: whole range, source vs result, timeframe chosen from the span ---
|
| bool CTFEngine::Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err)
|
| {
|
| m_job = job; m_bad = 0; err = "";
|
| if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; }
|
| datetime t0;
|
| if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; }
|
| m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
|
| double days = m_span / 86400.0;
|
| ENUM_TIMEFRAMES tf = (days <= 3 ? PERIOD_M5 : days <= 14 ? PERIOD_M30 : days <= 120 ? PERIOD_H1 : days <= 700 ? PERIOD_H4 : PERIOD_D1);
|
| tfname = StringSubstr(EnumToString(tf), 7);
|
| MqlRates r[];
|
| int n = -1;
|
| for(int a = 0; a < 3 && n <= 0; a++)
|
| {
|
| n = CopyRates(job.src, tf, job.from, job.to + 86400 - 1, r);
|
| if(n <= 0) TF_Wait(250);
|
| }
|
| if(n <= 1) { err = "no " + tfname + " bars in that range"; return false; }
|
| int stride = (n > 600 ? (n + 599) / 600 : 1), m = 0;
|
| ArrayResize(tm, n / stride + 1); ArrayResize(orig, n / stride + 1); ArrayResize(neu, n / stride + 1);
|
| for(int i = 0; i < n; i += stride)
|
| {
|
| double v = Map(r[i].close, r[i].time);
|
| if(v <= 0.0) { err = "function gives zero/negative/invalid prices"; return false; }
|
| tm[m] = r[i].time; orig[m] = r[i].close; neu[m] = v; m++;
|
| }
|
| ArrayResize(tm, m); ArrayResize(orig, m); ArrayResize(neu, m);
|
| return true;
|
| }
|
|
|
| //--- digits / contract properties --------------------------------------
|
| bool CTFEngine::SetupSymbol(CTFResult &res)
|
| {
|
| bool custom;
|
| bool exists = TF_SymbolExists(m_job.name, custom);
|
| if(exists && !custom) { res.err = "'" + m_job.name + "' is a real broker symbol - choose another name"; return false; }
|
| if(exists)
|
| {
|
| CustomRatesDelete(m_job.name, 0, LONG_MAX);
|
| CustomTicksDelete(m_job.name, 0, LONG_MAX);
|
| }
|
| else if(!CustomSymbolCreate(m_job.name, "Tick Forge", m_job.src))
|
| { res.err = StringFormat("CustomSymbolCreate failed (err %d)", GetLastError()); return false; }
|
| SymbolSelect(m_job.name, true);
|
|
|
| // trading permissions follow the source, so an Expert's orders are accepted exactly as on the real symbol
|
| const ENUM_SYMBOL_INFO_INTEGER props[] = {SYMBOL_FILLING_MODE, SYMBOL_ORDER_MODE, SYMBOL_EXPIRATION_MODE, SYMBOL_TRADE_EXEMODE, SYMBOL_TRADE_MODE};
|
| for(int k = 0; k < ArraySize(props); k++)
|
| CustomSymbolSetInteger(m_job.name, props[k], SymbolInfoInteger(m_job.src, props[k]));
|
|
|
| // sample the price factor across the range to size the digits
|
| m_digits_src = (int)SymbolInfoInteger(m_job.src, SYMBOL_DIGITS);
|
| double psrc = SymbolInfoDouble(m_job.src, SYMBOL_POINT);
|
| double sumlog = 0.0; int cnt = 0;
|
| MqlRates r[];
|
| for(int k = 0; k <= 12; k++)
|
| {
|
| datetime tk = m_job.from + (datetime)((m_job.to + 86400 - m_job.from) * k / 12);
|
| if(tk >= m_job.to + 86400) tk = m_job.to;
|
| if(CopyRates(m_job.src, PERIOD_M1, tk, 1, r) > 0 && r[0].close > 0.0)
|
| {
|
| double f = Map(r[0].close, tk);
|
| if(f > 0.0) { sumlog += MathLog10(f / r[0].close); cnt++; }
|
| }
|
| }
|
| if(m_bad > 0) { res.err = "function gives zero/negative/invalid prices on sampled data"; return false; }
|
| double lg = (cnt > 0 ? sumlog / cnt : 0.0);
|
| m_digits_dst = m_digits_src;
|
| if(m_job.auto_digits && lg < 0.0) m_digits_dst = (int)MathMin(8, m_digits_src + (int)MathRound(-lg));
|
| m_point_dst = MathPow(10.0, -m_digits_dst);
|
| res.digits = m_digits_dst;
|
|
|
| if(m_digits_dst != m_digits_src)
|
| {
|
| CustomSymbolSetInteger(m_job.name, SYMBOL_DIGITS, m_digits_dst);
|
| CustomSymbolSetDouble(m_job.name, SYMBOL_POINT, m_point_dst);
|
| double ts = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_SIZE);
|
| double tv = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE);
|
| double tvp = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_PROFIT);
|
| double tvl = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_LOSS);
|
| double nts = m_point_dst, ratio = (ts > 0.0 ? nts / ts : 1.0);
|
| CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_SIZE, nts);
|
| CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE, tv * ratio);
|
| CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_PROFIT, tvp * ratio);
|
| CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_LOSS, tvl * ratio);
|
| }
|
| m_ready_at = GetTickCount() + 1500; // a fresh custom symbol rejects its first write (err 4301): Step() waits
|
| return true;
|
| }
|
|
|
| //--- one UTC-day of ticks -----------------------------------------------
|
| void CTFEngine::FlushMinute(MqlRates &cur, MqlRates &out[], int &n)
|
| {
|
| if(cur.time == 0) return;
|
| if(ArraySize(out) <= n) ArrayResize(out, n + 1024);
|
| out[n++] = cur;
|
| }
|
|
|
| bool CTFEngine::WriteTicksDay(const datetime day, CTFResult &res)
|
| {
|
| MqlTick src[];
|
| ulong a = (ulong)day * 1000, b = ((ulong)day + 86400) * 1000 - 1;
|
| int got = -1;
|
| for(int attempt = 0; attempt < 3 && got < 0; attempt++)
|
| {
|
| ResetLastError();
|
| got = CopyTicksRange(m_job.src, src, COPY_TICKS_ALL, a, b);
|
| if(got < 0) TF_Wait(200);
|
| }
|
| if(got <= 0) return true; // weekend / no data: nothing to write
|
|
|
| MqlTick dst[]; ArrayResize(dst, got);
|
| MqlRates bars[]; ArrayResize(bars, 1500);
|
| int nb = 0;
|
| MqlRates cur; ZeroMemory(cur);
|
| long off_msc = m_job.offset_sec * 1000;
|
| double pt = m_point_dst;
|
| int w = 0;
|
| for(int i = 0; i < got; i++)
|
| {
|
| double bid = src[i].bid, ask = src[i].ask;
|
| if(bid <= 0.0) continue;
|
| datetime ot = (datetime)(src[i].time_msc / 1000);
|
| double nb_ = Map(bid, ot);
|
| if(nb_ <= 0.0) { res.err = "function produced an invalid price"; return false; }
|
| double k = nb_ / bid;
|
| double na = 0.0;
|
| if(ask > 0.0)
|
| {
|
| double sp = ask - bid;
|
| na = nb_ + (m_job.scale_spread ? sp * k : sp);
|
| }
|
| nb_ = NormalizeDouble(nb_, m_digits_dst);
|
| na = (na > 0.0 ? NormalizeDouble(na, m_digits_dst) : 0.0);
|
| dst[w].time_msc = src[i].time_msc + off_msc;
|
| dst[w].time = (datetime)(dst[w].time_msc / 1000);
|
| dst[w].bid = nb_; dst[w].ask = na; dst[w].last = 0.0; dst[w].volume = 0; dst[w].volume_real = 0.0;
|
| dst[w].flags = src[i].flags & (TICK_FLAG_BID | TICK_FLAG_ASK);
|
| if(dst[w].flags == 0) dst[w].flags = TICK_FLAG_BID | TICK_FLAG_ASK;
|
|
|
| // M1 aggregation on the shifted clock
|
| datetime mt = (datetime)((dst[w].time / 60) * 60);
|
| int spr = (na > 0.0 ? (int)MathRound((na - nb_) / pt) : 0);
|
| if(cur.time != mt)
|
| {
|
| FlushMinute(cur, bars, nb);
|
| cur.time = mt; cur.open = cur.high = cur.low = cur.close = nb_;
|
| cur.tick_volume = 1; cur.spread = spr; cur.real_volume = 0;
|
| }
|
| else
|
| {
|
| if(nb_ > cur.high) cur.high = nb_;
|
| if(nb_ < cur.low) cur.low = nb_;
|
| cur.close = nb_; cur.tick_volume++; cur.spread = spr;
|
| }
|
| w++;
|
| }
|
| FlushMinute(cur, bars, nb);
|
| if(w == 0) return true;
|
| ArrayResize(dst, w);
|
| ArrayResize(bars, nb);
|
|
|
| ResetLastError();
|
| int dt = CustomTicksReplace(m_job.name, dst[0].time_msc, dst[w - 1].time_msc, dst);
|
| if(dt < 0) { res.err = StringFormat("CustomTicksReplace failed (err %d)", GetLastError()); return false; }
|
| res.ticks += dt;
|
| if(nb > 0)
|
| {
|
| ResetLastError();
|
| int db = CustomRatesReplace(m_job.name, bars[0].time, bars[nb - 1].time + 60, bars);
|
| if(db < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; }
|
| res.bars += db;
|
| }
|
| return true;
|
| }
|
|
|
| //--- bars-only chunk -----------------------------------------------------
|
| bool CTFEngine::WriteBarsChunk(const datetime a, const datetime b, CTFResult &res)
|
| {
|
| MqlRates src[];
|
| int got = -1;
|
| for(int attempt = 0; attempt < 3 && got < 0; attempt++)
|
| {
|
| ResetLastError();
|
| got = CopyRates(m_job.src, PERIOD_M1, a, b, src);
|
| if(got < 0) TF_Wait(200);
|
| }
|
| if(got <= 0) return true;
|
| MqlRates dst[]; ArrayResize(dst, got);
|
| for(int i = 0; i < got; i++)
|
| {
|
| double c = src[i].close;
|
| double o = Map(src[i].open, src[i].time), h = Map(src[i].high, src[i].time);
|
| double l = Map(src[i].low, src[i].time), cl = Map(c, src[i].time);
|
| if(o <= 0.0 || h <= 0.0 || l <= 0.0 || cl <= 0.0) { res.err = "function produced an invalid price"; return false; }
|
| double hi = MathMax(MathMax(o, h), MathMax(l, cl)), lo = MathMin(MathMin(o, h), MathMin(l, cl));
|
| double k = (c > 0.0 ? cl / c : 1.0);
|
| dst[i].time = (datetime)(src[i].time + m_job.offset_sec);
|
| dst[i].open = NormalizeDouble(o, m_digits_dst);
|
| dst[i].high = NormalizeDouble(hi, m_digits_dst);
|
| dst[i].low = NormalizeDouble(lo, m_digits_dst);
|
| dst[i].close = NormalizeDouble(cl, m_digits_dst);
|
| dst[i].tick_volume = src[i].tick_volume;
|
| dst[i].real_volume = 0;
|
| // source spread is in source points; express it in destination points
|
| double sp_price = src[i].spread * SymbolInfoDouble(m_job.src, SYMBOL_POINT) * (m_job.scale_spread ? k : 1.0);
|
| dst[i].spread = (int)MathRound(sp_price / m_point_dst);
|
| }
|
| ResetLastError();
|
| int d = CustomRatesReplace(m_job.name, dst[0].time, dst[got - 1].time + 60, dst);
|
| if(d < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; }
|
| res.bars += d;
|
| return true;
|
| }
|
|
|
| bool CTFEngine::Begin(const CTFJob &job, CTFResult &res)
|
| {
|
| m_job = job; m_bad = 0; m_i = 0; m_days = 0; m_ready_at = 0;
|
| res.ok = false; res.err = ""; res.ticks = 0; res.bars = 0; res.digits = 0; res.p0 = 0.0;
|
| if(!m_ex.Compile(job.expr)) { res.err = "function: " + m_ex.Error(); return false; }
|
| if(job.to < job.from) { res.err = "end date is before start date"; return false; }
|
| datetime t0;
|
| if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { res.err = "no M1 bars for " + job.src + " in that range"; return false; }
|
| res.p0 = m_p0;
|
| m_from = job.from; m_span = (double)(job.to + 86400 - job.from);
|
| if(!SetupSymbol(res)) return false;
|
| m_days = (int)((job.to - job.from) / 86400) + 1;
|
| return true;
|
| }
|
|
|
| bool CTFEngine::Step(CTFResult &res)
|
| {
|
| if(Done()) { res.ok = true; return true; }
|
| if((int)(GetTickCount() - m_ready_at) < 0) return true; // symbol still settling
|
| datetime day = m_job.from + (datetime)m_i * 86400;
|
| bool ok = (m_job.mode == TF_MODE_TICKS ? WriteTicksDay(day, res) : WriteBarsChunk(day, day + 86400 - 60, res));
|
| if(!ok) return false;
|
| if(m_bad > 0) { res.err = "function produced an invalid price"; return false; }
|
| TF_Status(StringFormat("%s day %d/%d ticks %I64d M1 bars %I64d", TimeToString(day, TIME_DATE), m_i + 1, m_days, res.ticks, res.bars));
|
| m_i++;
|
| if(Done()) res.ok = true;
|
| return true;
|
| }
|
|
|
| bool CTFEngine::Run(const CTFJob &job, CTFResult &res)
|
| {
|
| if(!Begin(job, res)) return false;
|
| while(!Done())
|
| if(!Step(res)) return false;
|
| return true;
|
| }
|
| //+------------------------------------------------------------------+
|
