//+------------------------------------------------------------------+ //| Tick_Forge.mq5 | //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). | //| | //| Build custom backtesting symbols from your broker's own history: | //| - shift the raw timestamps by hours or days | //| - rescale / reshape prices with a typed function | //| - write real (transformed) ticks or M1 bars | //| The result is an ordinary custom symbol the Strategy Tester can | //| use (Every tick / Every tick based on real ticks / 1 minute OHLC).| //| | //| A utility: attach it to any chart to open its window. It never | //| trades. The source is open and it uses no DLL. It writes only the | //| custom symbol (its bars and ticks) - nothing else. | //+------------------------------------------------------------------+ #property copyright "Copyright (c) 2026 Joshua Dunnink - MIT License" #property link "https://forge.mql5.io/joshuadunnink/Tick_Forge" #property version "1.04" #property description "Tick Forge builds custom symbols for the Strategy Tester from your broker's own history." #property description "Shift the timestamps, rescale or reshape the prices with a typed function, test your EA on the result." #property description "Utility: it never trades. Free and open source (MIT). No DLL, no internet access." #property strict // ---- inlined from TF_Gui.mqh ---- //+------------------------------------------------------------------+ //| TF_Gui.mqh - Tick Forge window (CAppDialog) and overview plot | //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). | //| Shared by the indicator (Tick_Forge.mq5) and the developer Expert.| //| No Sleep() and no MessageBox() (not allowed in indicators): long | //| work runs one day / one month per Tick() call from a host timer. | //+------------------------------------------------------------------+ #include #include #include #include #include #include #include // ---- inlined from TF_Engine.mqh ---- //+------------------------------------------------------------------+ //| TF_Engine.mqh - Tick Forge data generator | //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). | //| | //| Reads the broker's own history for a source symbol, shifts the | //| raw timestamps, maps every price through a user function, and | //| writes the result into a custom symbol. | //+------------------------------------------------------------------+ // ---- inlined from TF_Expr.mqh ---- //+------------------------------------------------------------------+ //| TF_Expr.mqh - Tick Forge price-function parser | //| Copyright (c) 2026 Joshua Dunnink. MIT License (see LICENSE). | //| | //| A small, safe expression compiler: text -> reverse-polish program | //| -> fast evaluation. No eval, no file access, no side effects. | //| | //| Variables p (x) original price p0 first price in range | //| r p / p0 t progress in range, 0..1 | //| d days since range start | //| Operators + - * / ^ and unary minus, parentheses | //| Functions abs sqrt ln log10 exp sin cos pow(a,b) min(a,b) max(a,b)| //| Constants pi e | //+------------------------------------------------------------------+ enum ENUM_TF_OP { TFOP_NUM = 0, TFOP_P, TFOP_P0, TFOP_R, TFOP_T, TFOP_D, TFOP_ADD, TFOP_SUB, TFOP_MUL, TFOP_DIV, TFOP_POW, TFOP_NEG, TFOP_ABS, TFOP_SQRT, TFOP_LN, TFOP_LOG10, TFOP_EXP, TFOP_SIN, TFOP_COS, TFOP_MIN, TFOP_MAX }; class CTFExpr { private: string m_src; int m_pos; string m_err; int m_op[]; double m_val[]; int m_n; int m_depth, m_maxdepth; void Emit(const int op, const double v = 0.0); void SkipWs(); bool Fail(const string e) { if(m_err == "") m_err = e + " at position " + IntegerToString(m_pos + 1); return false; } bool ParseExpr(); bool ParseTerm(); bool ParseUnary(); bool ParsePower(); bool ParsePrimary(); bool Peek(const ushort c); int FuncOp(const string name, int &argc); public: CTFExpr() : m_n(0) {} bool Compile(const string text); string Error() const { return m_err; } double Eval(const double p, const double p0, const double t, const double d) const; }; void CTFExpr::Emit(const int op, const double v) { if(ArraySize(m_op) <= m_n) { ArrayResize(m_op, m_n + 32); ArrayResize(m_val, m_n + 32); } m_op[m_n] = op; m_val[m_n] = v; m_n++; // stack bookkeeping: pushes +1, binary ops -1, unary/func1 0 switch(op) { case TFOP_NUM: case TFOP_P: case TFOP_P0: case TFOP_R: case TFOP_T: case TFOP_D: m_depth++; if(m_depth > m_maxdepth) m_maxdepth = m_depth; break; case TFOP_ADD: case TFOP_SUB: case TFOP_MUL: case TFOP_DIV: case TFOP_POW: case TFOP_MIN: case TFOP_MAX: m_depth--; break; default: break; } } void CTFExpr::SkipWs() { while(m_pos < StringLen(m_src) && (m_src[m_pos] == ' ' || m_src[m_pos] == '\t')) m_pos++; } bool CTFExpr::Peek(const ushort c) { SkipWs(); return (m_pos < StringLen(m_src) && m_src[m_pos] == c); } bool CTFExpr::Compile(const string text) { m_src = text; m_pos = 0; m_err = ""; m_n = 0; m_depth = 0; m_maxdepth = 0; ArrayResize(m_op, 32); ArrayResize(m_val, 32); if(StringLen(text) == 0) { m_err = "empty expression"; return false; } if(!ParseExpr()) return false; SkipWs(); if(m_pos < StringLen(m_src)) return Fail("unexpected '" + StringSubstr(m_src, m_pos, 1) + "'"); if(m_maxdepth > 60) { m_err = "expression too deep"; return false; } return true; } bool CTFExpr::ParseExpr() { if(!ParseTerm()) return false; while(true) { SkipWs(); if(m_pos >= StringLen(m_src)) return true; ushort c = m_src[m_pos]; if(c != '+' && c != '-') return true; m_pos++; if(!ParseTerm()) return false; Emit(c == '+' ? TFOP_ADD : TFOP_SUB); } return true; } bool CTFExpr::ParseTerm() { if(!ParseUnary()) return false; while(true) { SkipWs(); if(m_pos >= StringLen(m_src)) return true; ushort c = m_src[m_pos]; if(c != '*' && c != '/') return true; m_pos++; if(!ParseUnary()) return false; Emit(c == '*' ? TFOP_MUL : TFOP_DIV); } return true; } bool CTFExpr::ParseUnary() { SkipWs(); if(m_pos < StringLen(m_src) && (m_src[m_pos] == '-' || m_src[m_pos] == '+')) { bool neg = (m_src[m_pos] == '-'); m_pos++; if(!ParseUnary()) return false; if(neg) Emit(TFOP_NEG); return true; } return ParsePower(); } bool CTFExpr::ParsePower() { if(!ParsePrimary()) return false; SkipWs(); if(m_pos < StringLen(m_src) && m_src[m_pos] == '^') { m_pos++; if(!ParseUnary()) return false; // right associative, allows 2^-1 Emit(TFOP_POW); } return true; } int CTFExpr::FuncOp(const string name, int &argc) { argc = 1; if(name == "abs") return TFOP_ABS; if(name == "sqrt") return TFOP_SQRT; if(name == "ln") return TFOP_LN; if(name == "log10") return TFOP_LOG10; if(name == "exp") return TFOP_EXP; if(name == "sin") return TFOP_SIN; if(name == "cos") return TFOP_COS; argc = 2; if(name == "pow") return TFOP_POW; if(name == "min") return TFOP_MIN; if(name == "max") return TFOP_MAX; return -1; } bool CTFExpr::ParsePrimary() { SkipWs(); if(m_pos >= StringLen(m_src)) return Fail("unexpected end"); ushort c = m_src[m_pos]; if(c == '(') { m_pos++; if(!ParseExpr()) return false; if(!Peek(')')) return Fail("missing ')'"); m_pos++; return true; } if((c >= '0' && c <= '9') || c == '.') { int s = m_pos; while(m_pos < StringLen(m_src) && ((m_src[m_pos] >= '0' && m_src[m_pos] <= '9') || m_src[m_pos] == '.')) m_pos++; // optional exponent 1e-3 if(m_pos < StringLen(m_src) && (m_src[m_pos] == 'e' || m_src[m_pos] == 'E')) { int save = m_pos; m_pos++; if(m_pos < StringLen(m_src) && (m_src[m_pos] == '+' || m_src[m_pos] == '-')) m_pos++; int ds = m_pos; while(m_pos < StringLen(m_src) && m_src[m_pos] >= '0' && m_src[m_pos] <= '9') m_pos++; if(m_pos == ds) m_pos = save; } Emit(TFOP_NUM, StringToDouble(StringSubstr(m_src, s, m_pos - s))); return true; } if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') { int s = m_pos; while(m_pos < StringLen(m_src) && ((m_src[m_pos] >= 'a' && m_src[m_pos] <= 'z') || (m_src[m_pos] >= 'A' && m_src[m_pos] <= 'Z') || (m_src[m_pos] >= '0' && m_src[m_pos] <= '9') || m_src[m_pos] == '_')) m_pos++; string id = StringSubstr(m_src, s, m_pos - s); StringToLower(id); if(Peek('(')) { int argc; int op = FuncOp(id, argc); if(op < 0) return Fail("unknown function '" + id + "'"); m_pos++; // ( if(!ParseExpr()) return false; if(argc == 2) { if(!Peek(',')) return Fail("function '" + id + "' needs 2 arguments"); m_pos++; if(!ParseExpr()) return false; } if(!Peek(')')) return Fail("missing ')'"); m_pos++; Emit(op); return true; } if(id == "p" || id == "x") { Emit(TFOP_P); return true; } if(id == "p0") { Emit(TFOP_P0); return true; } if(id == "r") { Emit(TFOP_R); return true; } if(id == "t") { Emit(TFOP_T); return true; } if(id == "d") { Emit(TFOP_D); return true; } if(id == "pi") { Emit(TFOP_NUM, M_PI); return true; } if(id == "e") { Emit(TFOP_NUM, M_E); return true; } m_pos = s; return Fail("unknown name '" + id + "'"); } return Fail("unexpected '" + StringSubstr(m_src, m_pos, 1) + "'"); } double CTFExpr::Eval(const double p, const double p0, const double t, const double d) const { double st[64]; int sp = 0; for(int i = 0; i < m_n; i++) { switch(m_op[i]) { case TFOP_NUM: st[sp++] = m_val[i]; break; case TFOP_P: st[sp++] = p; break; case TFOP_P0: st[sp++] = p0; break; case TFOP_R: st[sp++] = (p0 != 0.0 ? p / p0 : 0.0); break; case TFOP_T: st[sp++] = t; break; case TFOP_D: st[sp++] = d; break; case TFOP_ADD: sp--; st[sp - 1] += st[sp]; break; case TFOP_SUB: sp--; st[sp - 1] -= st[sp]; break; case TFOP_MUL: sp--; st[sp - 1] *= st[sp]; break; case TFOP_DIV: sp--; st[sp - 1] = (st[sp] != 0.0 ? st[sp - 1] / st[sp] : MathArcsin(2.0)); break; // NaN case TFOP_POW: sp--; st[sp - 1] = MathPow(st[sp - 1], st[sp]); break; case TFOP_NEG: st[sp - 1] = -st[sp - 1]; break; case TFOP_ABS: st[sp - 1] = MathAbs(st[sp - 1]); break; case TFOP_SQRT: st[sp - 1] = MathSqrt(st[sp - 1]); break; case TFOP_LN: st[sp - 1] = MathLog(st[sp - 1]); break; case TFOP_LOG10: st[sp - 1] = MathLog10(st[sp - 1]); break; case TFOP_EXP: st[sp - 1] = MathExp(st[sp - 1]); break; case TFOP_SIN: st[sp - 1] = MathSin(st[sp - 1]); break; case TFOP_COS: st[sp - 1] = MathCos(st[sp - 1]); break; case TFOP_MIN: sp--; st[sp - 1] = MathMin(st[sp - 1], st[sp]); break; case TFOP_MAX: sp--; st[sp - 1] = MathMax(st[sp - 1], st[sp]); break; } } return (sp == 1 ? st[0] : MathArcsin(2.0)); } //+------------------------------------------------------------------+ // ---- end TF_Expr.mqh ---- void TF_Status(const string s); // implemented by the host program (logs + updates the GUI) enum ENUM_TF_MODE { TF_MODE_TICKS = 0, // Real ticks (transformed) + M1 bars rebuilt from them TF_MODE_BARS = 1 // M1 bars only (tester synthesizes the ticks) }; struct CTFJob { string src; // source symbol (broker data) string name; // custom symbol to write datetime from; // first day (00:00) datetime to; // last day (inclusive) int mode; // ENUM_TF_MODE long offset_sec; // raw timestamp shift, seconds (may be negative) string expr; // price function bool scale_spread; // spread follows the price factor bool auto_digits; // add digits when the price level shrinks }; struct CTFResult { bool ok; string err; long ticks; long bars; int digits; double p0; }; //--- helpers ------------------------------------------------------- // Sleep() is not allowed in indicators; a short busy wait is enough to let a history download answer. void TF_Wait(const uint ms) { const uint t = GetTickCount(); while(GetTickCount() - t < ms) { } } bool TF_IsFinite(const double v) { return MathIsValidNumber(v); } bool TF_SymbolExists(const string name, bool &custom) { custom = false; for(int i = 0; i < SymbolsTotal(false); i++) if(SymbolName(i, false) == name) { custom = (bool)SymbolInfoInteger(name, SYMBOL_CUSTOM); return true; } return false; } string TF_DataPath(const string name) { return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\history\\" + name; } string TF_TickPath(const string name) { return TerminalInfoString(TERMINAL_DATA_PATH) + "\\bases\\Custom\\ticks\\" + name; } //--- the engine ---------------------------------------------------- class CTFEngine { private: CTFJob m_job; CTFExpr m_ex; double m_p0; double m_span; // seconds in the range datetime m_from; int m_digits_src, m_digits_dst; double m_point_dst; long m_bad; int m_i, m_days; // stepwise run: next day index, number of days uint m_ready_at; // GetTickCount() before which no write is attempted double Map(const double price, const datetime t); double PointDst() const { return m_point_dst; } bool Prepare(CTFResult &res); bool WriteTicksDay(const datetime day, CTFResult &res); bool WriteBarsChunk(const datetime a, const datetime b, CTFResult &res); void FlushMinute(MqlRates &cur, MqlRates &out[], int &n); bool SetupSymbol(CTFResult &res); public: CTFEngine() : m_bad(0), m_i(0), m_days(0), m_ready_at(0), m_p0(0), m_span(0), m_from(0), m_digits_src(0), m_digits_dst(0), m_point_dst(0) {} bool Preview(const CTFJob &job, double &first, double &last, double &mid, string &err); bool Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err); bool Begin(const CTFJob &job, CTFResult &res); // validate, create the symbol; then call Step() until Done() bool Step(CTFResult &res); // write one day (returns false on error) bool Done() const { return m_i >= m_days; } int DayIndex() const { return m_i; } int Days() const { return m_days; } bool Run(const CTFJob &job, CTFResult &res); // all steps at once (for tools; blocks) }; double CTFEngine::Map(const double price, const datetime t) { double tt = (m_span > 0.0 ? (double)(t - m_from) / m_span : 0.0); if(tt < 0.0) tt = 0.0; if(tt > 1.0) tt = 1.0; double v = m_ex.Eval(price, m_p0, tt, (double)(t - m_from) / 86400.0); if(!TF_IsFinite(v) || v <= 0.0) { m_bad++; return 0.0; } return v; } //--- first M1 open on/after the range start --------------------------- bool TF_FirstOpen(const string sym, const datetime from, const datetime to, double &p0, datetime &t0) { MqlRates r[]; for(int attempt = 0; attempt < 4; attempt++) { int n = CopyRates(sym, PERIOD_M1, from, MathMin(from + 7 * 86400, to + 86400), r); if(n > 0) { p0 = r[0].open; t0 = r[0].time; return true; } TF_Wait(250); } return false; } bool CTFEngine::Preview(const CTFJob &job, double &first, double &last, double &mid, string &err) { m_job = job; m_bad = 0; err = ""; if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; } m_from = job.from; m_span = (double)(job.to + 86400 - job.from); MqlRates r[]; int n = CopyRates(job.src, PERIOD_M1, job.to, 1, r); // 1 bar at/ before range end double pl = (n > 0 ? r[0].close : m_p0); first = Map(m_p0, job.from); mid = Map((m_p0 + pl) / 2.0, job.from + (datetime)(m_span / 2)); last = Map(pl, job.to + 86400); if(m_bad > 0) { err = "function gives zero/negative/invalid prices"; return false; } return true; } //--- overview series: whole range, source vs result, timeframe chosen from the span --- bool CTFEngine::Series(const CTFJob &job, datetime &tm[], double &orig[], double &neu[], string &tfname, string &err) { m_job = job; m_bad = 0; err = ""; if(!m_ex.Compile(job.expr)) { err = m_ex.Error(); return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { err = "no M1 bars for " + job.src + " in range"; return false; } m_from = job.from; m_span = (double)(job.to + 86400 - job.from); double days = m_span / 86400.0; ENUM_TIMEFRAMES tf = (days <= 3 ? PERIOD_M5 : days <= 14 ? PERIOD_M30 : days <= 120 ? PERIOD_H1 : days <= 700 ? PERIOD_H4 : PERIOD_D1); tfname = StringSubstr(EnumToString(tf), 7); MqlRates r[]; int n = -1; for(int a = 0; a < 3 && n <= 0; a++) { n = CopyRates(job.src, tf, job.from, job.to + 86400 - 1, r); if(n <= 0) TF_Wait(250); } if(n <= 1) { err = "no " + tfname + " bars in that range"; return false; } int stride = (n > 600 ? (n + 599) / 600 : 1), m = 0; ArrayResize(tm, n / stride + 1); ArrayResize(orig, n / stride + 1); ArrayResize(neu, n / stride + 1); for(int i = 0; i < n; i += stride) { double v = Map(r[i].close, r[i].time); if(v <= 0.0) { err = "function gives zero/negative/invalid prices"; return false; } tm[m] = r[i].time; orig[m] = r[i].close; neu[m] = v; m++; } ArrayResize(tm, m); ArrayResize(orig, m); ArrayResize(neu, m); return true; } //--- digits / contract properties -------------------------------------- bool CTFEngine::SetupSymbol(CTFResult &res) { bool custom; bool exists = TF_SymbolExists(m_job.name, custom); if(exists && !custom) { res.err = "'" + m_job.name + "' is a real broker symbol - choose another name"; return false; } if(exists) { CustomRatesDelete(m_job.name, 0, LONG_MAX); CustomTicksDelete(m_job.name, 0, LONG_MAX); } else if(!CustomSymbolCreate(m_job.name, "Tick Forge", m_job.src)) { res.err = StringFormat("CustomSymbolCreate failed (err %d)", GetLastError()); return false; } SymbolSelect(m_job.name, true); // trading permissions follow the source, so an Expert's orders are accepted exactly as on the real symbol const ENUM_SYMBOL_INFO_INTEGER props[] = {SYMBOL_FILLING_MODE, SYMBOL_ORDER_MODE, SYMBOL_EXPIRATION_MODE, SYMBOL_TRADE_EXEMODE, SYMBOL_TRADE_MODE}; for(int k = 0; k < ArraySize(props); k++) CustomSymbolSetInteger(m_job.name, props[k], SymbolInfoInteger(m_job.src, props[k])); // sample the price factor across the range to size the digits m_digits_src = (int)SymbolInfoInteger(m_job.src, SYMBOL_DIGITS); double psrc = SymbolInfoDouble(m_job.src, SYMBOL_POINT); double sumlog = 0.0; int cnt = 0; MqlRates r[]; for(int k = 0; k <= 12; k++) { datetime tk = m_job.from + (datetime)((m_job.to + 86400 - m_job.from) * k / 12); if(tk >= m_job.to + 86400) tk = m_job.to; if(CopyRates(m_job.src, PERIOD_M1, tk, 1, r) > 0 && r[0].close > 0.0) { double f = Map(r[0].close, tk); if(f > 0.0) { sumlog += MathLog10(f / r[0].close); cnt++; } } } if(m_bad > 0) { res.err = "function gives zero/negative/invalid prices on sampled data"; return false; } double lg = (cnt > 0 ? sumlog / cnt : 0.0); m_digits_dst = m_digits_src; if(m_job.auto_digits && lg < 0.0) m_digits_dst = (int)MathMin(8, m_digits_src + (int)MathRound(-lg)); m_point_dst = MathPow(10.0, -m_digits_dst); res.digits = m_digits_dst; if(m_digits_dst != m_digits_src) { CustomSymbolSetInteger(m_job.name, SYMBOL_DIGITS, m_digits_dst); CustomSymbolSetDouble(m_job.name, SYMBOL_POINT, m_point_dst); double ts = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_SIZE); double tv = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE); double tvp = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_PROFIT); double tvl = SymbolInfoDouble(m_job.src, SYMBOL_TRADE_TICK_VALUE_LOSS); double nts = m_point_dst, ratio = (ts > 0.0 ? nts / ts : 1.0); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_SIZE, nts); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE, tv * ratio); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_PROFIT, tvp * ratio); CustomSymbolSetDouble(m_job.name, SYMBOL_TRADE_TICK_VALUE_LOSS, tvl * ratio); } m_ready_at = GetTickCount() + 1500; // a fresh custom symbol rejects its first write (err 4301): Step() waits return true; } //--- one UTC-day of ticks ----------------------------------------------- void CTFEngine::FlushMinute(MqlRates &cur, MqlRates &out[], int &n) { if(cur.time == 0) return; if(ArraySize(out) <= n) ArrayResize(out, n + 1024); out[n++] = cur; } bool CTFEngine::WriteTicksDay(const datetime day, CTFResult &res) { MqlTick src[]; ulong a = (ulong)day * 1000, b = ((ulong)day + 86400) * 1000 - 1; int got = -1; for(int attempt = 0; attempt < 3 && got < 0; attempt++) { ResetLastError(); got = CopyTicksRange(m_job.src, src, COPY_TICKS_ALL, a, b); if(got < 0) TF_Wait(200); } if(got <= 0) return true; // weekend / no data: nothing to write MqlTick dst[]; ArrayResize(dst, got); MqlRates bars[]; ArrayResize(bars, 1500); int nb = 0; MqlRates cur; ZeroMemory(cur); long off_msc = m_job.offset_sec * 1000; double pt = m_point_dst; int w = 0; for(int i = 0; i < got; i++) { double bid = src[i].bid, ask = src[i].ask; if(bid <= 0.0) continue; datetime ot = (datetime)(src[i].time_msc / 1000); double nb_ = Map(bid, ot); if(nb_ <= 0.0) { res.err = "function produced an invalid price"; return false; } double k = nb_ / bid; double na = 0.0; if(ask > 0.0) { double sp = ask - bid; na = nb_ + (m_job.scale_spread ? sp * k : sp); } nb_ = NormalizeDouble(nb_, m_digits_dst); na = (na > 0.0 ? NormalizeDouble(na, m_digits_dst) : 0.0); dst[w].time_msc = src[i].time_msc + off_msc; dst[w].time = (datetime)(dst[w].time_msc / 1000); dst[w].bid = nb_; dst[w].ask = na; dst[w].last = 0.0; dst[w].volume = 0; dst[w].volume_real = 0.0; dst[w].flags = src[i].flags & (TICK_FLAG_BID | TICK_FLAG_ASK); if(dst[w].flags == 0) dst[w].flags = TICK_FLAG_BID | TICK_FLAG_ASK; // M1 aggregation on the shifted clock datetime mt = (datetime)((dst[w].time / 60) * 60); int spr = (na > 0.0 ? (int)MathRound((na - nb_) / pt) : 0); if(cur.time != mt) { FlushMinute(cur, bars, nb); cur.time = mt; cur.open = cur.high = cur.low = cur.close = nb_; cur.tick_volume = 1; cur.spread = spr; cur.real_volume = 0; } else { if(nb_ > cur.high) cur.high = nb_; if(nb_ < cur.low) cur.low = nb_; cur.close = nb_; cur.tick_volume++; cur.spread = spr; } w++; } FlushMinute(cur, bars, nb); if(w == 0) return true; ArrayResize(dst, w); ArrayResize(bars, nb); ResetLastError(); int dt = CustomTicksReplace(m_job.name, dst[0].time_msc, dst[w - 1].time_msc, dst); if(dt < 0) { res.err = StringFormat("CustomTicksReplace failed (err %d)", GetLastError()); return false; } res.ticks += dt; if(nb > 0) { ResetLastError(); int db = CustomRatesReplace(m_job.name, bars[0].time, bars[nb - 1].time + 60, bars); if(db < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; } res.bars += db; } return true; } //--- bars-only chunk ----------------------------------------------------- bool CTFEngine::WriteBarsChunk(const datetime a, const datetime b, CTFResult &res) { MqlRates src[]; int got = -1; for(int attempt = 0; attempt < 3 && got < 0; attempt++) { ResetLastError(); got = CopyRates(m_job.src, PERIOD_M1, a, b, src); if(got < 0) TF_Wait(200); } if(got <= 0) return true; MqlRates dst[]; ArrayResize(dst, got); for(int i = 0; i < got; i++) { double c = src[i].close; double o = Map(src[i].open, src[i].time), h = Map(src[i].high, src[i].time); double l = Map(src[i].low, src[i].time), cl = Map(c, src[i].time); if(o <= 0.0 || h <= 0.0 || l <= 0.0 || cl <= 0.0) { res.err = "function produced an invalid price"; return false; } double hi = MathMax(MathMax(o, h), MathMax(l, cl)), lo = MathMin(MathMin(o, h), MathMin(l, cl)); double k = (c > 0.0 ? cl / c : 1.0); dst[i].time = (datetime)(src[i].time + m_job.offset_sec); dst[i].open = NormalizeDouble(o, m_digits_dst); dst[i].high = NormalizeDouble(hi, m_digits_dst); dst[i].low = NormalizeDouble(lo, m_digits_dst); dst[i].close = NormalizeDouble(cl, m_digits_dst); dst[i].tick_volume = src[i].tick_volume; dst[i].real_volume = 0; // source spread is in source points; express it in destination points double sp_price = src[i].spread * SymbolInfoDouble(m_job.src, SYMBOL_POINT) * (m_job.scale_spread ? k : 1.0); dst[i].spread = (int)MathRound(sp_price / m_point_dst); } ResetLastError(); int d = CustomRatesReplace(m_job.name, dst[0].time, dst[got - 1].time + 60, dst); if(d < 0) { res.err = StringFormat("CustomRatesReplace failed (err %d)", GetLastError()); return false; } res.bars += d; return true; } bool CTFEngine::Begin(const CTFJob &job, CTFResult &res) { m_job = job; m_bad = 0; m_i = 0; m_days = 0; m_ready_at = 0; res.ok = false; res.err = ""; res.ticks = 0; res.bars = 0; res.digits = 0; res.p0 = 0.0; if(!m_ex.Compile(job.expr)) { res.err = "function: " + m_ex.Error(); return false; } if(job.to < job.from) { res.err = "end date is before start date"; return false; } datetime t0; if(!TF_FirstOpen(job.src, job.from, job.to, m_p0, t0)) { res.err = "no M1 bars for " + job.src + " in that range"; return false; } res.p0 = m_p0; m_from = job.from; m_span = (double)(job.to + 86400 - job.from); if(!SetupSymbol(res)) return false; m_days = (int)((job.to - job.from) / 86400) + 1; return true; } bool CTFEngine::Step(CTFResult &res) { if(Done()) { res.ok = true; return true; } if((int)(GetTickCount() - m_ready_at) < 0) return true; // symbol still settling datetime day = m_job.from + (datetime)m_i * 86400; bool ok = (m_job.mode == TF_MODE_TICKS ? WriteTicksDay(day, res) : WriteBarsChunk(day, day + 86400 - 60, res)); if(!ok) return false; if(m_bad > 0) { res.err = "function produced an invalid price"; return false; } TF_Status(StringFormat("%s day %d/%d ticks %I64d M1 bars %I64d", TimeToString(day, TIME_DATE), m_i + 1, m_days, res.ticks, res.bars)); m_i++; if(Done()) res.ok = true; return true; } bool CTFEngine::Run(const CTFJob &job, CTFResult &res) { if(!Begin(job, res)) return false; while(!Done()) if(!Step(res)) return false; return true; } //+------------------------------------------------------------------+ // ---- end TF_Engine.mqh ---- #define TF_W 1030 #define TF_CW 540 #define TF_H 596 #define TF_PLOT_H 554 #define TF_LBL_W 150 #define TF_X 10 string g_presets_txt[] = {"Original prices (control)", "Multiply x2", "Multiply x0.5", "Squared move (extreme)", "Ramp x1 -> x2 over range", "Offset +1000", "Inverted move"}; string g_presets_fn[] = {"p", "p*2", "p*0.5", "p0*r^2", "p*(1+t)", "p+1000", "p0*p0/p"}; //+------------------------------------------------------------------+ //| Overview plot: source and result price over the whole range | //+------------------------------------------------------------------+ class CTFPlot : public CWnd { private: CCanvas m_cv; string m_obj; bool m_ok; void Panel(const int top, const int h, const string title, const datetime &t[], const double &v[], const int n, const long shift, const uint clr); public: CTFPlot() : m_ok(false) {} virtual bool Create(const long chart, const string name, const int subwin, const int x1, const int y1, const int x2, const int y2); virtual bool OnMove(void); virtual bool OnShow(void) { if(m_ok) ObjectSetInteger(m_chart_id, m_obj, OBJPROP_TIMEFRAMES, OBJ_ALL_PERIODS); return true; } virtual bool OnHide(void) { if(m_ok) ObjectSetInteger(m_chart_id, m_obj, OBJPROP_TIMEFRAMES, OBJ_NO_PERIODS); return true; } virtual bool OnDestroy(void) { if(m_ok) { m_cv.Destroy(); m_ok = false; } return true; } void Hint(const string a, const string b = ""); void Lines(const string a, const string b = "", const string c = ""); void Para(const string a, const string b = ""); void Draw(const datetime &t[], const double &o[], const double &v[], const int n, const long shift, const string srcTitle, const string dstTitle, const string note); }; bool CTFPlot::Create(const long chart, const string name, const int subwin, const int x1, const int y1, const int x2, const int y2) { if(!CWnd::Create(chart, name, subwin, x1, y1, x2, y2)) return false; m_obj = name + "cv"; m_ok = m_cv.CreateBitmapLabel(chart, subwin, m_obj, x1, y1, x2 - x1, y2 - y1, COLOR_FORMAT_ARGB_NORMALIZE); if(m_ok) { ObjectSetInteger(chart, m_obj, OBJPROP_SELECTABLE, false); Hint("Press Preview to see the whole range:", "source on top, result below."); } return m_ok; } bool CTFPlot::OnMove(void) { if(!m_ok) return true; ObjectSetInteger(m_chart_id, m_obj, OBJPROP_XDISTANCE, Left()); ObjectSetInteger(m_chart_id, m_obj, OBJPROP_YDISTANCE, Top()); return true; } void CTFPlot::Hint(const string a, const string b) { if(!m_ok) return; m_cv.Erase(ColorToARGB(clrWhite, 255)); m_cv.Rectangle(0, 0, m_cv.Width() - 1, m_cv.Height() - 1, ColorToARGB(C'150,150,150', 255)); m_cv.FontSet("Arial", -100); m_cv.TextOut(m_cv.Width() / 2, m_cv.Height() / 2 - 14, a, ColorToARGB(C'90,90,90', 255), TA_CENTER); m_cv.TextOut(m_cv.Width() / 2, m_cv.Height() / 2 + 6, b, ColorToARGB(C'90,90,90', 255), TA_CENTER); m_cv.Update(); } void CTFPlot::Para(const string a, const string b) { if(!m_ok) return; m_cv.Erase(0); m_cv.FontSet("Arial", -80); const uint clr = ColorToARGB(C'40,40,40', 255); int y = 0; for(int pass = 0; pass < 2; pass++) { string rest = (pass == 0 ? a : b); while(StringLen(rest) > 0 && y < m_cv.Height() - 12) { string line = rest; if(StringLen(rest) > 92) { int cut = 92; while(cut > 40 && StringGetCharacter(rest, cut) != ' ') cut--; line = StringSubstr(rest, 0, cut); rest = StringSubstr(rest, cut + 1); } else rest = ""; m_cv.TextOut(0, y, line, clr, TA_LEFT); y += 14; } } m_cv.Update(); } void CTFPlot::Lines(const string a, const string b, const string c) { if(!m_ok) return; m_cv.Erase(0); m_cv.FontSet("Arial", -70); const uint clr = ColorToARGB(C'60,60,60', 255); m_cv.TextOut(0, 0, a, clr, TA_LEFT); if(b != "") m_cv.TextOut(0, 16, b, clr, TA_LEFT); if(c != "") m_cv.TextOut(0, 32, c, clr, TA_LEFT); m_cv.Update(); } void CTFPlot::Panel(const int top, const int h, const string title, const datetime &t[], const double &v[], const int n, const long shift, const uint clr) { const int L = 62, R = m_cv.Width() - 10, B = top + h - 22, T = top + 18; double lo = v[0], hi = v[0]; for(int i = 1; i < n; i++) { if(v[i] < lo) lo = v[i]; if(v[i] > hi) hi = v[i]; } double pad = (hi - lo) * 0.05; if(pad <= 0.0) pad = MathMax(MathAbs(hi) * 0.01, 1e-9); lo -= pad; hi += pad; m_cv.FontSet("Arial", -80); m_cv.TextOut(L, top, title, ColorToARGB(C'30,30,30', 255), TA_LEFT); uint grid = ColorToARGB(C'225,225,225', 255), axis = ColorToARGB(C'120,120,120', 255); for(int g = 0; g <= 4; g++) { int y = B - (B - T) * g / 4; m_cv.Line(L, y, R, y, grid); m_cv.TextOut(L - 4, y - 6, DoubleToString(lo + (hi - lo) * g / 4.0, hi > 1000 ? 0 : hi > 10 ? 2 : 5), ColorToARGB(C'80,80,80', 255), TA_RIGHT); } m_cv.Rectangle(L, T, R, B, axis); for(int g = 0; g <= 4; g++) { int idx = (n - 1) * g / 4, x = L + (R - L) * idx / MathMax(n - 1, 1); m_cv.Line(x, B, x, B + 3, axis); m_cv.TextOut(x, B + 4, TimeToString((datetime)(t[idx] + shift), TIME_DATE), ColorToARGB(C'80,80,80', 255), g == 0 ? TA_LEFT : g == 4 ? TA_RIGHT : TA_CENTER); } int px = 0, py = 0; for(int i = 0; i < n; i++) { int x = L + (int)((double)(R - L) * i / MathMax(n - 1, 1)); int y = B - (int)((B - T) * (v[i] - lo) / (hi - lo)); if(i > 0) { m_cv.Line(px, py, x, y, clr); m_cv.Line(px, py + 1, x, y + 1, clr); } px = x; py = y; } } void CTFPlot::Draw(const datetime &t[], const double &o[], const double &v[], const int n, const long shift, const string srcTitle, const string dstTitle, const string note) { if(!m_ok || n < 2) return; m_cv.Erase(ColorToARGB(clrWhite, 255)); m_cv.Rectangle(0, 0, m_cv.Width() - 1, m_cv.Height() - 1, ColorToARGB(C'150,150,150', 255)); const int h = (m_cv.Height() - 30) / 2; Panel(4, h, srcTitle, t, o, n, 0, ColorToARGB(C'90,90,90', 255)); Panel(4 + h + 4, h, dstTitle, t, v, n, shift, ColorToARGB(C'20,110,60', 255)); m_cv.FontSet("Arial", -70); m_cv.TextOut(8, m_cv.Height() - 16, note, ColorToARGB(C'90,90,90', 255), TA_LEFT); m_cv.Update(); } //+------------------------------------------------------------------+ class CTickForgeDlg : public CAppDialog { private: CLabel m_l[16]; CComboBox m_src, m_mode, m_unit, m_preset; CEdit m_symname, m_from, m_to, m_off, m_fn; CTFPlot m_hint, m_paths; // canvas text blocks (a CLabel clips long text) CCheckBox m_spread, m_digits; CButton m_btnPrev, m_btnScan, m_btnFull, m_btnGo, m_btnAudit, m_btnDel; CLabel m_avail, m_prev; CTFPlot m_status; // wrapped status text (a CLabel clips long text) string m_sa, m_sb; CTFPlot m_plot; bool m_busy; bool m_closed; CTFEngine m_eng; // stepwise generator, driven by Tick() from the host timer CTFResult m_res; bool m_running; bool m_audit; // audit-set run in progress int m_ai; // index of the audit symbol being written CTFJob m_ajob; string m_abase; string m_confirm, m_jname; // overwrite confirmation (second click), symbol being written bool m_scanning; datetime m_sm, m_sto, m_srs, m_sl; int m_srun; string m_sres, m_ssrc; bool AddLabel(const int idx, const string text, const int y); bool BuildJob(CTFJob &job, string &err); void RefreshAvail(); void RefreshPaths(); void Say(const string a, const string b = ""); void OnChangeSource(); void OnChangePreset(); void OnEndEditName() { ClearConfirm(); RefreshPaths(); } void ClearConfirm() { if(m_confirm != "") { m_confirm = ""; m_btnGo.Text("GENERATE SYMBOL"); AuditButtonsReset(); } } void ScanStep(); void OnClickFull(); void OnClickPreview(); void OnClickScan(); void OnClickGenerate(); void OnClickAudit(); void OnClickDeleteAudit(); bool StartAuditJob(); bool WriteManifest(); void AuditButtonsReset(); public: CTickForgeDlg() : m_busy(false), m_closed(false), m_running(false), m_audit(false), m_ai(0), m_scanning(false), m_srun(0) {} virtual bool Create(const long chart, const string name, const int subwin, const int x1, const int y1, const int x2, const int y2); virtual bool OnEvent(const int id, const long &lparam, const double &dparam, const string &sparam); virtual void OnClickButtonClose(void) { m_closed = true; } // the host program removes itself bool Closed() const { return m_closed; } void Tick(); // call from the host timer (about every 30 ms) void SetStatus(const string s) { m_sa = s; m_status.Para(m_sa, m_sb); ChartRedraw(); } string Status() { return m_sa; } bool Busy() const { return m_running || m_scanning; } // developer hook: fill the form and press GENERATE exactly as a user would (twice, to pass the overwrite confirmation) void FillAndGenerate(const string src, const string name, const string from, const string to, const int mode, const double offh, const string fn) { m_src.SelectByText(src); OnChangeSource(); m_symname.Text(name); m_from.Text(from); m_to.Text(to); m_mode.Select(mode); m_unit.Select(0); m_off.Text(DoubleToString(offh, 2)); m_fn.Text(fn); OnClickGenerate(); OnClickGenerate(); } void FillAndAudit(const string src, const string name, const string from, const string to, const int mode) { m_src.SelectByText(src); OnChangeSource(); m_symname.Text(name); m_from.Text(from); m_to.Text(to); m_mode.Select(mode); m_unit.Select(0); m_off.Text("0"); m_fn.Text("p"); OnClickAudit(); OnClickAudit(); } void ShotPreview(const string src, const string from, const string to, const double offh, const string fn) { if(src != "") m_src.SelectByText(src); OnChangeSource(); if(from != "") m_from.Text(from); if(to != "") m_to.Text(to); m_off.Text(DoubleToString(offh, 2)); for(int i = 0; i < ArraySize(g_presets_fn); i++) if(g_presets_fn[i] == fn) m_preset.Select(i); m_fn.Text(fn); OnClickPreview(); } }; EVENT_MAP_BEGIN(CTickForgeDlg) ON_EVENT(ON_CHANGE, m_src, OnChangeSource) ON_EVENT(ON_CHANGE, m_preset, OnChangePreset) ON_EVENT(ON_END_EDIT, m_symname, OnEndEditName) ON_EVENT(ON_CLICK, m_btnFull, OnClickFull) ON_EVENT(ON_CLICK, m_btnPrev, OnClickPreview) ON_EVENT(ON_CLICK, m_btnScan, OnClickScan) ON_EVENT(ON_CLICK, m_btnGo, OnClickGenerate) ON_EVENT(ON_CLICK, m_btnAudit, OnClickAudit) ON_EVENT(ON_CLICK, m_btnDel, OnClickDeleteAudit) EVENT_MAP_END(CAppDialog) CTickForgeDlg *g_dlg = NULL; void TF_Status(const string s) { Print("TickForge: ", s); if(g_dlg != NULL) g_dlg.SetStatus(s); } bool CTickForgeDlg::AddLabel(const int idx, const string text, const int y) { if(!m_l[idx].Create(m_chart_id, m_name + "L" + IntegerToString(idx), m_subwin, TF_X, y + 4, TF_X + TF_LBL_W, y + 24)) return false; m_l[idx].Text(text); return Add(m_l[idx]); } bool CTickForgeDlg::Create(const long chart, const string name, const int subwin, const int x1, const int y1, const int x2, const int y2) { if(!CAppDialog::Create(chart, name, subwin, x1, y1, x2, y2)) return false; const int X2 = TF_X + TF_LBL_W + 5; // control column const int XR = TF_CW - 30; // right edge int y = 8; AddLabel(0, "Source symbol", y); m_src.Create(m_chart_id, m_name + "src", m_subwin, X2, y, XR, y + 24); Add(m_src); for(int i = 0, k = 0; i < SymbolsTotal(true); i++) { string s = SymbolName(i, true); if((bool)SymbolInfoInteger(s, SYMBOL_CUSTOM)) continue; m_src.ItemAdd(s, k++); } m_src.Select(0); string cur = Symbol(); m_src.SelectByText(cur); y += 30; AddLabel(1, "Custom symbol name", y); m_symname.Create(m_chart_id, m_name + "nm", m_subwin, X2, y, XR, y + 24); Add(m_symname); y += 30; AddLabel(2, "Broker data available", y); m_avail.Create(m_chart_id, m_name + "av", m_subwin, X2, y + 4, XR, y + 24); m_avail.Text(""); Add(m_avail); y += 30; AddLabel(3, "From (YYYY.MM.DD)", y); m_from.Create(m_chart_id, m_name + "fr", m_subwin, X2, y, X2 + 110, y + 24); Add(m_from); m_l[12].Create(m_chart_id, m_name + "L12", m_subwin, X2 + 120, y + 4, X2 + 170, y + 24); m_l[12].Text("to"); Add(m_l[12]); m_to.Create(m_chart_id, m_name + "to", m_subwin, X2 + 140, y, X2 + 250, y + 24); Add(m_to); m_btnFull.Create(m_chart_id, m_name + "bf", m_subwin, X2 + 262, y, XR, y + 24); m_btnFull.Text("Use full range"); Add(m_btnFull); y += 30; AddLabel(4, "Data type", y); m_mode.Create(m_chart_id, m_name + "md", m_subwin, X2, y, XR, y + 24); Add(m_mode); m_mode.ItemAdd("Real ticks (transformed) + M1 bars", 0); m_mode.ItemAdd("M1 bars only (tester builds the ticks)", 1); m_mode.Select(0); y += 30; AddLabel(5, "Time offset", y); m_off.Create(m_chart_id, m_name + "of", m_subwin, X2, y, X2 + 90, y + 24); m_off.Text("0"); Add(m_off); m_unit.Create(m_chart_id, m_name + "un", m_subwin, X2 + 100, y, X2 + 200, y + 24); Add(m_unit); m_unit.ItemAdd("hours", 0); m_unit.ItemAdd("days", 1); m_unit.Select(0); m_l[13].Create(m_chart_id, m_name + "L13", m_subwin, X2 + 210, y + 4, XR, y + 24); m_l[13].Text("raw clock shift, - = earlier"); Add(m_l[13]); y += 30; AddLabel(6, "Price function preset", y); m_preset.Create(m_chart_id, m_name + "ps", m_subwin, X2, y, XR, y + 24); Add(m_preset); for(int i = 0; i < ArraySize(g_presets_txt); i++) m_preset.ItemAdd(g_presets_txt[i], i); m_preset.Select(0); y += 30; AddLabel(7, "New price = f(...)", y); m_fn.Create(m_chart_id, m_name + "fn", m_subwin, X2, y, XR, y + 24); m_fn.Text("p"); Add(m_fn); y += 26; m_hint.Create(m_chart_id, m_name + "hint", m_subwin, TF_X, y, XR, y + 16); Add(m_hint); m_hint.Lines("p price | p0 first price | r = p/p0 | t 0..1 over range | d days + - * / ^ ( ) abs sqrt ln log10 exp sin cos pow min max"); y += 26; m_spread.Create(m_chart_id, m_name + "sp", m_subwin, X2, y, XR, y + 22); m_spread.Text("Spread follows the price factor"); m_spread.Checked(true); Add(m_spread); y += 24; m_digits.Create(m_chart_id, m_name + "dg", m_subwin, X2, y, XR, y + 22); m_digits.Text("Add digits when prices shrink"); m_digits.Checked(true); Add(m_digits); y += 30; AddLabel(8, "Old -> new price", y); m_prev.Create(m_chart_id, m_name + "pv", m_subwin, X2, y + 4, XR, y + 24); m_prev.Text(""); Add(m_prev); y += 34; m_btnPrev.Create(m_chart_id, m_name + "b1", m_subwin, TF_X, y, TF_X + 120, y + 28); m_btnPrev.Text("Preview"); Add(m_btnPrev); m_btnScan.Create(m_chart_id, m_name + "b2", m_subwin, TF_X + 130, y, TF_X + 290, y + 28); m_btnScan.Text("Scan tick availability"); Add(m_btnScan); m_btnGo.Create(m_chart_id, m_name + "b3", m_subwin, TF_X + 300, y, XR, y + 28); m_btnGo.Text("GENERATE SYMBOL"); m_btnGo.ColorBackground(C'30,120,60'); m_btnGo.Color(clrWhite); Add(m_btnGo); y += 34; m_btnAudit.Create(m_chart_id, m_name + "b4", m_subwin, TF_X, y, TF_X + 250, y + 28); m_btnAudit.Text("Audit set (7 symbols)"); Add(m_btnAudit); m_btnDel.Create(m_chart_id, m_name + "b5", m_subwin, TF_X + 260, y, XR, y + 28); m_btnDel.Text("Delete audit set"); Add(m_btnDel); y += 38; m_paths.Create(m_chart_id, m_name + "paths", m_subwin, TF_X, y, XR, y + 50); Add(m_paths); y += 52; m_status.Create(m_chart_id, m_name + "status", m_subwin, TF_X, y, XR, y + 60); Add(m_status); Say("Ready."); m_plot.Create(m_chart_id, m_name + "plot", m_subwin, TF_CW + 10, 8, TF_W - 40, 8 + TF_PLOT_H); Add(m_plot); OnChangeSource(); return true; } void CTickForgeDlg::Say(const string a, const string b) { m_sa = a; m_sb = b; m_status.Para(a, b); ChartRedraw(); } void CTickForgeDlg::OnChangeSource() { ClearConfirm(); string s = m_src.Select(); if(s != "") m_symname.Text(s + "_TF"); RefreshAvail(); RefreshPaths(); } void CTickForgeDlg::OnChangePreset() { int i = (int)m_preset.Value(); if(i >= 0 && i < ArraySize(g_presets_fn)) m_fn.Text(g_presets_fn[i]); } void CTickForgeDlg::RefreshPaths() { string n = m_symname.Text(), root = TerminalInfoString(TERMINAL_DATA_PATH); if(StringLen(root) > 92) root = StringSubstr(root, 0, 28) + " ... " + StringSubstr(root, StringLen(root) - 58); m_paths.Lines("Data folder: " + root, "Bars stored in: \\bases\\Custom\\history\\" + n, "Ticks stored in: \\bases\\Custom\\ticks\\" + n); } void CTickForgeDlg::RefreshAvail() { string s = m_src.Select(); if(s == "") return; SymbolSelect(s, true); long a = SeriesInfoInteger(s, PERIOD_M1, SERIES_SERVER_FIRSTDATE); long b = SeriesInfoInteger(s, PERIOD_M1, SERIES_LASTBAR_DATE); if(a <= 0) { Bars(s, PERIOD_M1); a = SeriesInfoInteger(s, PERIOD_M1, SERIES_SERVER_FIRSTDATE); b = SeriesInfoInteger(s, PERIOD_M1, SERIES_LASTBAR_DATE); } if(a > 0) { m_avail.Text("M1 bars " + TimeToString((datetime)a, TIME_DATE) + " .. " + TimeToString((datetime)b, TIME_DATE) + " (ticks: use Scan)"); m_from.Text(TimeToString((datetime)MathMax(a, (long)(b - 30 * 86400)), TIME_DATE)); m_to.Text(TimeToString((datetime)b, TIME_DATE)); } else m_avail.Text("no M1 history yet - open a chart of this symbol first"); } void CTickForgeDlg::OnClickFull() { string s = m_src.Select(); long a = SeriesInfoInteger(s, PERIOD_M1, SERIES_SERVER_FIRSTDATE); long b = SeriesInfoInteger(s, PERIOD_M1, SERIES_LASTBAR_DATE); if(a > 0) { m_from.Text(TimeToString((datetime)a, TIME_DATE)); m_to.Text(TimeToString((datetime)b, TIME_DATE)); } } bool CTickForgeDlg::BuildJob(CTFJob &job, string &err) { err = ""; job.src = m_src.Select(); job.name = m_symname.Text(); StringTrimLeft(job.name); StringTrimRight(job.name); if(job.src == "") { err = "choose a source symbol"; return false; } if(StringLen(job.name) == 0 || StringLen(job.name) > 31) { err = "symbol name must be 1-31 characters"; return false; } for(int i = 0; i < StringLen(job.name); i++) { ushort c = job.name[i]; bool ok = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_' || c == '.' || c == '-' || c == '#'; if(!ok) { err = "symbol name: letters, digits, _ . - # only"; return false; } } job.from = StringToTime(m_from.Text() + " 00:00"); job.to = StringToTime(m_to.Text() + " 00:00"); if(job.from <= 0 || job.to <= 0) { err = "dates must look like 2024.03.15"; return false; } if(job.to < job.from) { err = "end date is before start date"; return false; } job.mode = (int)m_mode.Value(); double v = StringToDouble(m_off.Text()); job.offset_sec = (long)MathRound(v * (m_unit.Value() == 1 ? 86400.0 : 3600.0)); job.expr = m_fn.Text(); job.scale_spread = m_spread.Checked(); job.auto_digits = m_digits.Checked(); return true; } void CTickForgeDlg::OnClickPreview() { CTFJob job; string err; if(!BuildJob(job, err)) { Say("Cannot preview: " + err); return; } Say("Reading price history for the overview..."); CTFEngine eng; datetime t[]; double o[], v[]; string tfn; if(!eng.Series(job, t, o, v, tfn, err)) { Say("Cannot preview: " + err); m_prev.Text(""); m_plot.Hint("Cannot preview:", err); return; } int n = ArraySize(t); int dg = (int)SymbolInfoInteger(job.src, SYMBOL_DIGITS); m_prev.Text("start " + DoubleToString(o[0], dg) + " -> " + DoubleToString(v[0], dg) + " end " + DoubleToString(o[n - 1], dg) + " -> " + DoubleToString(v[n - 1], dg)); string off = (job.offset_sec == 0 ? "no time shift" : StringFormat("shifted %+.1f h", job.offset_sec / 3600.0)); m_plot.Draw(t, o, v, n, job.offset_sec, "Source " + job.src + " (broker dates)", "Result " + job.name + " f(p) = " + job.expr + " " + off, StringFormat("%d points, %s closes, whole range. Each panel has its own price axis.", n, tfn)); if(job.offset_sec > 0 && (long)job.to + 86400 + job.offset_sec > (long)TimeCurrent()) Say("Preview OK - check the shape in the panel on the right.", "NOTE: with this shift the data ends after today; the Strategy Tester can only use the part up to today."); else Say("Preview OK - check the shape in the panel on the right.", ""); } void CTickForgeDlg::OnClickScan() { if(m_scanning || m_running) return; CTFJob job; string err; if(!BuildJob(job, err)) { Say("Cannot scan: " + err); return; } m_ssrc = job.src; m_sto = job.to; m_sres = ""; m_srun = 0; m_srs = 0; m_sl = 0; MqlDateTime d; TimeToStruct(job.from, d); d.day = 1; d.hour = d.min = d.sec = 0; m_sm = StructToTime(d); m_scanning = true; m_btnScan.Text("Scanning..."); Say("Scanning tick history month by month (may download from the broker)..."); } void CTickForgeDlg::ScanStep() { MqlTick tk[]; MqlDateTime e; TimeToStruct(m_sm, e); e.mon++; if(e.mon > 12) { e.mon = 1; e.year++; } datetime nx = StructToTime(e); bool has = (CopyTicks(m_ssrc, tk, COPY_TICKS_ALL, (ulong)m_sm * 1000, 1) > 0 && tk[0].time < nx); if(has) { if(m_srun == 0) m_srs = m_sm; m_srun++; m_sl = m_sm; } else if(m_srun > 0) { m_sres += TimeToString(m_srs, TIME_DATE) + ".." + TimeToString(m_sl, TIME_DATE) + " "; m_srun = 0; } m_sm = nx; if(m_sm <= m_sto) return; if(m_srun > 0) m_sres += TimeToString(m_srs, TIME_DATE) + ".." + TimeToString(m_sl, TIME_DATE); m_scanning = false; m_btnScan.Text("Scan tick availability"); Say(m_sres == "" ? "No ticks for this range - use 'M1 bars only'." : "Months with ticks: " + m_sres, "Format: first day of first month .. first day of last month"); } void CTickForgeDlg::OnClickGenerate() { if(m_running || m_scanning) return; CTFJob job; string err; if(!BuildJob(job, err)) { Say("Cannot generate: " + err); return; } bool custom, exists = TF_SymbolExists(job.name, custom); if(exists && !custom) { Say("'" + job.name + "' is a real broker symbol - choose another name."); return; } if(exists && m_confirm != job.name) { m_confirm = job.name; m_btnGo.Text("CONFIRM OVERWRITE"); Say("Custom symbol '" + job.name + "' already exists.", "Press the button again to overwrite all of its bars and ticks, or change the name."); return; } ClearConfirm(); m_btnGo.Text("Working..."); Say("Starting..."); m_jname = job.name; if(!m_eng.Begin(job, m_res)) { m_btnGo.Text("GENERATE SYMBOL"); Say("FAILED: " + m_res.err, "Nothing was changed."); return; } m_running = true; } //--- audit set: control + calendar shifts + price level + mirror, one confirmed click --------------- string g_audit_sfx[] = {"_C", "_S7", "_S14", "_S28", "_L50", "_L200", "_MIR"}; string g_audit_lbl[] = {"control (no change)", "shifted +7 days", "shifted +14 days", "shifted +28 days", "prices x0.5", "prices x2", "mirrored moves"}; string g_audit_fn[] = {"p", "p", "p", "p", "p*0.5", "p*2", "p0*p0/p"}; int g_audit_day[] = {0, 7, 14, 28, 0, 0, 0}; void CTickForgeDlg::AuditButtonsReset() { m_btnAudit.Text("Audit set (7 symbols)"); m_btnDel.Text("Delete audit set"); } void CTickForgeDlg::OnClickAudit() { if(m_running || m_scanning) return; CTFJob job; string err; if(!BuildJob(job, err)) { Say("Cannot build the audit set: " + err); return; } if(StringLen(job.name) > 25) { Say("Cannot build the audit set: the name must be 25 characters or less (suffixes are added)."); return; } const int n = ArraySize(g_audit_sfx); int existing = 0; for(int i = 0; i < n; i++) { bool custom, ex = TF_SymbolExists(job.name + g_audit_sfx[i], custom); if(ex && !custom) { Say("'" + job.name + g_audit_sfx[i] + "' is a real broker symbol - choose another base name."); return; } if(ex) existing++; } const string key = "AUDIT:" + job.name; if(m_confirm != key) { ClearConfirm(); m_confirm = key; m_btnAudit.Text("CONFIRM AUDIT SET"); const int days = (int)((job.to - job.from) / 86400) + 1; Say(StringFormat("Audit set: %d symbols (%s_C, _S7, _S14, _S28, _L50, _L200, _MIR), %d days of %s each.", n, job.name, days, job.mode == TF_MODE_TICKS ? "real ticks" : "M1 bars"), (existing > 0 ? StringFormat("%d already exist and will be overwritten. ", existing) : "") + ((long)job.to + 86400 + 28 * 86400 > (long)TimeCurrent() ? "NOTE: the +14/+28 day symbols end after today; the tester only uses data up to today, so end the range earlier for a full comparison. " : "") + "Each symbol uses the disk space of one generated symbol. Press again to create; 'Delete audit set' removes them."); return; } ClearConfirm(); m_ajob = job; m_abase = job.name; m_ai = 0; m_audit = true; StartAuditJob(); } bool CTickForgeDlg::StartAuditJob() { CTFJob job = m_ajob; job.name = m_abase + g_audit_sfx[m_ai]; job.offset_sec = (long)g_audit_day[m_ai] * 86400; job.expr = g_audit_fn[m_ai]; m_jname = job.name; m_btnAudit.Text(StringFormat("Audit %d/%d...", m_ai + 1, ArraySize(g_audit_sfx))); Say(StringFormat("Audit set %d/%d: %s - %s", m_ai + 1, ArraySize(g_audit_sfx), job.name, g_audit_lbl[m_ai])); if(!m_eng.Begin(job, m_res)) { Say("FAILED at " + job.name + ": " + m_res.err, "Symbols made so far stay. Fix the cause and press Audit set again."); m_audit = false; m_running = false; AuditButtonsReset(); return false; } m_running = true; return true; } bool CTickForgeDlg::WriteManifest() { FolderCreate("TickForge"); // inside MQL5\Files const string fn = "TickForge\\audit_" + m_abase + ".json"; const int h = FileOpen(fn, FILE_WRITE | FILE_TXT | FILE_ANSI); if(h == INVALID_HANDLE) return false; const int n = ArraySize(g_audit_sfx); FileWriteString(h, "{\n \"tool\": \"Tick Forge\",\n \"version\": \"1.04\",\n"); FileWriteString(h, " \"source\": \"" + m_ajob.src + "\",\n"); FileWriteString(h, " \"from\": \"" + TimeToString(m_ajob.from, TIME_DATE) + "\",\n"); FileWriteString(h, " \"to\": \"" + TimeToString(m_ajob.to, TIME_DATE) + "\",\n"); FileWriteString(h, StringFormat(" \"mode\": %d,\n \"scale_spread\": %s,\n \"symbols\": [\n", m_ajob.mode, m_ajob.scale_spread ? "true" : "false")); for(int i = 0; i < n; i++) FileWriteString(h, StringFormat(" {\"name\": \"%s\", \"label\": \"%s\", \"offset_sec\": %d, \"expr\": \"%s\"}%s\n", m_abase + g_audit_sfx[i], g_audit_lbl[i], g_audit_day[i] * 86400, g_audit_fn[i], i < n - 1 ? "," : "")); FileWriteString(h, " ]\n}\n"); FileClose(h); return true; } void CTickForgeDlg::OnClickDeleteAudit() { if(m_running || m_scanning) return; CTFJob job; string err; if(!BuildJob(job, err)) { Say("Cannot delete: " + err); return; } const int n = ArraySize(g_audit_sfx); int found = 0; for(int i = 0; i < n; i++) { const string nm = job.name + g_audit_sfx[i]; bool custom; if(!TF_SymbolExists(nm, custom) || !custom) continue; SymbolSelect(nm, true); if(StringFind(SymbolInfoString(nm, SYMBOL_PATH), "Tick Forge") == 0) found++; } if(found == 0) { Say("No audit-set symbols named " + job.name + "_C, _S7 ... were found."); return; } const string key = "DEL:" + job.name; if(m_confirm != key) { ClearConfirm(); m_confirm = key; m_btnDel.Text("CONFIRM DELETE"); Say(StringFormat("%d symbol(s) named %s_... and all their bars and ticks will be deleted.", found, job.name), "Only symbols created by Tick Forge are touched. Press again to delete."); return; } ClearConfirm(); int gone = 0, failed = 0; for(int i = 0; i < n; i++) { const string nm = job.name + g_audit_sfx[i]; bool custom; if(!TF_SymbolExists(nm, custom) || !custom) continue; SymbolSelect(nm, true); if(StringFind(SymbolInfoString(nm, SYMBOL_PATH), "Tick Forge") != 0) continue; SymbolSelect(nm, false); CustomRatesDelete(nm, 0, LONG_MAX); CustomTicksDelete(nm, 0, LONG_MAX); if(CustomSymbolDelete(nm)) gone++; else failed++; } FileDelete("TickForge\\audit_" + job.name + ".json"); Say(StringFormat("Deleted %d symbol(s).", gone), failed > 0 ? StringFormat("%d could not be deleted (open in a chart or the tester?). Close them and try again.", failed) : ""); } void CTickForgeDlg::Tick() { if(m_scanning) { ScanStep(); return; } if(!m_running) return; if(!m_eng.Step(m_res)) { m_running = false; const bool was_audit = m_audit; m_audit = false; m_btnGo.Text("GENERATE SYMBOL"); AuditButtonsReset(); Say("FAILED: " + m_res.err, was_audit ? "Symbols made so far stay. Press Audit set again after fixing the cause." : "The partly written symbol '" + m_jname + "' can be regenerated (it will ask before overwriting)."); return; } if(!m_eng.Done()) return; SymbolSelect(m_jname, true); Print("TickForge DONE ", m_jname, " ticks=", m_res.ticks, " bars=", m_res.bars, " digits=", m_res.digits, " p0=", m_res.p0); if(m_audit) { m_ai++; if(m_ai < ArraySize(g_audit_sfx)) { StartAuditJob(); return; } m_audit = false; m_running = false; AuditButtonsReset(); const bool wrote = WriteManifest(); Say(StringFormat("DONE: audit set %s_C ... %s_MIR (%d symbols).", m_abase, m_abase, ArraySize(g_audit_sfx)), wrote ? "Manifest written to MQL5\\Files\\TickForge. Test your EA on each symbol, control first." : "Could not write the manifest file."); RefreshPaths(); return; } m_running = false; m_btnGo.Text("GENERATE SYMBOL"); Say(StringFormat("DONE: %s ticks %I64d, M1 bars %I64d, digits %d", m_jname, m_res.ticks, m_res.bars, m_res.digits), "In the Strategy Tester choose symbol " + m_jname + ". Data lives in the folders above."); RefreshPaths(); } // ---- end TF_Gui.mqh ---- int OnInit() { PrintFormat("Tick Forge v1.04 | chart %s", _Symbol); if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION)) { Print("Tick Forge is a utility for a live chart (it creates symbols); it does nothing in the Strategy Tester."); return INIT_SUCCEEDED; } g_dlg = new CTickForgeDlg(); if(!g_dlg.Create(0, "Tick Forge 1.04", 0, 20, 20, 20 + TF_W, 20 + TF_H)) { const string need = StringFormat("Tick Forge needs a chart of at least %d x %d pixels (enlarge the window or maximise the chart).", TF_W + 40, TF_H + 40); Print(need); Comment(need); delete g_dlg; g_dlg = NULL; return INIT_SUCCEEDED; } g_dlg.Run(); EventSetMillisecondTimer(30); return INIT_SUCCEEDED; } void OnDeinit(const int reason) { EventKillTimer(); if(g_dlg != NULL) { g_dlg.Destroy(reason); delete g_dlg; g_dlg = NULL; } Comment(""); } void OnTick() {} void OnTimer() { if(g_dlg == NULL) return; g_dlg.Tick(); // one day / one month of work per call, so the window stays responsive if(g_dlg.Closed()) ExpertRemove(); } void OnChartEvent(const int id, const long &lparam, const double &dparam, const string &sparam) { if(g_dlg != NULL) g_dlg.ChartEvent(id, lparam, dparam, sparam); } //+------------------------------------------------------------------+