47a5ef338b
- Replaced standard library signal modules with custom implementations to allow for named patterns and improved voting. - Added new input parameters for module weights, allowing for optimization of individual signal contributions. - Enhanced the management of trades with new options for breakeven and management cut. - Introduced a mechanism for dynamic ranking of signal weights based on historical performance. - Improved initialization logic to ensure proper registration of filters and handling of trading conditions. - Added detailed logging for trading permissions and account status during initialization.
363 lines
18 KiB
MQL5
363 lines
18 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| WarriorJournal.mqh |
|
|
//| AnimateDread |
|
|
//| |
|
|
//| THE QUEUE. Every filter evaluation is recorded during the vote |
|
|
//| and written to SQLite on the timer, in one transaction. |
|
|
//| |
|
|
//| THE SHAPE IS THE ORIGINAL ONE and it was right the first time: |
|
|
//| collect during Direction() because that is the only place that |
|
|
//| knows what each filter said, and FLUSH ON THE TIMER because a |
|
|
//| write per evaluation would put SQLite in the tick path. What this |
|
|
//| version fixes is the two things the 2025 import got wrong - an |
|
|
//| O(n^2) sort over a bogus date key, and reading whole tables back |
|
|
//| per buffered row - both already solved upstream in 6819bb4 and |
|
|
//| ea2552e. Nothing here re-imports them. |
|
|
//| |
|
|
//| THE CONNECTION IS OPENED BY THE TIMER AND CLOSED BY THE TIMER, |
|
|
//| and during a backtest it is not closed at all. That is not an |
|
|
//| optimisation detail, it is the reason this can run in the tester: |
|
|
//| open/close per flush is what made it too slow to leave on, and |
|
|
//| "too slow to leave on" is why it was disabled in backtests for |
|
|
//| weeks - which is why no ranking corpus existed to rank from. |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef WARRIOR_SIMPLE_JOURNAL_MQH
|
|
#define WARRIOR_SIMPLE_JOURNAL_MQH
|
|
|
|
#include "DatabaseManager.mqh"
|
|
|
|
//--- One row per filter evaluation. Deliberately small: this is written on every bar by every
|
|
//--- module, so a wide struct here is a cost paid millions of times in a decade-long pass.
|
|
struct SWarriorFiring
|
|
{
|
|
datetime when;
|
|
string table; // "<filterID>_<pattern>_<Buy|Sell>"
|
|
string pattern;
|
|
string direction;
|
|
double price;
|
|
double vote; // this filter's signed contribution, before normalisation
|
|
string result; // "Profit" | "Loss" - only resolved firings are ever written
|
|
double maeR; // worst excursion against the firing, in units of the stop
|
|
double mfeR; // best excursion in its favour, same units
|
|
int barsHeld;
|
|
double rMultiple; // THE OUTCOME THAT MATTERS: +target/stop on a target, -1 on a stop
|
|
//--- WHAT ELSE WAS SAYING THE SAME THING. Stamped after the vote closes, not at Record() time.
|
|
int agree;
|
|
int oppose;
|
|
double barVote; // the ensemble's normalised result for that bar
|
|
int regime; // 0 chop / 1 trend / 2 mean-revert, AT THE FIRING
|
|
};
|
|
|
|
//--- A FIRING WAITING FOR ITS ANSWER.
|
|
//---
|
|
//--- A row that records only that a module SPOKE cannot rank anything - which is exactly why
|
|
//--- ApplyPatternWeight() sat in this repo with no caller: FetchWinLossCounts() counts
|
|
//--- result='Profit' against result='Loss', and nothing ever wrote either. So every firing becomes
|
|
//--- a virtual trade and is held here until the market answers it.
|
|
//---
|
|
//--- THE BARRIERS ARE THE EA'S OWN. Entry, stop and target come from the vote, so a pattern is
|
|
//--- scored on the trade it actually implies rather than on some separate notion of "right". A
|
|
//--- module whose signal is real but smaller than the stop it would trade behind SHOULD rank badly.
|
|
struct SWarriorPending
|
|
{
|
|
datetime when;
|
|
string table;
|
|
string pattern;
|
|
string direction; // "Buy" or "Sell"; anything else is never resolved
|
|
double entry;
|
|
double stopDist; // > 0
|
|
double targetDist; // > 0
|
|
double vote;
|
|
int agree; // filters voting the SAME way on this bar
|
|
int oppose; // filters voting the other way
|
|
double barVote; // the ensemble's normalised result
|
|
int regime;
|
|
double mae; // in price, against the firing
|
|
double mfe; // in price, in its favour
|
|
int bars;
|
|
};
|
|
|
|
//--- The schema. Columns are positional on read, so a new one goes at the END or every existing
|
|
//--- database silently re-maps. Broker time, not GMT: rows written live and rows written by a
|
|
//--- backfill landed hours apart in the same column when these disagreed.
|
|
#define WARRIOR_FIRING_SCHEMA "ts INTEGER, year INTEGER, month INTEGER, day INTEGER, " \
|
|
"dayOfWeek INTEGER, hour INTEGER, minutes INTEGER, " \
|
|
"pattern TEXT, direction TEXT, price REAL, vote REAL, " \
|
|
"result TEXT, maeR REAL, mfeR REAL, barsHeld INTEGER, " \
|
|
"rMultiple REAL, agree INTEGER, oppose INTEGER, barVote REAL, regime INTEGER"
|
|
|
|
//--- HOW LONG A FIRING MAY WAIT FOR AN ANSWER, in bars. A firing that has hit neither barrier by
|
|
//--- then is DISCARDED rather than scored: it has no outcome, and inventing one - "positive at
|
|
//--- the horizon" - would be a second, different definition of winning mixed into one column.
|
|
//--- This biases the corpus toward firings that resolve quickly, so a pattern that is right
|
|
//--- slowly looks like a pattern with no data. That is the honest cost to pay: the alternative
|
|
//--- biases the WIN RATE itself rather than the sample.
|
|
#define WARRIOR_RESOLVE_BARS 60
|
|
|
|
class CWarriorJournal
|
|
{
|
|
private:
|
|
CDatabaseManager *m_dbm;
|
|
SWarriorFiring m_buf[];
|
|
SWarriorPending m_pend[];
|
|
int m_resolved;
|
|
int m_expired;
|
|
bool m_backtest;
|
|
int m_written;
|
|
int m_dropped;
|
|
|
|
//--- Table names reach SQL, so they are built from an allow-list rather than interpolated.
|
|
//--- A filter id is author-controlled today; that is not a reason to make it a SQL injection
|
|
//--- surface tomorrow.
|
|
bool SafeName(const string s) const
|
|
{
|
|
const int n = StringLen(s);
|
|
if(n <= 0 || n > 64)
|
|
return false;
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
const ushort c = StringGetCharacter(s, i);
|
|
const bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
|
(c >= '0' && c <= '9') || c == '_';
|
|
if(!ok)
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public:
|
|
CWarriorJournal(void) : m_dbm(NULL), m_backtest(false),
|
|
m_written(0), m_dropped(0),
|
|
m_resolved(0), m_expired(0) {}
|
|
~CWarriorJournal(void) {}
|
|
|
|
void Bind(CDatabaseManager *dbm, const bool isBacktest)
|
|
{ m_dbm = dbm; m_backtest = isBacktest; }
|
|
int Pending(void) const { return ArraySize(m_buf); }
|
|
int Written(void) const { return m_written; }
|
|
|
|
void Record(const string filterID, const string pattern, const string direction,
|
|
const double price, const double vote,
|
|
const double stopDist, const double targetDist);
|
|
void AdvanceBar(const double high, const double low);
|
|
//--- STAMP THE BAR'S CONFLUENCE onto every firing opened on it. Called once, AFTER the vote
|
|
//--- loop closes, because "how many agreed" is not knowable while the loop is still asking.
|
|
//---
|
|
//--- This is the input the networks in this project have never been given, and the one the
|
|
//--- research says carries the edge: one trigger alone measured -2.15 bp, three or more
|
|
//--- agreeing +1.39 bp out of sample, and opposed -5.80. A journal that records WHAT fired but
|
|
//--- not WHAT ELSE was firing cannot answer the only question worth asking of it.
|
|
void StampBar(const datetime bar, const int longVotes, const int shortVotes,
|
|
const double barVote, const int regime)
|
|
{
|
|
const int n = ArraySize(m_pend);
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
if(m_pend[i].when != bar || m_pend[i].bars > 0)
|
|
continue; // only this bar's still-unscored firings
|
|
const bool isLong = (m_pend[i].direction == "Buy");
|
|
m_pend[i].agree = isLong ? longVotes : shortVotes;
|
|
m_pend[i].oppose = isLong ? shortVotes : longVotes;
|
|
m_pend[i].barVote = barVote;
|
|
m_pend[i].regime = regime;
|
|
}
|
|
}
|
|
bool Flush(void);
|
|
void Report(void) const
|
|
{
|
|
//--- RESOLVED AND EXPIRED ARE BOTH REPORTED. A corpus that looks thin can be thin for two
|
|
//--- opposite reasons - few firings, or firings that never answer inside the horizon - and
|
|
//--- the ranking layer behaves completely differently in each case.
|
|
PrintFormat("CWarriorJournal: %d row(s) written, %d resolved, %d expired unanswered, "
|
|
"%d refused (unsafe name), %d queued, %d still open.",
|
|
m_written, m_resolved, m_expired, m_dropped, ArraySize(m_buf), ArraySize(m_pend));
|
|
}
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
void CWarriorJournal::Record(const string filterID, const string pattern, const string direction,
|
|
const double price, const double vote,
|
|
const double stopDist, const double targetDist)
|
|
{
|
|
if(m_dbm == NULL || direction == "")
|
|
return;
|
|
if(!SafeName(filterID) || !SafeName(pattern))
|
|
{
|
|
m_dropped++;
|
|
return;
|
|
}
|
|
//--- UNRESOLVABLE FIRINGS ARE NOT RECORDED AT ALL. Without a side there is no barrier to cross,
|
|
//--- and without a positive stop the R units below are a division by nothing. A row that can
|
|
//--- never carry an outcome is not a cheap row, it is a permanent 'NA' diluting every count.
|
|
if((direction != "Buy" && direction != "Sell") || stopDist <= 0.0 || targetDist <= 0.0 ||
|
|
price <= 0.0)
|
|
return;
|
|
const string table = filterID + "_" + pattern + "_" + direction;
|
|
const datetime now = TimeCurrent();
|
|
const int n = ArraySize(m_pend);
|
|
//--- IDENTITY DEDUP, same rule as the original: one row per table per bar. Direction() runs on
|
|
//--- every tick, and without this a quiet bar with 400 ticks writes 400 identical rows and the
|
|
//--- win rate becomes a tick-count weighting.
|
|
for(int i = 0; i < n; i++)
|
|
if(m_pend[i].table == table && m_pend[i].when == now)
|
|
return;
|
|
ArrayResize(m_pend, n + 1);
|
|
m_pend[n].when = now;
|
|
m_pend[n].table = table;
|
|
m_pend[n].pattern = pattern;
|
|
m_pend[n].direction = direction;
|
|
m_pend[n].entry = price;
|
|
m_pend[n].stopDist = stopDist;
|
|
m_pend[n].targetDist = targetDist;
|
|
m_pend[n].vote = vote;
|
|
m_pend[n].agree = 0;
|
|
m_pend[n].oppose = 0;
|
|
m_pend[n].barVote = 0.0;
|
|
m_pend[n].regime = 0;
|
|
m_pend[n].mae = 0.0;
|
|
m_pend[n].mfe = 0.0;
|
|
//--- -1 = CREATED ON THIS BAR, NOT YET LIVE. Direction() calls Record() and then, in the same
|
|
//--- call, AdvanceBar(High(1), Low(1)) - and High(1)/Low(1) at that moment are the bar that
|
|
//--- just CLOSED, the signal bar, which ended before this entry existed. A pending at 0 would be
|
|
//--- credited with that bar's whole range as its first excursion. Measured 2026-09-12: at a
|
|
//--- crowd-long signal the bar rose into its close, so its high is near the entry (~0 traded MFE)
|
|
//--- and its low is far below (large traded MAE) - every traded-side loss was overstated and the
|
|
//--- crowd fade priced at +0.31R on bars where the live trade made +0.03R. The first AdvanceBar
|
|
//--- after creation only flips this to 0; the next one applies the first REAL bar.
|
|
m_pend[n].bars = -1;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| ONE BAR OF THE MARKET'S ANSWER, applied to every open firing. |
|
|
//| |
|
|
//| Called once per bar by the vote with the CLOSED bar's range. Both |
|
|
//| barriers are tested against the same bar, and when a bar spans |
|
|
//| both this scores it a LOSS - the pessimistic reading. Bar data |
|
|
//| cannot say which came first, and a scorer that resolved ties in |
|
|
//| its own favour would report a win rate no live account could |
|
|
//| reproduce. Every stop-and-target study in this repo that looked |
|
|
//| too good was this assumption made the other way. |
|
|
//+------------------------------------------------------------------+
|
|
void CWarriorJournal::AdvanceBar(const double high, const double low)
|
|
{
|
|
if(high <= 0.0 || low <= 0.0 || high < low)
|
|
return;
|
|
int n = ArraySize(m_pend);
|
|
for(int i = n - 1; i >= 0; i--)
|
|
{
|
|
if(m_pend[i].bars < 0)
|
|
{ m_pend[i].bars = 0; continue; } // born this bar: the range passed in predates it
|
|
const bool isLong = (m_pend[i].direction == "Buy");
|
|
const double entry = m_pend[i].entry;
|
|
//--- Excursions, signed so that "favour" means the same thing on both sides.
|
|
const double fav = isLong ? (high - entry) : (entry - low);
|
|
const double adv = isLong ? (entry - low) : (high - entry);
|
|
if(fav > m_pend[i].mfe) m_pend[i].mfe = fav;
|
|
if(adv > m_pend[i].mae) m_pend[i].mae = adv;
|
|
m_pend[i].bars++;
|
|
|
|
const bool hitStop = (m_pend[i].mae >= m_pend[i].stopDist);
|
|
const bool hitTarget = (m_pend[i].mfe >= m_pend[i].targetDist);
|
|
string result = "";
|
|
double rMult = 0.0;
|
|
//--- THE R-MULTIPLE IS THE POINT, and the reason both barriers are carried per firing.
|
|
//--- A binary win/loss on a SYMMETRIC barrier makes expectancy identical to win rate by
|
|
//--- construction, which designs out the pattern worth finding: one that is right 30% of the
|
|
//--- time and wins three times what it loses. Scored against the trade's own stop and target,
|
|
//--- a target is +target/stop R and a stop is -1R, so mean R ranks the two correctly.
|
|
if(hitStop)
|
|
{ result = "Loss"; rMult = -1.0; } // pessimistic on a both-touched bar
|
|
else if(hitTarget)
|
|
{ result = "Profit"; rMult = m_pend[i].targetDist / m_pend[i].stopDist; }
|
|
else if(m_pend[i].bars >= WARRIOR_RESOLVE_BARS)
|
|
{
|
|
//--- Expired with no answer. Dropped, not scored - see WARRIOR_RESOLVE_BARS.
|
|
m_expired++;
|
|
ArrayRemove(m_pend, i, 1);
|
|
continue;
|
|
}
|
|
if(result == "")
|
|
continue;
|
|
const int w = ArraySize(m_buf);
|
|
ArrayResize(m_buf, w + 1);
|
|
m_buf[w].when = m_pend[i].when;
|
|
m_buf[w].table = m_pend[i].table;
|
|
m_buf[w].pattern = m_pend[i].pattern;
|
|
m_buf[w].direction = m_pend[i].direction;
|
|
m_buf[w].price = m_pend[i].entry;
|
|
m_buf[w].vote = m_pend[i].vote;
|
|
m_buf[w].result = result;
|
|
m_buf[w].maeR = m_pend[i].mae / m_pend[i].stopDist;
|
|
m_buf[w].mfeR = m_pend[i].mfe / m_pend[i].stopDist;
|
|
m_buf[w].barsHeld = m_pend[i].bars;
|
|
m_buf[w].rMultiple = rMult;
|
|
m_buf[w].agree = m_pend[i].agree;
|
|
m_buf[w].oppose = m_pend[i].oppose;
|
|
m_buf[w].barVote = m_pend[i].barVote;
|
|
m_buf[w].regime = m_pend[i].regime;
|
|
m_resolved++;
|
|
ArrayRemove(m_pend, i, 1);
|
|
}
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
bool CWarriorJournal::Flush(void)
|
|
{
|
|
const int n = ArraySize(m_buf);
|
|
if(m_dbm == NULL || n <= 0)
|
|
return true;
|
|
if(!m_dbm.BeginTransaction())
|
|
{
|
|
//--- LEFT QUEUED, NOT DROPPED. A failed flush that discarded its buffer would silently thin
|
|
//--- the corpus exactly when the database is under stress - and the gap would look like a
|
|
//--- quiet market rather than a failed write.
|
|
Print("CWarriorJournal: could not begin a transaction; ", n, " row(s) left queued for retry.");
|
|
return false;
|
|
}
|
|
string cols[] = {"ts", "year", "month", "day", "dayOfWeek", "hour", "minutes",
|
|
"pattern", "direction", "price", "vote",
|
|
"result", "maeR", "mfeR", "barsHeld", "rMultiple",
|
|
"agree", "oppose", "barVote", "regime"};
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
MqlDateTime t;
|
|
TimeToStruct(m_buf[i].when, t);
|
|
m_dbm.CreateTable(m_buf[i].table, WARRIOR_FIRING_SCHEMA); // no-op when it already exists
|
|
string vals[];
|
|
ArrayResize(vals, 20);
|
|
vals[0] = IntegerToString((long)m_buf[i].when);
|
|
vals[1] = IntegerToString(t.year);
|
|
vals[2] = IntegerToString(t.mon);
|
|
vals[3] = IntegerToString(t.day);
|
|
vals[4] = IntegerToString(t.day_of_week);
|
|
vals[5] = IntegerToString(t.hour);
|
|
vals[6] = IntegerToString(t.min);
|
|
//--- NO QUOTES. InsertTradeRecord() binds these as parameters (PlaceholderList +
|
|
//--- PrepareAndExecuteBound), so a manual "'" makes the apostrophes part of the VALUE:
|
|
//--- every row since this was written stored 'Pattern_0' rather than Pattern_0. It went
|
|
//--- unnoticed because the table NAME carries the real key and nothing ever read these two
|
|
//--- columns back - until `result` joined them and FetchWinLossCounts' result='Profit'
|
|
//--- matched nothing, leaving every weight unset and the whole ranking pass silent.
|
|
vals[7] = m_buf[i].pattern;
|
|
vals[8] = m_buf[i].direction;
|
|
vals[9] = DoubleToString(m_buf[i].price, 8);
|
|
vals[10] = DoubleToString(m_buf[i].vote, 4);
|
|
vals[11] = m_buf[i].result;
|
|
vals[12] = DoubleToString(m_buf[i].maeR, 4);
|
|
vals[13] = DoubleToString(m_buf[i].mfeR, 4);
|
|
vals[14] = IntegerToString(m_buf[i].barsHeld);
|
|
vals[15] = DoubleToString(m_buf[i].rMultiple, 4);
|
|
vals[16] = IntegerToString(m_buf[i].agree);
|
|
vals[17] = IntegerToString(m_buf[i].oppose);
|
|
vals[18] = DoubleToString(m_buf[i].barVote, 4);
|
|
vals[19] = IntegerToString(m_buf[i].regime);
|
|
if(m_dbm.InsertTradeRecord(m_buf[i].table, cols, vals))
|
|
m_written++;
|
|
}
|
|
if(!m_dbm.CommitTransaction())
|
|
{
|
|
m_dbm.RollbackTransaction();
|
|
Print("CWarriorJournal: commit failed, rolled back; ", n, " row(s) left queued for retry.");
|
|
return false;
|
|
}
|
|
ArrayResize(m_buf, 0);
|
|
return true;
|
|
}
|
|
#endif // WARRIOR_SIMPLE_JOURNAL_MQH
|