forked from animatedread/Warrior_EA
68459208bf
The warning lived inside the VerboseMode-gated corpus report, so a forgotten wipe silently voided an entire 18-year corpus run - the outdated-row guard rejected the whole replay against leftover rows and the run appended 35 rows instead of building a corpus. The check now runs unconditionally at tester OnInit (MetaCorpusStaleCheck): 52 quiet one-row newest-key probes vs the test start, with a loud stop- wipe-rerun instruction when the DB is newer than the test. Absent tables probe quietly via FetchNewestTimeKey''s new quiet flag. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
235 lines
10 KiB
MQL5
235 lines
10 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| Warrior_EA |
|
|
//| AnimateDread |
|
|
//| |
|
|
//| Meta-label training corpus - stage S1 of Meta_Labeling_Design.md |
|
|
//| |
|
|
//| The corpus is READ FROM THE SIGNAL DATABASE, not produced by a |
|
|
//| training-time re-implementation of the classic ladders. The DB |
|
|
//| logs every pattern instance the live ladders fire - both sides, |
|
|
//| uncensored, with the filter's net vote and the touchable entry |
|
|
//| price (per-side journaling, 652bf81/195be20) - so a corpus read |
|
|
//| from it is BY CONSTRUCTION identical to what fires live, and the |
|
|
//| several hundred lines of condition-mirroring a separate sweep |
|
|
//| would need (a silent-divergence trap) never get written. The |
|
|
//| cost of the trade: corpus coverage equals whatever backtest |
|
|
//| last populated the DB, so a corpus build IS a tester run with |
|
|
//| UseDatabaseRanking on and DB_MaxRowsPerTable raised. |
|
|
//| |
|
|
//| S1 ships the reader and its report; S2 attaches labels (triple- |
|
|
//| barrier at the EA's own geometry from the candidate's bar) and |
|
|
//| the setup-descriptor features. The report already measures the |
|
|
//| one mapping S2 depends on: DB rows carry GMT timestamps while |
|
|
//| bar history is in server time, so the report tries hour offsets |
|
|
//| 0..+3 and prints the exact-bar match rate of each - measured, |
|
|
//| not assumed (the broker is UTC+2-ish; DST behaviour is exactly |
|
|
//| the kind of fact this project checks empirically). |
|
|
//+------------------------------------------------------------------+
|
|
#include "..\..\Structures\tradeRecordStructure.mqh"
|
|
#include "..\..\System\PrintVerbose.mqh"
|
|
|
|
//--- One candidate = one journaled pattern instance. `win` is the DB's stop-and-reverse outcome
|
|
//--- (entry-to-reversal), kept for reporting; S2's training label is computed independently from
|
|
//--- price history (triple-barrier at the EA's geometry) and does NOT reuse this field.
|
|
struct SMetaCandidate
|
|
{
|
|
datetime gmt; // journal timestamp (GMT, minute resolution)
|
|
short family; // 0=MA 1=RSI 2=MACD 3=Ichimoku
|
|
short pattern; // Pattern_N index within the family
|
|
short side; // +1 Buy / -1 Sell
|
|
double netVote; // the filter's own vote margin when it fired (decision context)
|
|
double entryPrice; // touchable side of the spread at fire time
|
|
bool closed; // result != NA
|
|
bool win; // result == Profit (stop-and-reverse outcome, reporting only)
|
|
};
|
|
|
|
string MetaFamilyName(const int f)
|
|
{
|
|
switch(f)
|
|
{
|
|
case 0: return "MA";
|
|
case 1: return "RSI";
|
|
case 2: return "MACD";
|
|
case 3: return "Ichimoku";
|
|
}
|
|
return "";
|
|
}
|
|
|
|
int MetaFamilyPatterns(const int f)
|
|
{
|
|
switch(f)
|
|
{
|
|
case 0: return 4; // Signals\SignalMA.mqh m_pattern_count
|
|
case 1: return 4; // Signals\SignalRSI.mqh
|
|
case 2: return 6; // Signals\SignalMACD.mqh
|
|
case 3: return 12; // Signals\SignalIchimoku.mqh
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
class CMetaCorpus
|
|
{
|
|
public:
|
|
SMetaCandidate m_rows[];
|
|
int m_count;
|
|
|
|
CMetaCorpus(void) : m_count(0) {}
|
|
|
|
//--- Pull every pattern table into m_rows. Missing tables (family disabled in the run that
|
|
//--- populated this DB) are skipped and counted, not treated as errors.
|
|
int Load(void)
|
|
{
|
|
m_count = 0;
|
|
ArrayResize(m_rows, 0);
|
|
if(!dbm.OpenDatabase())
|
|
{
|
|
Print(__FUNCTION__ + ": signal database is not available - no corpus.");
|
|
return 0;
|
|
}
|
|
int missing = 0;
|
|
TradeRecord rec;
|
|
for(int f = 0; f < 4; f++)
|
|
for(int p = 0; p < MetaFamilyPatterns(f); p++)
|
|
for(int s = 0; s < 2; s++)
|
|
{
|
|
string table = MetaFamilyName(f) + "_Pattern_" + IntegerToString(p) +
|
|
(s == 0 ? "_Buy" : "_Sell");
|
|
TradeRecord rows[];
|
|
if(!dbm.FetchTradeRecords(table, rec, rows))
|
|
{
|
|
missing++;
|
|
continue;
|
|
}
|
|
int base = m_count;
|
|
ArrayResize(m_rows, base + ArraySize(rows));
|
|
for(int i = 0; i < ArraySize(rows); i++)
|
|
{
|
|
MqlDateTime t;
|
|
t.year = rows[i].year;
|
|
t.mon = rows[i].month;
|
|
t.day = rows[i].day;
|
|
t.hour = rows[i].hour;
|
|
t.min = rows[i].minutes;
|
|
t.sec = 0;
|
|
m_rows[base + i].gmt = StructToTime(t);
|
|
m_rows[base + i].family = (short)f;
|
|
m_rows[base + i].pattern = (short)p;
|
|
m_rows[base + i].side = (short)(rows[i].direction == "Buy" ? 1 : -1);
|
|
m_rows[base + i].netVote = rows[i].netVote;
|
|
m_rows[base + i].entryPrice = rows[i].entryPrice;
|
|
m_rows[base + i].closed = (rows[i].result != "NA");
|
|
m_rows[base + i].win = (rows[i].result == "Profit");
|
|
}
|
|
m_count += ArraySize(rows);
|
|
}
|
|
if(missing > 0)
|
|
PrintVerbose(__FUNCTION__ + ": " + IntegerToString(missing) +
|
|
" pattern table(s) absent (family disabled in the populating run) - skipped.");
|
|
return m_count;
|
|
}
|
|
|
|
//--- Corpus state, printed for the log: volume per family, closed fraction, span, and the
|
|
//--- GMT->server-bar offset measurement S2's label plumbing will pin itself to.
|
|
void Report(void)
|
|
{
|
|
if(m_count == 0)
|
|
{
|
|
Print("MetaCorpus: EMPTY - run a backtest with UseDatabaseRanking=true (and "
|
|
"DB_MaxRowsPerTable raised for a long window) to build the corpus.");
|
|
return;
|
|
}
|
|
datetime lo = m_rows[0].gmt, hi = m_rows[0].gmt;
|
|
int famRows[4] = {0, 0, 0, 0}, famClosed[4] = {0, 0, 0, 0}, famWins[4] = {0, 0, 0, 0};
|
|
for(int i = 0; i < m_count; i++)
|
|
{
|
|
lo = MathMin(lo, m_rows[i].gmt);
|
|
hi = MathMax(hi, m_rows[i].gmt);
|
|
famRows[m_rows[i].family]++;
|
|
if(m_rows[i].closed)
|
|
{
|
|
famClosed[m_rows[i].family]++;
|
|
if(m_rows[i].win)
|
|
famWins[m_rows[i].family]++;
|
|
}
|
|
}
|
|
Print("MetaCorpus: ", m_count, " candidates, ", TimeToString(lo, TIME_DATE), " .. ",
|
|
TimeToString(hi, TIME_DATE));
|
|
//--- The cross-run trap, caught loudly: ProcessSignal's outdated-row guard rejects any
|
|
//--- registration OLDER than a row its table already holds (correct for a live stream,
|
|
//--- append-only-forward across runs). A tester run that starts before the DB's newest row
|
|
//--- will therefore silently register NOTHING for the overlap - an 18-year corpus build
|
|
//--- against leftover recent rows yields a corpus of only the leftover (measured 2026-08-12:
|
|
//--- 3,681 rows, all 2026, from exactly that mistake). Corpus builds start from an empty DB.
|
|
if(MQLInfoInteger(MQL_TESTER) && hi > TimeCurrent())
|
|
Print("MetaCorpus: WARNING - the DB already holds rows newer than this test's start (",
|
|
TimeToString(hi, TIME_DATE), " > ", TimeToString(TimeCurrent(), TIME_DATE),
|
|
"). Historical registrations will be REJECTED by the outdated-row guard; ",
|
|
"wipe the signal DB before a corpus-building backtest.");
|
|
for(int f = 0; f < 4; f++)
|
|
{
|
|
if(famRows[f] == 0)
|
|
continue;
|
|
double wr = famClosed[f] > 0 ? 100.0 * famWins[f] / famClosed[f] : 0.0;
|
|
Print(" ", MetaFamilyName(f), ": ", famRows[f], " rows, ", famClosed[f],
|
|
" closed, S&R win rate ", DoubleToString(wr, 1), "%");
|
|
}
|
|
//--- The offset measurement. A candidate is only labelable if its timestamp resolves to an
|
|
//--- exact bar; rows are journaled at bar opens, so the RIGHT offset should match ~100% and
|
|
//--- wrong ones near-0% on an H1 chart - a clean empirical signature. Measured on every row.
|
|
Print(" GMT->server bar match rate by hour offset (S2 pins to the winner):");
|
|
for(int off = 0; off <= 3; off++)
|
|
{
|
|
int hit = 0;
|
|
for(int i = 0; i < m_count; i++)
|
|
if(iBarShift(_Symbol, _Period, m_rows[i].gmt + off * 3600, true) >= 0)
|
|
hit++;
|
|
Print(" +", off, "h: ", DoubleToString(100.0 * hit / m_count, 1), "% exact");
|
|
}
|
|
}
|
|
};
|
|
|
|
//--- OnInit hook: load + report under VerboseMode, so a corpus-building backtest is verifiable
|
|
//--- straight from its log. Deliberately report-only in S1 - nothing downstream consumes it yet.
|
|
void MetaCorpusReport(void)
|
|
{
|
|
CMetaCorpus corpus;
|
|
corpus.Load();
|
|
corpus.Report();
|
|
}
|
|
|
|
//--- ALWAYS-ON tester guard, deliberately NOT VerboseMode-gated: the stale-DB warning inside
|
|
//--- Report() was, and forgetting the wipe silently wasted a full 18-year corpus run - twice.
|
|
//--- 52 one-row queries at OnInit; absent tables (family disabled, or a fresh wipe before
|
|
//--- AddFilter creates them) probe quietly as "no rows".
|
|
void MetaCorpusStaleCheck(void)
|
|
{
|
|
if(!MQLInfoInteger(MQL_TESTER))
|
|
return;
|
|
long newest = 0;
|
|
bool any = false;
|
|
for(int f = 0; f < 4; f++)
|
|
for(int p = 0; p < MetaFamilyPatterns(f); p++)
|
|
for(int s = 0; s < 2; s++)
|
|
{
|
|
string table = MetaFamilyName(f) + "_Pattern_" + IntegerToString(p) +
|
|
(s == 0 ? "_Buy" : "_Sell");
|
|
long k = 0;
|
|
bool found = false;
|
|
if(dbm.FetchNewestTimeKey(table, k, found, true) && found)
|
|
{
|
|
newest = MathMax(newest, k);
|
|
any = true;
|
|
}
|
|
}
|
|
if(!any)
|
|
return;
|
|
MqlDateTime g;
|
|
TimeGMT(g);
|
|
long nowKey = SignalTimeKey(g.year, g.mon, g.day, g.hour, g.min);
|
|
if(newest >= nowKey)
|
|
Print("MetaCorpus: *** THE SIGNAL DB ALREADY HOLDS ROWS AT/AFTER THIS TEST'S START (key ",
|
|
newest, " >= ", nowKey, "). Every registration in the overlap will be REJECTED by the ",
|
|
"outdated-row guard and this run will build NO corpus. Stop the test, wipe the ",
|
|
"Signals folder, and rerun. ***");
|
|
}
|
|
//+------------------------------------------------------------------+
|