//+------------------------------------------------------------------+ //| Warrior_EA | //| AnimateDread | //| | //| Era loop, plateau ladder, checkpoint selection, deploy/finalise. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AIBASE_TRAINING_MQH #define WARRIOR_AIBASE_TRAINING_MQH //+------------------------------------------------------------------+ //| Does the checkpoint about to deploy survive having been CHOSEN? | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::BestCheckpointSurvivesSelection(double &zObs, double &pFamily, int &nTried) { zObs = 0.0; pFamily = 1.0; nTried = MathMax(m_deployCandidateEras, 1); //--- No ranked era yet, or a degenerate chance rate: nothing to test, so nothing to deploy. if(m_bestDirCalls <= 0 || m_bestDirPrecPct < 0.0 || m_bestChancePrecPct <= 0.0 || m_bestChancePrecPct >= 100.0) return false; //--- RAW calls, not EffectiveSampleSize(): alone among the SEs in this project this one is not //--- deflated for label overlap, which makes it the most permissive test here. Left as measured //--- rather than corrected in passing - tightening a live deploy bar is a policy change. double se = BinomialSEPct(m_bestChancePrecPct / 100.0, (double)m_bestDirCalls); if(se <= 0.0) return false; zObs = (m_bestDirPrecPct - m_bestChancePrecPct) / se; pFamily = SidakFamilyP(zObs, nTried); return (pFamily <= DEPLOY_FAMILY_WISE_ALPHA); } //+------------------------------------------------------------------+ //| ENSEMBLE GATE - the same test as above, asked of the VOTE. | //| See the ENSEMBLE DEPLOY GATE block in ExpertSignalAIBase.mqh for | //| why the vote rather than the member is the thing being gated. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::EnsembleSurvivesSelection(double &zObs, double &pFamily, int &nTried) { zObs = 0.0; pFamily = 1.0; //--- ERAS x RUNGS. The deployed configuration is the maximum over every era AND over every //--- threshold rung the derivation could have landed on (THE DERIVED THRESHOLD, this file), so //--- the correction has to span both or the gate is testing a smaller family than was searched. //--- Sidak is conservative under the positive dependence between nested rungs, which is the safe //--- direction. Measured cost: nothing - all six charts clear this by 6.5-12 sigma even when the //--- z is formed on EFFECTIVE rather than raw calls. nTried = MathMax(g_ensCandidateEras, 1) * ENS_THRESHOLD_SWEEP_N; if(g_ensBestCalls <= 0 || g_ensBestPrecPct < 0.0 || g_ensBestChancePct <= 0.0 || g_ensBestChancePct >= 100.0) return false; //--- Raw calls here too, matching the member gate above so neither is the easier one to clear. double se = BinomialSEPct(g_ensBestChancePct / 100.0, (double)g_ensBestCalls); if(se <= 0.0) return false; zObs = (g_ensBestPrecPct - g_ensBestChancePct) / se; pFamily = SidakFamilyP(zObs, nTried); return (pFamily <= DEPLOY_FAMILY_WISE_ALPHA); } //+------------------------------------------------------------------+ //| JOINT CHECKPOINT: snapshot EVERY member's weights, at this one | //| era, and commit each member's own era statistics as the stats | //| its best checkpoint is described by. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleCommitJointCheckpoint(const long votedEra) { int captured = 0, members = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; members++; //--- The member's OWN figures at the winning era. They describe this member's contribution to a //--- checkpoint the ENSEMBLE selected, which is why they are committed from the stash rather //--- than from a per-member ranking: no member "won" this era, the vote did. mm.m_bestOosForecast = mm.m_eraStatBlended; mm.m_bestSelectionScore = mm.m_eraStatScore; mm.m_bestPassedRecall = mm.m_eraStatTradeable; mm.m_bestBothSidesLive = mm.m_eraStatTwoSided; mm.m_bestDirPrecPct = mm.m_eraStatPrecPct; mm.m_bestChancePrecPct = mm.m_eraStatChancePct; mm.m_bestDirCalls = mm.m_eraStatCalls; mm.m_bestDirConfThreshold = mm.m_eraStatThreshold; //--- In-memory snapshot, same primitive the solo path uses. A member whose capture fails keeps //--- m_haveOosCheckpoint false and is reported - it would otherwise deploy whatever weights it //--- happens to hold at the end of the run, silently breaking the "deploy what was measured" //--- guarantee this whole mechanism exists for. if(CheckPointer(mm.Net) != POINTER_INVALID && mm.Net.CaptureWeights()) { mm.m_haveOosCheckpoint = true; mm.m_checkpointEra = votedEra; // the deploy gate cross-checks this against the winning era captured++; } else Print(mm.ID + ": WARNING - joint ensemble checkpoint capture FAILED at era " + IntegerToString((int)votedEra) + ". This member cannot contribute the weights the vote" " was measured with; the ensemble will not deploy a checkpoint it cannot reproduce."); //--- a new joint best retires the shared ladder for everyone mm.m_erasSinceBest = 0; mm.m_plateauStage = 0; mm.m_restartBoostErasLeft = 0; mm.m_consecutiveRegressions = 0; } //--- PARTIAL CAPTURE IS NOT A CHECKPOINT. Rolling it back lets the run carry on and simply find //--- its best again. if(captured < members) { g_ensBestScore = -1.0; g_ensBestTradeable = false; g_ensBestTwoSided = false; g_ensBestCalls = 0; g_ensBestEra = -1; //--- Cleared with the rest: a refusal must not describe an era that is no longer the best. g_ensBestCoveragePct = -1.0; g_ensBestMinCoverPct = -1.0; g_ensBestEdgeFloorPct = -1.0; Print("AI ensemble: joint checkpoint INCOMPLETE at era " + IntegerToString((int)votedEra) + " (" + IntegerToString(captured) + " of " + IntegerToString(members) + " members captured)" " - discarding this era as the best; the search continues from no joint checkpoint."); } } //+------------------------------------------------------------------+ //| Once per era, on the LAST still-training member to finish its | //| pass-3 scan: score the combined vote, rank the era, checkpoint, | //| advance the shared plateau ladder, and decide deployment. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleEraVerdict(const int needMask, const long votedEra, double &etaLocal) { int members = EnsembleBitCount(needMask); //--- One trainer left (the others deployed, paused or stopped) is not an ensemble read: the //--- "vote" would be that member's own signal and the gate would silently become the solo gate //--- under an ensemble label. Members keep training; nothing is ranked or deployed from here. if(members < 2) return; //--- SHARED BARS ONLY. A bar one member skipped (feature-window failure) has an average over a //--- different membership, which is a different quantity - averaging it in would make the score //--- depend on which member happened to fail where. int shared = 0, fired = 0, wins = 0, firedLong = 0, firedShort = 0; int dirLabelBars = 0, labelBuyBars = 0, labelSellBars = 0; //--- PER-RUNG TALLIES, accumulated in the row loop below. No longer a side diagnostic: the era's //--- own verdict is read out of these at the DERIVED rung (see THE DERIVED THRESHOLD below), so //--- these ARE the era's numbers. Long/short are split because the anti-degenerate test needs to //--- know whether the rung fired both ways, and a rung that only ever fired one side is not an //--- operating point anyone can trade however precise it looked. int sweepFired[ENS_THRESHOLD_SWEEP_N], sweepWins[ENS_THRESHOLD_SWEEP_N]; int sweepLong[ENS_THRESHOLD_SWEEP_N], sweepShort[ENS_THRESHOLD_SWEEP_N]; ArrayInitialize(sweepFired, 0); ArrayInitialize(sweepWins, 0); ArrayInitialize(sweepLong, 0); ArrayInitialize(sweepShort, 0); for(int r = 0; r < g_ensVoteRows; r++) { if((g_ensVoteMask[r] & needMask) != needMask) continue; shared++; if(g_ensVoteDirLabel[r]) dirLabelBars++; //--- zero-skill reference, measured over EVERY shared bar (see chancePrecPct's derivation in //--- the era-end block): what always-Buy and always-Sell would have scored here if(g_ensVoteLabelBuy[r]) labelBuyBars++; if(g_ensVoteLabelSell[r]) labelSellBars++; //--- THE LIVE AGGREGATION, reproduced exactly (CExpertSignalCustom::Direction(), pass 2 plus //--- the `result /= number` normalization): sum the members' signed votes, divide by how many //--- of them ACTUALLY VOTED, and compare the magnitude against Signal_ThresholdOpen on the //--- same 0..100 scale the tier weights already live on. int voters = EnsembleBitCount(g_ensVoteVoterMask[r] & needMask); if(voters <= 0 || g_ensVoteWeightSum[r] <= 0.0) continue; // every member abstained: no vote, no trade, not a fired bar double net = g_ensVoteSum[r] / g_ensVoteWeightSum[r]; //--- THRESHOLD SWEEP - PURE DIAGNOSTIC, CHANGES NOTHING. What this same era's vote would have //--- scored at the other Signal_ThresholdOpen rungs, measured on these exact rows rather than //--- modelled. It exists because the threshold is the one parameter this gate cannot reason //--- about from its own output: the refusal can say "coverage too low" but not "and here is //--- what it would be one rung down", and MT5 stores the input PER CHART (profiles\Charts\* //--- \chart*.chr), so an operator cannot cheaply A/B it either - an already-attached EA //--- ignores a changed default entirely. Accumulated before the live threshold test below so //--- the sweep sees every scored row, and gated by the same direction policy so its numbers //--- are comparable with the ones the gate actually certifies. //--- THE DIRECTION POLICY IS PART OF WHAT GETS CERTIFIED (2026-08-19). Under LONG_ONLY/ //--- SHORT_ONLY blocks live from ever placing the other side's trades - scoring them here //--- would certify a vote the EA does not cast, the exact certified!=traded defect this //--- gate was rebuilt to end (2c443ba). Hoisted above the tally (it used to sit below the //--- sweep) so EVERY rung is scored on the same population the gate will certify. if(!WarriorDirectionAllows(net > 0.0)) continue; double mag = MathAbs(net); bool swHit = (net > 0.0) ? g_ensVoteLabelBuy[r] : g_ensVoteLabelSell[r]; for(int s = 0; s < ENS_THRESHOLD_SWEEP_N; s++) if(mag >= g_ensThresholdSweep[s]) { sweepFired[s]++; if(swHit) sweepWins[s]++; if(net > 0.0) sweepLong[s]++; else sweepShort[s]++; } } bool measurable = (shared > 0 && dirLabelBars > 0); //--- The zero-skill reference must be ACHIEVABLE under the direction policy: with shorts //--- blocked, always-Sell is not a strategy anyone could run, and ranking the vote against it //--- would score a long-only book against a baseline the policy forbids. This is one of the two //--- places the vote genuinely differs from a member - see DeployGate.mqh. double chancePct = -1.0; if(measurable) { double chanceL = 100.0 * labelBuyBars / shared; double chanceS = 100.0 * labelSellBars / shared; bool allowL = WarriorDirectionAllows(true); bool allowS = WarriorDirectionAllows(false); chancePct = (allowL && allowS) ? MathMax(chanceL, chanceS) : (allowL ? chanceL : (allowS ? chanceS : MathMax(chanceL, chanceS))); } //--- The other genuine difference: the vote's anti-degenerate test reads whether it actually //--- FIRED both ways, where a member reads its per-side recalls. bool bothAllowed = (WarriorDirectionAllows(true) && WarriorDirectionAllows(false)); //================================================================================================== // THE DERIVED THRESHOLD //================================================================================================== //--- THE RULE: the HIGHEST rung whose vote still clears the WHOLE deploy gate - coverage floor, //--- exact-binomial precision bar and two-sidedness together. Not the rung with the best //--- precision. That distinction is the entire safety argument and must not be "improved" away: //--- //--- Picking the best-PRECISION rung is a best-of-6 on a noisy statistic, and this project has //--- already crowned noise that way four separate times ([[project_family_wise_gate_rule]]). //--- Picking the highest rung that PASSES orders the candidates by a fixed, data-independent //--- key (the threshold itself) and asks one pass/fail question per rung. The multiplicity is //--- real but bounded and known, so it is PAID FOR below rather than ignored: nTried in //--- EnsembleSurvivesSelection() is now eras x rungs, not eras. //--- //--- MEASURED, on 619 era verdicts across all six live charts (2026-08-26 overnight run): //--- * every era on every symbol had at least one rung that cleared the full gate. At a FIXED //--- 25% - what the fleet actually ran - four of six symbols had none, ever. The threshold, //--- not the models, was the entire reason nothing deployed. //--- * WALK-FORWARD (rung derived on era N, then scored on era N+1) lands at 10.2% coverage / //--- 31.8% precision against an ORACLE that re-picks on era N+1 itself of 10.3% / 31.7%. //--- Near-zero shrinkage, which is what says this is a measurement and not a fit. It holds //--- because the binding constraint is COVERAGE - a near-deterministic step function of the //--- vote distribution - and not precision. //--- * against a fixed 15% (the best single global value): +0.6pp precision for 3.4pp less //--- coverage. Against a fixed 20%: deployable on all six instead of four of six. //--- //--- WHY THIS IS AN OUTPUT AND NOT AN INPUT. The era verdict below is computed AT this rung, so //--- the threshold is part of what gets certified rather than a knob applied afterwards. That is //--- also why Warrior_EA.mq5 must push it into the live signal's m_threshold_open: certifying at //--- one threshold and trading at another is the defect 2c443ba was written to end, and MT5 //--- stores an input PER CHART (profiles\Charts\*\chart*.chr) - so as an INPUT this number //--- could never be corrected from source at all. int derivedIdx = -1, coverageIdx = -1; if(measurable) { for(int s = ENS_THRESHOLD_SWEEP_N - 1; s >= 0; s--) { if(sweepFired[s] <= 0) continue; double swPrec = 100.0 * sweepWins[s] / sweepFired[s]; bool swTwo = bothAllowed ? (sweepLong[s] > 0 && sweepShort[s] > 0) : (sweepFired[s] > 0); SDeployVerdict rungGate; rungGate.EvaluateRates(sweepFired[s], shared, dirLabelBars, swPrec, chancePct, EffectiveSampleSize((double)sweepFired[s]), swTwo); //--- Remembered on the way down so the fallback below can reach for it without a second scan. //--- FIRST WRITE WINS, and the guard is load-bearing: this loop runs HIGH rung to LOW and //--- coverage only rises as the threshold falls, so without it every clearing rung would //--- overwrite the last and the fallback would end up holding the LOWEST clearing rung - //--- the exact opposite of what it is documented to do. if(coverageIdx < 0 && rungGate.coveragePct >= rungGate.minCoveragePct) coverageIdx = s; if(rungGate.tradeable) { derivedIdx = s; break; } } } //--- FALLBACK, when no rung clears the gate. Take the highest rung that at least clears COVERAGE, //--- so the era fails on precision - the informative failure, and the one the refusal text can act //--- on - rather than on a thin population that inflates its own bar. If not even the lowest rung //--- covers enough, take the lowest: maximum evidence is the only thing left worth having. if(derivedIdx < 0) derivedIdx = (coverageIdx >= 0) ? coverageIdx : 0; //--- FOLLOW UNTIL PINNED, PIN THEREAFTER. Once a checkpoint exists the live rung belongs to it and //--- is set in the isBetter block below. BEFORE that there is nothing to protect, and leaving the //--- chart on the Signal_ThresholdOpen seed is strictly worse than following the latest era's own //--- measurement. //--- //--- IT WAS ALSO A LIVE DEFECT, from combining the pin with the burn-in: no era below //--- ENSEMBLE_CHECKPOINT_MIN_ERA may checkpoint, so nothing published until era 20 and the chart //--- sat on the seed until then. Under the edge-over-chance currency that seed (25, an ABSOLUTE //--- win rate) is above what the vote can even reach - USDJPY's strongest vote was 19.3% against //--- it - so those charts drew ZERO arrows and would have placed zero trades. Reported as "all 3 //--- forex charts are visibly quiet" while their OOS coverage read 17-18%. if(measurable && g_ensBestEra < 0) { g_ensDerivedThreshold = g_ensThresholdSweep[derivedIdx]; g_ensembleVoteThreshold = g_ensDerivedThreshold; } //--- Every era derives its own rung - that is how the best one is found - but once a checkpoint //--- exists the rung the LIVE SIGNAL trades is PINNED to the checkpointed era's, in the isBetter //--- block. Measured over 619 eras, the per-era rung moves on 6-34% of steps (always by one rung); //--- publishing each era's would make the deployed operating point chase noise between eras that //--- were never chosen, and the live run showed exactly that within a minute of starting //--- (SP500 15 -> 10 -> 15). The threshold belongs to the CHECKPOINT, like the weights do. fired = sweepFired[derivedIdx]; wins = sweepWins[derivedIdx]; firedLong = sweepLong[derivedIdx]; firedShort = sweepShort[derivedIdx]; double votePrecPct = (fired > 0) ? 100.0 * wins / fired : -1.0; bool twoSided = bothAllowed ? (firedLong > 0 && firedShort > 0) : (fired > 0); //--- THE SAME ARITHMETIC THE MEMBER GATE RUNS, on the vote's population. EFFECTIVE sample and //--- not the raw fire count: the vote's outcomes are overlapping triple-barrier labels exactly //--- as a member's are, and the two gates applying different corrections is precisely how the //--- ensemble becomes the easier one to clear. SDeployVerdict voteGate; voteGate.EvaluateRates(fired, shared, dirLabelBars, votePrecPct, chancePct, EffectiveSampleSize((double)fired), twoSided); double coveragePct = voteGate.coveragePct; double minCoverPct = voteGate.minCoveragePct; double edgeFloorPct = voteGate.edgeFloorPct; bool tradeableOK = voteGate.tradeable; double score = voteGate.selectionScore; //--- N for the family-wise correction: every era that COULD have won, mirroring the member gate's //--- exclusion of eras with nothing to trade. bool degenerate = voteGate.degenerate; //--- A BURN-IN ERA IS NOT A CANDIDATE. g_ensCandidateEras is the N the family-wise correction //--- divides by - "how many eras could have won". An era that is barred from taking the checkpoint //--- could not have won, so counting it would inflate N and RAISE the deploy bar for no reason. if(measurable && !degenerate && votedEra >= ENSEMBLE_CHECKPOINT_MIN_ERA) g_ensCandidateEras++; //--- Same lexicographic ordering as isBetterEra: deployable outranks two-sided outranks score. //--- The score comparison carries a NOISE BAND - see PLATEAU_NEW_BEST_SIGMAS for why a bare //--- `>` here is what lets a run spend thousands of eras without ever reaching the deploy stage. //--- The first scoring era still takes the checkpoint unconditionally (nothing to beat yet). double newBestBand = (g_ensBestScore >= 0.0) ? PLATEAU_NEW_BEST_SIGMAS * voteGate.scoreSE : 0.0; //--- BURN-IN. An era below ENSEMBLE_CHECKPOINT_MIN_ERA is scored and reported like any other but //--- cannot take the checkpoint - see that constant for the era-2 deploys this prevents. Note this //--- also gates the "first scoring era takes it unconditionally" path: the first era that may take //--- the checkpoint is the first MATURE one, not the first one to produce a number. bool mayCheckpoint = (votedEra >= ENSEMBLE_CHECKPOINT_MIN_ERA); bool isBetter = mayCheckpoint && ((tradeableOK && !g_ensBestTradeable) || (tradeableOK == g_ensBestTradeable && twoSided && !g_ensBestTwoSided) || (tradeableOK == g_ensBestTradeable && twoSided == g_ensBestTwoSided && !degenerate && score > g_ensBestScore + newBestBand)); if(isBetter) { g_ensBestScore = score; g_ensBestTradeable = tradeableOK; g_ensBestTwoSided = twoSided; g_ensBestPrecPct = votePrecPct; g_ensBestChancePct = chancePct; g_ensBestCalls = fired; g_ensBestEra = votedEra; //--- Kept with the rest so the stage-3 refusal can name the condition that failed instead of //--- listing all three - see their declaration comment. g_ensBestCoveragePct = coveragePct; g_ensBestMinCoverPct = minCoverPct; g_ensBestEdgeFloorPct = edgeFloorPct; //--- THE PIN. The threshold moves only when this era's weights become the checkpoint, so the //--- rung that trades is always the one the deployed weights were certified at - never a later //--- era's rung applied to an earlier era's model. //--- FROZEN ONCE DEPLOYED. After g_ensDeployApproved the certified configuration is settled and //--- nothing may move it: a post-deployment change would be an uncertified operating point on a //--- model that is already trading. See the walks that never ran for what "armed at //--- convergence" costs when it is not thought through. if(!g_ensDeployApproved) { double pinned = g_ensThresholdSweep[derivedIdx]; if(MathAbs(pinned - g_ensDerivedThreshold) > 0.01) PrintFormat("AI ensemble: vote threshold PINNED to %.0f%% by the era-%d checkpoint (was %s)." " It moves again only if a later era takes the checkpoint, and never once the" " ensemble deploys.", pinned, (int)votedEra, (g_ensDerivedThreshold > 0.0 ? StringFormat("%.0f%%", g_ensDerivedThreshold) : "the Signal_ThresholdOpen seed")); g_ensDerivedThreshold = pinned; g_ensembleVoteThreshold = pinned; } g_ensErasSinceBest = 0; g_ensPlateauStage = 0; EnsembleCommitJointCheckpoint(votedEra); } else //--- ONLY ONCE A CHECKPOINT IS POSSIBLE. This is the plateau counter that drives the escalation //--- ladder and, at stage 3, the deploy decision. Letting it run through the burn-in would have //--- the run reach "no better vote for N eras" while there was no best to beat, escalating - and //--- potentially exhausting the ladder - before the first era was even allowed to compete. if(mayCheckpoint) g_ensErasSinceBest++; //--- SHARED PLATEAU LADDER. One counter, one stage, applied to every member at the same era, so //--- the four nets escalate and finish together instead of drifting into different stages of //--- different searches. int plateauedMembers = 0, learningMembers = 0; for(int pi = 0; pi < ArraySize(g_warriorEnsemble); pi++) { CExpertSignalAIBase *pm = g_warriorEnsemble[pi]; if(CheckPointer(pm) == POINTER_INVALID || pm.m_ensembleIndex < 0) continue; //--- Same participation test the barrier uses: a member that has finished or been stopped is not //--- something the rest should wait on, and must not veto the collective stop either. if(pm.m_trainingComplete || pm.m_trainingStopRequested || !pm.m_isInitialized || pm.m_barrierExcluded) continue; if(pm.m_isErrorPlateaued) plateauedMembers++; else learningMembers++; } bool allIsPlateaued = (plateauedMembers > 0 && learningMembers == 0); if(allIsPlateaued && !g_ensIsPlateauAnnounced) { g_ensIsPlateauAnnounced = true; PrintFormat("AI ensemble: EVERY member's IN-SAMPLE error has plateaued (%d participating members)." " No member is still learning from the data it can see, so more eras cannot find a" " better vote - they would only enlarge the family the deploy gate corrects over." " Ending the search on the joint checkpoint at the next era that does not improve it." " This stop never read an out-of-sample number, which is what makes the smaller family" " legitimate rather than a peek.", plateauedMembers); } string ladderNote = ""; if(!isBetter) { int dueStage = g_ensErasSinceBest / TrainPlateauPatienceEras(); //--- FED IN AS A DUE STAGE rather than written straight to g_ensPlateauStage, and the //--- difference is the whole fix: the block that actually ends the run sits under `dueStage > //--- g_ensPlateauStage`, so assigning the stage directly makes that test FALSE and the deploy //--- never happens. if(allIsPlateaued && g_ensGateTestedEra != g_ensBestEra) dueStage = PLATEAU_STAGE_DEPLOY; if(dueStage > g_ensPlateauStage) { g_ensPlateauStage = dueStage; if(g_ensPlateauStage == PLATEAU_STAGE_RESTART || g_ensPlateauStage == PLATEAU_STAGE_ANNEAL) { for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || !mm.m_isInitialized) continue; mm.m_modelEta = mm.m_etaCeiling * PLATEAU_RESTART_BOOST; mm.m_restartBoostErasLeft = TrainPlateauPatienceEras(); mm.m_plateauStage = g_ensPlateauStage; if(CheckPointer(mm.Net) != POINTER_INVALID) mm.Net.ResetOptimizerState(); //--- THIS member is the one still inside Train(), holding g_eta in a local that would //--- overwrite m_modelEta on the way out - so its restart has to reach the local too. if(mm == GetPointer(this)) etaLocal = mm.m_modelEta; } ladderNote = StringFormat(" | PLATEAU stage %d: %d eras with no better vote - boosted warm" " restart on all %d models (learning rate x%.1f, optimizer momentum" " reset). The joint checkpoint is safe.", g_ensPlateauStage, g_ensErasSinceBest, members, PLATEAU_RESTART_BOOST); } else if(g_ensPlateauStage >= PLATEAU_STAGE_DEPLOY) { //--- EXHAUSTED. Both escapes tried, nothing better found: this is the best vote this //--- ensemble reaches. Now the gate that matters - has the best-of-N vote survived //--- having been chosen? g_ensGateTestedEra = g_ensBestEra; double zBest = 0.0, pFam = 1.0; int nTried = 0; bool survives = EnsembleSurvivesSelection(zBest, pFam, nTried); bool haveJoint = true; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; //--- Era-stamped, not just present: a snapshot from an EARLIER era would make the //--- deployed quartet one that was never measured together (see m_checkpointEra). if(!mm.m_haveOosCheckpoint || mm.m_checkpointEra != g_ensBestEra) haveJoint = false; } string testNote = StringFormat(" best-of-%d test on the VOTE: edge %.1fpp (%.1f%% vs chance" " %.1f%%) on %d fired bars = %.2f sigma, family-wise p=%.4f" " (need <=%.2f)", nTried, g_ensBestPrecPct - g_ensBestChancePct, g_ensBestPrecPct, g_ensBestChancePct, g_ensBestCalls, zBest, pFam, DEPLOY_FAMILY_WISE_ALPHA); if(g_ensBestTradeable && haveJoint && survives) { g_ensDeployApproved = true; Print("AI ensemble: PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - no better vote for " + IntegerToString(g_ensErasSinceBest) + " eras across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " warm restarts." + testNote + " - CLEARS. Deploying the JOINT checkpoint from era " + IntegerToString((int)g_ensBestEra) + ": every model reverts to the weights it held" " at the era whose combined vote scored best, so the ensemble that trades is" " exactly the one that was measured."); ladderNote = " | ENSEMBLE DEPLOY APPROVED"; } else { //--- Restart the ladder and keep training, exactly as the solo gate does on a //--- failed selection test. The era cap stays the backstop. THROTTLED //--- (2026-08-19): the refusal repeated ~450x/day with an unchanged reason. int refusalKey = (!g_ensBestTradeable ? 1 : (!haveJoint ? 2 : 3)); if(refusalKey != m_lastEnsRefusalKey || TrainLogDue()) Print("AI ensemble: PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + (!g_ensBestTradeable ? "no era's combined vote ever cleared the deployability floor, so there is nothing" " safe to deploy. THE BEST ERA FAILED ON: " + //--- NAME THE CONDITION. All three used to be listed and none identified; they //--- have nothing in common as fixes, so the list was not a diagnosis. (g_ensBestCoveragePct >= 0.0 && g_ensBestMinCoverPct > 0.0 && g_ensBestCoveragePct < g_ensBestMinCoverPct ? StringFormat("COVERAGE - it fired on %.1f%% of scored bars against a %.1f%% floor" " (a quarter of the directional base rate). %s The ensemble vote is a" " weighted mean of member tier weights, so Signal_ThresholdOpen is" " effectively a quorum - check it against what the members can" " actually cast before assuming the models are at fault.", g_ensBestCoveragePct, g_ensBestMinCoverPct, //--- THE TWO CASES ARE OPPOSITE DIAGNOSES and must not share a //--- sentence. Cleared-bar means the calls were good and there were //--- too few of them; missed-bar does NOT mean the model is simply //--- weak, because the exact-binomial floor is computed from the //--- INDEPENDENT call count - so thin coverage inflates the very bar //--- it is being judged against. Reporting those as two separate //--- failures would send a reader off to fix the model when the //--- coverage is what moved the target. (g_ensBestPrecPct > g_ensBestEdgeFloorPct ? StringFormat("Precision %.1f%% DID clear its %.1f%% bar: the calls it" " made were good enough and there were simply too few of" " them - the vote is too SELECTIVE, not too weak.", g_ensBestPrecPct, g_ensBestEdgeFloorPct) : StringFormat("Precision %.1f%% also missed its %.1f%% bar - but that" " bar is inflated BY the thin coverage, since the exact" " binomial floor rises as independent calls fall. These" " are not two independent failures: fix coverage first," " then re-read the bar.", g_ensBestPrecPct, g_ensBestEdgeFloorPct))) : (!g_ensBestTwoSided ? "ONE-SIDEDNESS - the vote never fired both long and short, so its precision" " is a one-direction book's, not a strategy's." : StringFormat("PRECISION - %.1f%% against a %.1f%% bar (chance %.1f%% on %d" " calls). Coverage was fine at %.1f%% against a %.1f%% floor.", g_ensBestPrecPct, g_ensBestEdgeFloorPct, g_ensBestChancePct, g_ensBestCalls, g_ensBestCoveragePct, g_ensBestMinCoverPct))) : (!haveJoint ? "the joint checkpoint is incomplete - at least one model has no snapshot of the" " winning era, so the measured ensemble cannot be reproduced." : "the best combined vote clears the per-era floor but DOES NOT clear the null of" " the MAXIMUM over the eras it was chosen from." + testNote + " A best-of-N this large happens routinely when every era is a noise draw.")) + " Restarting the ladder and continuing to train; the era cap remains the backstop."); m_lastEnsRefusalKey = refusalKey; g_ensErasSinceBest = 0; g_ensPlateauStage = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) != POINTER_INVALID && mm.m_ensembleIndex >= 0) mm.m_plateauStage = 0; } ladderNote = " | ladder restarted (gate not cleared)"; } } } } //--- Mirror the shared ladder onto every member. for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; mm.m_erasSinceBest = g_ensErasSinceBest; mm.m_plateauStage = g_ensPlateauStage; } //--- LIFETIME ACCUMULATION - same cadence as a solo model's m_cumOosTotal (see its increment sites): //--- every scored era adds the bars the vote fired on and how many paid, monotonically, never reset //--- per era. `wins`/`fired` above are this era's OOS rows only; the panel reads the running total. g_ensCumOosTotal += fired; g_ensCumOosCorrect += wins; //--- PANEL + JOURNAL. Body in PublishEnsembleAccuracyLine (ExpertSignalAIBase.mqh) - it is also //--- called at init, from the record restored out of .stats, which is what gives a reloaded //--- deployed chart an aggregate line at all. PublishEnsembleAccuracyLine(votePrecPct, fired); //--- ONCE DEPLOYED, DROP THE ERA TAIL RATHER THAN FREEZE IT. This line is written once per era, //--- here at pass-3 completion, and a deployed/converged ensemble runs no further eras (see //--- ScheduleTrainingIfNeeded's trainingComplete branch) - so a static "(era 69, DEPLOYING)" //--- would sit on the panel forever, unrefreshed, looking live when it is not (user report //--- 2026-08-24). g_liveVoteLine, appended right after this by PublishEnsembleStatus, is what //--- actually refreshes every tick from here on; the era count has nothing left to say. //--- The era/model/deployable suffix was dropped 2026-08-26 with the rest of the line's baggage: //--- the panel shows the accuracy, the era log line carries the diagnostics. Kept as a comment //--- rather than deleted silently so the next reader knows where those three facts went. //--- THE HIGHEST VOTE THIS ENSEMBLE CAN PRODUCE: every member voting, each at its best tier. The //--- divisor in Direction() is the CAPABLE weight, so unanimity returns the capability-weighted mean //--- of the tier weights - which is roughly the pooled holdout win rate. A threshold above that can //--- NEVER fire, and "0 fired" then reads as "the models are unsure" when it means "unreachable in //--- this configuration". Measured on USDJPY 2026-08-22: pooled win rates 15.6-19.4% against a //--- 25% threshold, highest vote ever seen 13. Same class as the excursion head's disjoint gate. double capSum = 0.0, bestSum = 0.0; int rankedMembers = 0, enrolledMembers = 0; for(int ci = 0; ci < ArraySize(g_warriorEnsemble); ci++) { CExpertSignalAIBase *cm = g_warriorEnsemble[ci]; if(CheckPointer(cm) == POINTER_INVALID || cm.m_ensembleIndex < 0) continue; enrolledMembers++; //--- Unranked members abstain (see LiveVoteContribution), so they are not part of the ceiling //--- either - counting their stock 100 would put the ceiling above anything reachable. if(!cm.SelfRanked()) continue; rankedMembers++; double capW = cm.VoteCapableWeight(); if(!MathIsValidNumber(capW) || capW <= 0.0) continue; int best = 0; for(int t = 0; t < 4; t++) best = (int)MathMax(best, cm.PatternWeightForTier(t)); capSum += capW; bestSum += capW * best; } double voteCeiling = (capSum > 0.0) ? bestSum / capSum : 0.0; //--- "0 fired because nobody has ranked yet" and "0 fired because the threshold is unreachable" //--- look identical in the coverage number and are completely different problems. string rankNote = (rankedMembers < enrolledMembers) ? StringFormat(" | %d of %d members have NOT ranked their tiers yet and are" " abstaining: tier weights only exist as the output of a" " completed pass 3 and are not persisted in the .nnw, so every" " fresh deploy and every resume starts here. Self-corrects after" " one era per member.", enrolledMembers - rankedMembers, enrolledMembers) : ""; //--- THE SWEEP, rendered. Coverage and precision at each selectable rung, plus a marker on the //--- one in force, so "the vote is too selective" comes with the number that would fix it. The //--- coverage floor is the same minCoverPct the gate applies, so a rung can be read as passing //--- or failing at a glance. Printed only when there is a floor to compare against. string sweepNote = ""; if(measurable && minCoverPct > 0.0) { sweepNote = " | THRESHOLD SWEEP (what this era's vote would score at each rung, floor " + StringFormat("%.1f%%", minCoverPct) + "):"; for(int s = 0; s < ENS_THRESHOLD_SWEEP_N; s++) { double swCov = 100.0 * sweepFired[s] / MathMax(shared, 1); double swPrec = (sweepFired[s] > 0) ? 100.0 * sweepWins[s] / sweepFired[s] : -1.0; sweepNote += StringFormat(" %.0f%%->%s/%.1f%%cov%s", g_ensThresholdSweep[s], (swPrec >= 0.0 ? StringFormat("%.1f%%prec", swPrec) : "n/a"), swCov, (s == derivedIdx ? "[DERIVED]" : (swCov >= minCoverPct ? "[clears floor]" : ""))); } sweepNote += ". [DERIVED] is the rung this era's verdict was computed at and the one the live" " signal now trades: the HIGHEST rung clearing the whole gate (coverage floor," " exact binomial bar, both sides live), or - if none did - the highest still" " clearing coverage, so the failure is reported on precision rather than on a" " population too thin to judge. Nothing to set by hand: Signal_ThresholdOpen is" " now only the seed used before the first scored era."; } string ceilingNote = (voteCeiling > 0.0 && voteCeiling < g_ensembleVoteThreshold) ? StringFormat(" | THRESHOLD UNREACHABLE: the highest vote this ensemble can" " cast is %.1f%% (every member voting at its best tier) against" " a %.0f%% threshold. Coverage cannot rise above 0 until the" " threshold sits below that ceiling, which IS the pooled win" " rate - no amount of training moves it.", voteCeiling, g_ensembleVoteThreshold) : ""; Print(StringFormat("AI ensemble: combined-vote era %d - %d models, %d shared OOS bars, %d fired at" " vote>=%.0f%% (%.1f%% coverage, floor %.1f%%), precision %s vs chance %.1f%% (needs" " >%.1f%% at %d sigma)%s -> score %s%s%s. The vote that actually trades: each" " member's DB-ranked tier weight x module weight, averaged over the members that" " VOTED (abstentions excluded, as live), graded on swing-label agreement.", (int)votedEra, members, shared, fired, g_ensThresholdSweep[derivedIdx], coveragePct, minCoverPct, (fired > 0 ? StringFormat("%.1f%%", votePrecPct) : "n/a"), chancePct, edgeFloorPct, (int)EDGE_MIN_SIGMAS, (tradeableOK ? " DEPLOYABLE" : ""), DeployScoreText(score), (isBetter ? StringFormat(" <-- NEW BEST, joint checkpoint captured (era %d)", (int)votedEra) : StringFormat(" (best %s at era %d, %d eras ago)", DeployScoreText(g_ensBestScore), (int)g_ensBestEra, g_ensErasSinceBest)), ladderNote + rankNote + ceilingNote + sweepNote)); } //+------------------------------------------------------------------+ //| Per-member era-end hook: mark this member done for the era and, | //| when it is the last one, run the verdict above. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleOosPassComplete(const long votedEra, double &etaLocal) { if(!m_ensembleMember || m_ensembleIndex < 0) return; //--- The rows were stamped during pass 3, BEFORE this member incremented its era counter, so the //--- buffer's era is the era that just finished. A mismatch means this member contributed nothing //--- to the current buffer (no OOS bars scored this era) - it cannot be counted as having read the //--- vote, or the verdict would be taken on a subset that silently excludes it. if(g_ensVoteEra != votedEra) return; g_ensVoteDoneMask |= (1 << m_ensembleIndex); int need = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; need |= (1 << mm.m_ensembleIndex); } if(need == 0 || (g_ensVoteDoneMask & need) != need) return; //--- Idempotence: one verdict per era, whatever order the members arrive in. if(g_ensLastVerdictEra == votedEra) return; g_ensLastVerdictEra = votedEra; EnsembleEraVerdict(need, votedEra, etaLocal); } //+------------------------------------------------------------------+ //| Log the selection-gate verdict for a deploy the gate does NOT | //| block - the era-cap path and the panel's Deploy button, both of | //| which are explicit operator decisions and stay that way. The point | //| is that "I chose to ship this" and "this cleared the bar" should | //| never be confusable in the log afterwards. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportSelectionGateVerdict(string context) { double z = 0.0, pFam = 1.0; int nTried = 0; //--- ENSEMBLE: report the gate that actually governs this model. Quoting the member's own //--- best-of-N here would answer a question nobody asked - the member never deploys alone, and a //--- member-level "CLEARS" next to a vote that did not is precisely the confusion this function //--- exists to prevent. if(m_ensembleMember) { bool okEns = EnsembleSurvivesSelection(z, pFam, nTried); if(g_ensBestCalls <= 0) { Print(ID + ": " + context + " - the ENSEMBLE selection gate cannot be evaluated (no era's" " combined vote has been ranked yet). Treat this ensemble as unvalidated."); return; } Print(ID + ": " + context + " - ENSEMBLE best-of-" + IntegerToString(nTried) + " test on the" " combined VOTE: edge " + DoubleToString(g_ensBestPrecPct - g_ensBestChancePct, 1) + "pp (" + DoubleToString(g_ensBestPrecPct, 1) + "% vs chance " + DoubleToString(g_ensBestChancePct, 1) + "%) on " + IntegerToString(g_ensBestCalls) + " fired bars = " + DoubleToString(z, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ") - " + (okEns ? "CLEARS." : "DOES NOT CLEAR. A maximum this size arises routinely when every era is a noise" " draw, so this ensemble is being deployed on operator authority, NOT on measured" " evidence of an edge.")); return; } bool ok = BestCheckpointSurvivesSelection(z, pFam, nTried); if(m_bestDirCalls <= 0) { Print(ID + ": " + context + " - selection gate cannot be evaluated (no ranked checkpoint with" " directional calls). Treat this model as unvalidated."); return; } Print(ID + ": " + context + " - best-of-" + IntegerToString(nTried) + " selection test: edge " + DoubleToString(m_bestDirPrecPct - m_bestChancePrecPct, 1) + "pp (" + DoubleToString(m_bestDirPrecPct, 1) + "% vs chance " + DoubleToString(m_bestChancePrecPct, 1) + "%) on " + IntegerToString(m_bestDirCalls) + " directional calls = " + DoubleToString(z, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ") - " + (ok ? "CLEARS." : "DOES NOT CLEAR. A maximum this size arises routinely when every era is a noise draw, so" " this model is being deployed on operator authority, NOT on measured evidence of an edge.")); } //+------------------------------------------------------------------+ //| Training and Signal Methods Where the TRAINING window starts: | //| ALL available history, floored by MinTrainYear. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportTrainStall(const string branch) { const uint STALL_AFTER_MS = 180000; // 3 min: ~2x the slowest healthy era seen on this config const uint STALL_REPORT_INTERVAL = 60000; uint nowTick = GetTickCount(); //--- First call ever: adopt now as the baseline rather than reporting instantly against tick 0. if(m_lastEraCompleteTick == 0) { m_lastEraCompleteTick = nowTick; return; } uint since = nowTick - m_lastEraCompleteTick; if(since < STALL_AFTER_MS) return; if(m_lastStallReportTick != 0 && nowTick - m_lastStallReportTick < STALL_REPORT_INTERVAL) return; m_lastStallReportTick = nowTick; PrintFormat("%s: TRAIN STALL - no era has completed for %.0fs and Train() is taking the '%s' branch" " | era %d | runActive=%s prebuildActive=%s cachePrebuilt=%s simOos=%s eraResume=%s" " paused=%s stopReq=%s | labelCacheBars=%d anchor=%s dtStudied=%s", ID, since / 1000.0, branch, (int)m_eraCount, m_trainRunActive ? "Y" : "N", m_labelPrebuildActive ? "Y" : "N", m_labelCachePrebuilt ? "Y" : "N", m_onlineLearning.SimRunActive() ? "Y" : "N", m_eraResumePending ? "Y" : "N", m_trainingPaused ? "Y" : "N", m_trainingStopRequested ? "Y" : "N", m_labelCacheBars, TimeToString(m_labelCacheAnchorTime), TimeToString(dtStudied)); } //+------------------------------------------------------------------+ //| THE WINDOW THE ERA ACTUALLY GOT, and the three quantities that | //| decide it. Reported on change only. | //| | //| barsNow is MathMin(Bars(symbol, PERIOD_CURRENT, dtStudied, now) + | //| historyBars, Bars(symbol, PERIOD_CURRENT)), so a short era is | //| either a dtStudied that is too RECENT or a price series that is | //| short - and those need opposite fixes. Printing only the result | //| ("3671 bars") cannot tell them apart, which is why all three go | //| on the line together with the date dtStudied resolved to. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportEraWindow(const int barsNow) { if(barsNow == m_lastEraWindowBars) return; m_lastEraWindowBars = barsNow; PrintFormat("%s: era %d TRAINING WINDOW = %d bars | Bars(since dtStudied %s) = %d | Bars(series) = %d" " | series starts %s | historyBars %d. The smaller of the first two is what this era" " trains and scores on - NOT the in-sample estimate the CAPACITY and DETECTABILITY" " lines quote, which is derived from the configuration.", ID, (int)m_eraCount, barsNow, TimeToString(dtStudied), Bars(m_symbol.Name(), PERIOD_CURRENT, dtStudied, TimeCurrent()), Bars(m_symbol.Name(), PERIOD_CURRENT), TimeToString((datetime)SeriesInfoInteger(m_symbol.Name(), PERIOD_CURRENT, SERIES_FIRSTDATE)), (int)m_historyBars); } //+------------------------------------------------------------------+ //| Speaks ONLY when an era is genuinely slow: nothing for the first | //| 60 seconds of an era, at most 6 lines after that, one per 4096 | //| processed items. Reports where the time actually went, split into | //| the two candidate costs and the remainder, because "the era is | //| slow" without the split is exactly the undiagnosable state the | //| 2026-08-10 restart produced (see the member declarations). | //+------------------------------------------------------------------+ void CExpertSignalAIBase::TrainHeartbeat(const string tag, int done, int total, const string shortLabel) { //--- Panel progress is published on EVERY call, before the 4096-item gate below: the gate exists to //--- keep the JOURNAL quiet, and applying it to the panel too would leave the display frozen between //--- boundaries. Two assignments, no formatting - cheap enough for a per-item path. m_passLabel = shortLabel; m_passProgressPct = (total > 0) ? (int)MathMin(100.0, 100.0 * done / total) : 0; //--- TIME-gated, not item-gated. A diagnostic whose trigger can be outrun by the condition it //--- watches for is worse than none - it produces confident wrong conclusions. The 255-item mask //--- only keeps GetTickCount() off the hot path. if((done & 255) != 0) return; uint nowTick = GetTickCount(); uint elapsedMs = nowTick - m_eraStartTick; if(elapsedMs < 60000 || m_passHeartbeatPrints >= 12) return; if(m_lastHeartbeatTick != 0 && nowTick - m_lastHeartbeatTick < 30000) return; m_lastHeartbeatTick = nowTick; m_passHeartbeatPrints++; double featS = (double)m_passFeatUs / 1000000.0; double netS = (double)m_passNetUs / 1000000.0; PrintFormat("%s: SLOW ERA heartbeat - %s %d of %d after %.0fs | feature windows %.1fs | net fwd/back %.1fs | everything else %.1fs", ID, tag, done, total, elapsedMs / 1000.0, featS, netS, MathMax(elapsedMs / 1000.0 - featS - netS, 0.0)); } //+------------------------------------------------------------------+ datetime CExpertSignalAIBase::TrainWindowStart(datetime startTrainBar) { datetime firstAvailableBar = (datetime)SeriesInfoInteger(m_symbol.Name(), PERIOD_CURRENT, SERIES_FIRSTDATE); MqlDateTime floor_time; TimeCurrent(floor_time); floor_time.year = m_minTrainYear; floor_time.mon = 1; floor_time.day = 1; floor_time.hour = 0; floor_time.min = 0; floor_time.sec = 0; datetime st_time = StructToTime(floor_time); if(firstAvailableBar > st_time) st_time = firstAvailableBar; return MathMax(startTrainBar, st_time); } //+------------------------------------------------------------------+ //| Save the era-loop context a yielding chunk resumes against. | //| | //| Each pass keeps its OWN cursor (m_isTrainCursor, m_calibIndex, | //| m_oosScoreIndex); these five are what every pass shares. One | //| writer, because a field missed at one of the four yield points | //| resumes the next chunk against a different era than the one that | //| yielded, and nothing reports that until the numbers drift. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::StashEraResume(const int bars, const int totalIter, const int oosCutoff, const bool add_loop, const int barIndex) { m_resumeBars = bars; m_resumeTotalIter = totalIter; m_resumeOosCutoff = oosCutoff; m_resumeAddLoop = add_loop; m_resumeBarIndex = barIndex; m_eraResumePending = true; //--- This model's own learning-rate trajectory, out of the shared global before yielding. m_modelEta = g_eta; } //+------------------------------------------------------------------+ //| THE ERA LINE. Everything below is string building over already- | //| measured state - it decides nothing and changes nothing, which | //| is exactly why it does not belong inside the era loop. | //| | //| Self-guarding on tel.shouldLog: the throttle is decided where | //| the tick count is known and carried here, so the caller is one | //| unconditional call rather than a 200-line branch. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportEraProgress(const SEraTelemetry &tel) { if(!tel.shouldLog) return; string recallInfo = (tel.buyRecall < 0 && tel.sellRecall < 0 && tel.neutralRecall < 0) ? "" : (" | OOS recall Buy:" + (tel.buyRecall < 0 ? "n/a" : IntegerToString(tel.buyRecall) + "%") + " Sell:" + (tel.sellRecall < 0 ? "n/a" : IntegerToString(tel.sellRecall) + "%") + " Neutral:" + (tel.neutralRecall < 0 ? "n/a" : IntegerToString(tel.neutralRecall) + "%")); string selectionInfo = (tel.dirPrec < 0) ? " | SELECT: no directional calls survived the threshold" : (" | SELECT precision " + IntegerToString(tel.dirPrec) + "% on " + IntegerToString(tel.coverage) + "% of bars (post-threshold)" + (tel.chancePrec >= 0 ? " (chance=base-rate " + IntegerToString(tel.chancePrec) + "%, edge " + (tel.dirPrec - tel.chancePrec >= 0 ? "+" : "") + IntegerToString(tel.dirPrec - tel.chancePrec) + "pp)" : "")); //--- The operating point that produced the coverage figure just above it, so the two are read //--- together: coverage falling is only good news if it is this that caused it. selectionInfo += " @margin>=" + DoubleToString(m_dirConfThreshold, 2); //--- TRADED precision: the same calls after declustering, which since 2026-08-09 is //--- exactly the set that becomes positions (live NMS gates the trade, not just the //--- arrow). if(m_signalClusterWindow > 0 && m_oosNmsFired > 0) { int nmsPrec = (int)MathRound(100.0 * m_oosNmsHits / m_oosNmsFired); selectionInfo += " | TRADED (declustered) " + IntegerToString(nmsPrec) + "% on " + IntegerToString(m_oosNmsFired) + " calls" + (tel.chancePrec >= 0 ? " (edge " + (nmsPrec - tel.chancePrec >= 0 ? "+" : "") + IntegerToString(nmsPrec - tel.chancePrec) + "pp)" : ""); } // See tel.buyPred's declaration comment for why this is worth logging alongside recall - // it's what tells apart a suppressed/dead output (predicted rate stuck at 0%) from a // miscalibrated boundary (predicted rate healthy, precision poor), which look identical from // recall alone. string predictedInfo = (tel.buyPred < 0 && tel.sellPred < 0) ? "" : (" | OOS calls Buy:" + (tel.buyPred < 0 ? "n/a" : IntegerToString(tel.buyPred) + "%") + " (precision " + (tel.buyPrec < 0 ? "n/a" : IntegerToString(tel.buyPrec) + "%") + ")" + " Sell:" + (tel.sellPred < 0 ? "n/a" : IntegerToString(tel.sellPred) + "%") + " (precision " + (tel.sellPrec < 0 ? "n/a" : IntegerToString(tel.sellPrec) + "%") + ")"); //--- CALIBRATION - "does the model call each class as often as the class actually occurs". //--- Ratios, because 1.0x is the answer and the distance from it is the error. string calibInfo = (tel.buyTrue < 0 || tel.buyFired < 0) ? "" : StringFormat(" | CALIBRATION traded vs true rate Buy %d%% vs %d%% (%s) Sell %d%% vs %d%% (%s)" " Neutral %d%% vs %d%% (%s) | pre-threshold argmax Buy %d%% Sell %d%% Neutral %d%%", tel.buyFired, tel.buyTrue, CalibrationRatio(tel.buyFired, tel.buyTrue), tel.sellFired, tel.sellTrue, CalibrationRatio(tel.sellFired, tel.sellTrue), tel.neutralFired, tel.neutralTrue, CalibrationRatio(tel.neutralFired, tel.neutralTrue), tel.buyPred, tel.sellPred, tel.neutralPred); //--- Live-fired precision: the number that actually predicts forward-trading performance - only //--- the directional calls that cleared the confidence floor under the live/prior-corrected rule //--- (see AdjustedSignalFromSoftmax). Count in parentheses = how many bars the model would have //--- traded this era. "0" fires = the calibration is (this era) suppressing all directional trades. string liveInfo = (m_lastBuyFired <= 0 && m_lastSellFired <= 0) ? " | live fires 0 this era" : (" | live precision Buy:" + (tel.buyFiredPrec < 0 ? "n/a" : IntegerToString(tel.buyFiredPrec) + "%") + " (" + IntegerToString(m_lastBuyFired) + ")" + " Sell:" + (tel.sellFiredPrec < 0 ? "n/a" : IntegerToString(tel.sellFiredPrec) + "%") + " (" + IntegerToString(m_lastSellFired) + ")"); //--- Precision BY CONFIDENCE TIER, and cumulatively from each tier upward - the two //--- numbers a decision about Signal_ThresholdOpen actually needs. string tierInfo = ""; int tierFiredTotal = 0; for(int ti = 0; ti < 4; ti++) tierFiredTotal += m_oosTierFired[ti]; if(tierFiredTotal > 0) { tierInfo = " | tier prec"; for(int ti = 0; ti < 4; ti++) { int cumFired = 0, cumHits = 0; for(int tj = ti; tj < 4; tj++) { cumFired += m_oosTierFired[tj]; cumHits += m_oosTierHits[tj]; } tierInfo += " T" + IntegerToString(ti) + ":" + (m_oosTierFired[ti] > 0 ? IntegerToString((int)MathRound(100.0 * m_oosTierHits[ti] / m_oosTierFired[ti])) + "%" : "n/a") + "(" + IntegerToString(m_oosTierFired[ti]) + ")" + (cumFired > 0 ? "[>=" + IntegerToString((int)MathRound(100.0 * cumHits / cumFired)) + "%/" + IntegerToString(cumFired) + "]" : ""); } } //--- Per-layer weight movement. Pairs with rawOutInfo below: a collapsed constant- //--- classifier state has two very different causes, and only this tells them apart. See //--- CNet::LayerLearningReport. string layerInfo = (CheckPointer(Net) == POINTER_INVALID) ? "" : (" | dW/W" + Net.LayerLearningReport()); // Raw-output saturation diagnostic - see m_oosOutMin's declaration comment. Spread ~0 with // all six min/max values pinned together = the collapsed constant-classifier state. string rawOutInfo = (m_oosOutCount <= 0) ? "" : StringFormat(" | OOS raw out B:%.3f..%.3f S:%.3f..%.3f N:%.3f..%.3f spread avg %.4f", m_oosOutMin[0], m_oosOutMax[0], m_oosOutMin[1], m_oosOutMax[1], m_oosOutMin[2], m_oosOutMax[2], m_oosOutSpreadSum / m_oosOutCount); //--- DENOMINATOR IS THE PER-ERA BAR COUNT, not m_oosSamples (fixed 2026-08-17). int zsBars = m_oos.Bars(); string zeroSkillInfo = (zsBars <= 0) ? "" : StringFormat(" | zero-skill on these bars: always-Buy %.1f%%, always-Sell %.1f%%" " (the gate ranks on the LARGER of the two; the gap between them IS the" " directional drift, and a model that only reproduces it has found the drift," " not an edge)", 100.0 * (double)m_oos.buyTotal / zsBars, 100.0 * (double)m_oos.sellTotal / zsBars); //--- THE DEPLOY BAR, stated. Reading "edge -1pp" era after era tells you the model is //--- short; it does not tell you whether it is short by a hair or by an amount no strategy //--- could ever cover. //--- STATE THE FORMULA THAT ACTUALLY PRODUCED THE NUMBER. This line used to read //--- "chance + 2 x SE 3.9pp" beside a printed bar of 100.0% - two quantities that cannot both //--- be true, sitting in the same parenthesis, every era, on every chart. The bar is the EXACT //--- binomial floor (ExactEdgeFloorPct); chance+sigmas*SE is only the normal approximation it //--- replaced, still shown alongside because a large gap between the two is itself the signal //--- that the sample is too small for the approximation - and because a disagreement between //--- them is what a reader can actually check. string gateInfo = (m_lastEdgeFloorPct < 0.0 || m_lastEffN <= 0.0) ? "" : StringFormat(" | DEPLOY BAR %.1f%% (EXACT binomial at %.0f sigma; the chance+%.0fxSE" " approximation would say %.1f%%, SE %.1fpp) on %.0f INDEPENDENT calls -" " %d raw calls deflated by the %.1f-bar mean label lifespan%s", m_lastEdgeFloorPct, EDGE_MIN_SIGMAS, EDGE_MIN_SIGMAS, m_oos.ChancePrecPct() + EDGE_MIN_SIGMAS * m_lastPrecSE, m_lastPrecSE, m_lastEffN, m_oos.DirCalls(), MeanLabelLifespan(), //--- A bar above 100% is not "hard", it is unreachable, and no amount of //--- training addresses it - only a bigger independent sample does. (m_lastEdgeFloorPct >= 100.0 ? " <-- UNREACHABLE: no win rate can clear this. The OOS window does not hold" " enough independent observations to certify ANY edge; widen the sample" " (more instruments / lower timeframe) or narrow the barrier." : "")); string neutralWhy = (m_oosOutCount <= 0) ? "" : StringFormat(" | Neutral CHOSE %.1f%% / TIED %.1f%% (of which B=S %d) | rail %.1f%%", 100.0 * (double)m_oosNeutralStrict / m_oosOutCount, 100.0 * (double)m_oosNeutralTie / m_oosOutCount, (int)m_oosTieBuySell, 100.0 * (double)m_oosRailBars / m_oosOutCount); //--- No "(target X%)" any more - there is no absolute accuracy target. What replaces it as the //--- progress indicator is the plateau counter: how many eras since the last new best, and how //--- close that is to ending the run (see the PLATEAU_* ladder). string plateauInfo = (m_bestSelectionScore < 0) ? "" : (" | best score " + DeployScoreText(m_bestSelectionScore) + ", " + IntegerToString(m_erasSinceBest) + " eras since (stage " + IntegerToString(m_plateauStage) + "/" + IntegerToString(PLATEAU_STAGE_DEPLOY) + ")"); //--- Lifetime IS/OOS directional accuracy. The GAP between the two is still the over- //--- fitting read, so it survives here, once per era, behind the compile-time //--- DebuggingMode constant. string lifetimeInfo = (!DebuggingMode || (m_cumIsTotal <= 0 && m_cumOosTotal <= 0)) ? "" : (" | lifetime dir acc IS " + (m_cumIsTotal > 0 ? IntegerToString((int)MathRound(m_cumIsCorrect * 100.0 / m_cumIsTotal)) + "%" : "n/a") + " OOS " + (m_cumOosTotal > 0 ? IntegerToString((int)MathRound(m_cumOosCorrect * 100.0 / m_cumOosTotal)) + "%" : "n/a") + " over " + IntegerToString(m_cumIsTotal + m_cumOosTotal) + " calls"); //--- Wall-clock split for the era that just finished, but only when it was SLOW - a //--- healthy era stays exactly one line. An era finished: the stall clock restarts from //--- here (see m_lastEraCompleteTick). m_lastEraCompleteTick = GetTickCount(); string eraTimeInfo = ""; { double eraS = (GetTickCount() - m_eraStartTick) / 1000.0; if(eraS > 120.0) eraTimeInfo = StringFormat(" | ERA TOOK %.0fs (feature windows %.0fs, net fwd/back %.0fs," " other %.0fs)", eraS, m_passFeatUs / 1000000.0, m_passNetUs / 1000000.0, MathMax(eraS - m_passFeatUs / 1000000.0 - m_passNetUs / 1000000.0, 0.0)); } //--- THROTTLED (2026-08-19): this is the ~2KB deep-dive block, and it printed every era //--- for every member - ~3.7MB per member per day, the single largest line in a measured //--- 22MB/9.5h journal. VerboseMode = every era again. if(TrainLogDue()) Print(ID + ": training in progress - era " + IntegerToString(m_eraCount) + ", OOS accuracy " + DoubleToString(dOosForecast, 1) + "%, IS error " + DoubleToString(dError, 2) + recallInfo + selectionInfo + predictedInfo + calibInfo + liveInfo + tierInfo + plateauInfo + lifetimeInfo + zeroSkillInfo + gateInfo + m_lastPoolReport + rawOutInfo + neutralWhy + layerInfo + eraTimeInfo); // Forced (unthrottled) panel refresh, right here alongside the console line above, using this // era's own just-finalized m_eraCount/dOosForecast - see UpdateTrainingStatusLabel's // declaration comment for why this can't just rely on the next throttled bar-scan call to // catch up (it would, but a full era later than the console already reported it). RefreshStatusLabel(); } //+------------------------------------------------------------------+ //| PASS 1: the scan/queue sweep that walks the era backwards from | //| era.i, building the sample queue and training on it. | //| | //| era.i is a MEMBER of the era state rather than a loop local | //| because this loop yields on the wall-clock budget and resumes at | //| the same bar on the next call. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::RunPass1(STrainEra &era) { for(; era.i >= 0 && !era.stop; era.i--) { //--- Build THIS bar's own feature window and feed it forward BEFORE checking/training against //--- its label - see r's declaration comment below for why the window must end AT bar i, and //--- why this must run before the label-check block rather than after: the label check needs //--- this bar's own freshly-computed prediction, not the previous iteration's (see windowOk). TempData.Clear(); //--- Window ends AT (includes) bar i itself, extending m_historyBars bars into the past - //--- i.e. int r = era.i; bool windowOk = false; double displayNeuron0 = 0, displayNeuron1 = 0, displayNeuron2 = 0; if(r <= era.bars) { ulong hbT = GetMicrosecondCount(); windowOk = BuildFeatureWindow(r); m_passFeatUs += GetMicrosecondCount() - hbT; if(windowOk) { era.addLoop = true; m_passWindowOk++; } else m_passWindowFail++; } TrainHeartbeat("pass 1 (scan/queue), bar", era.bars - MathMax(m_historyBars, 0) - era.i, era.totalIter, "scan"); //--- Determine label/queue-eligibility BEFORE running any feedForward this bar - see //--- wouldQueue's use below for why. Mirrors the label-check condition this block used to //--- gate on (moved earlier, unchanged). bool haveLabel = false, buy = false, sell = false, wouldQueue = false; //--- "some LATER pass in this same era will feed this exact bar forward anyway", which is a //--- strictly wider set than wouldQueue - see its use at the feedForward below. Declared out //--- here because the three membership tests that decide it are scoped to the label block. bool laterPassForwards = false; if(windowOk && era.i < (int)(era.bars - MathMax(m_historyBars, 0) - 1) && era.i > 1 && m_Time.GetData(era.i) > dtStudied && (m_outputNeuronsCount == 1 || m_outputNeuronsCount == 3)) { //--- The swing label at now-relative index i only depends on price/ZigZag history, never //--- on model state, so it's identical every era until a new bar closes and shifts the //--- index frame (see the cache invalidation check above) - cache it rather than //--- recomputing from scratch every single era. A cache miss here is a bar the prebuild //--- could not resolve yet (P2 uncommitted); try again now, and a bar that is STILL //--- unresolved is simply not trainable this era. if(!m_labelCacheHasValue[era.i]) AdvanceSwingLabelState(era.i, era.bars); if(m_labelCacheHasValue[era.i]) { buy = m_labelCacheBuy[era.i]; sell = m_labelCacheSell[era.i]; haveLabel = true; } bool isOOS = (era.i < era.oosCutoff); //--- Embargo: a bar's swing label is decided by the bars that follow it, out to its //--- pivot pair - the purge width covers the measured mean resolution lag. int calibLo = CalibLoIndex(era.oosCutoff); // = oosCutoff + one purge width int calibHi = CalibHiIndex(era.totalIter, era.oosCutoff); // == calibLo when the band is empty bool isEmbargoed = (!isOOS && era.i < calibLo); //--- The calibration slice and its far-side purge are held out of backprop for the same //--- reason the OOS window is, and the layout is documented once at CalibLoIndex(). bool isCalib = (era.i >= calibLo && era.i < calibHi); bool isCalibPurge = (calibHi > calibLo && era.i >= calibHi && era.i < calibHi + CalibPurgeBars()); //--- An unresolved bar (haveLabel false) is not trainable: queueing it would backprop a //--- provisional Neutral against a label that does not exist yet. wouldQueue = (haveLabel && !isOOS && !isEmbargoed && !isCalib && !isCalibPurge); //--- Pass 2 re-forwards every queued bar, pass 2.5 re-forwards the whole calibration //--- band, and pass 3 re-forwards the whole OOS window - each over EXACTLY this bar set //--- (all three derive their bounds from the same helpers and apply the identical //--- eligibility test this block gates on). laterPassForwards = (wouldQueue || isOOS || isCalib); } //--- Only run this bar's feedForward (and the display/count/chart-draw work that depends //--- on it) when NO later pass is about to redo it anyway. ulong hbFwd = GetMicrosecondCount(); bool scanForwardOk = (windowOk && !laterPassForwards && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbFwd; if(scanForwardOk) { Net.getResults(TempData); if(m_outputNeuronsCount == 1) dPrevSignal = TempData[0]; else if(m_outputNeuronsCount == 3) dPrevSignal = ApplyClassificationSoftmax(); //--- Snapshot the just-computed neuron output(s) for the status label display below, before //--- the label-check block clears/refills TempData with the target label (Step A always //--- runs after this point now) - reading TempData directly for display after that would //--- show the TRUE LABEL of the bar just trained on, not the network's own prediction. if(TempData.Total() > 0) displayNeuron0 = TempData[0]; if(TempData.Total() > 1) displayNeuron1 = TempData[1]; if(TempData.Total() > 2) displayNeuron2 = TempData[2]; switch(DoubleToSignal(dPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } m_lastBarTime = m_Time.GetData(era.i); if(era.i > 0) { // NMS on: record only - the era-end sweep is the SOLE renderer, so no raw (un- // declustered) arrow is ever drawn mid-era. NMS off: draw inline as before. if(m_signalClusterWindow > 0) { if(era.i < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[era.i] = dPrevSignal; } else if(DoubleToSignal(dPrevSignal) == Neutral) DeleteObject(m_lastBarTime); else DrawObject(m_lastBarTime, dPrevSignal, m_Close.GetData(era.i)); } UpdateTrainingStatusLabel( StringFormat("Bar %d of %d -> %.2f%% (scan)", era.bars - era.i + 1, era.bars, (double)(era.bars - era.i + 1.0) / era.bars * 100), displayNeuron0, displayNeuron1, displayNeuron2, dPrevSignal); } else //--- Bars a later pass will re-forward skip the feedForward above, and they are now //--- very nearly ALL of pass 1 - the queued IS bars (~58%, processed FIRST because the //--- loop walks oldest-to-newest), plus the calibration band and the OOS slice. UpdateTrainingStatusLabel( StringFormat("Bar %d of %d -> %.2f%% (scan)", era.bars - era.i + 1, era.bars, (double)(era.bars - era.i + 1.0) / era.bars * 100), displayNeuron0, displayNeuron1, displayNeuron2, dPrevSignal); if(haveLabel) { // True label as an ENUM_SIGNAL, derived directly from the buy/sell bools - not read // back from TempData, which no longer holds a target at this point at all (see above). ENUM_SIGNAL trueSignal = buy ? Buy : (sell ? Sell : Neutral); // Track the true class distribution this era (used below to weight IS oversampling, // and surfaced in the status label text alongside the predicted-class counts) switch(trueSignal) { case Buy: m_trueBuyCount++; break; case Sell: m_trueSellCount++; break; default: m_trueNeutralCount++; break; } // OOS scoring used to happen right here, against whatever weights this bar's earlier // feedForward (this pass) happened to be using - which for era 0 is the network's // still-untrained cold-start state (100% Neutral - see the output-layer bias seed's // declaration comment), and for every later era is last era's END-of-training state, // never THIS era's. That silently gave every era's OOS score a full one-era lag behind // its own training, and made era 0's OOS score meaningless by construction. OOS scoring // now happens in its own pass (see m_isPass3Active's declaration comment), AFTER pass 2 // has actually trained on this era's IS data, against a fresh feedForward on each OOS // bar rather than this scan's now-stale one. if(wouldQueue) { //--- Queue this bar for pass 2's shuffled backProp instead of training on it here, //--- immediately, in strict chronological order - see m_isTrainQueue's declaration //--- comment for the full rationale. //--- MINORITY REPLAY IS GONE (2026-07-31): every bar is queued exactly ONCE and the //--- class imbalance is corrected analytically in the gradient by the logit-adjusted //--- loss (Menon et al. 2021). if(m_isTrainQueueCount + 1 > ArraySize(m_isTrainQueue)) { int newQueueSize = m_isTrainQueueCount + 1; ArrayResize(m_isTrainQueue, newQueueSize, 16384); } m_isTrainQueue[m_isTrainQueueCount] = era.i; m_isTrainQueueCount++; } } era.stop = IsStopped() || m_trainingStopRequested; if(!era.stop && era.i > 0 && era.BudgetSpent()) { //--- yield: save exactly enough to resume this same era, mid-bar-loop, on the next call - //--- see m_trainRunActive's declaration comment for why this must happen instead of //--- letting one era (or the whole run) process synchronously to completion StashEraResume(era.bars, era.totalIter, era.oosCutoff, era.addLoop, era.i - 1); return; } } } //+------------------------------------------------------------------+ //| One adopted peer row: forward, target, backward. Nothing else. | //| | //| The local pass-2 body cannot be reused for this. Every line of it | //| after the forward pass reaches for something indexed by a LOCAL | //| bar - m_labelCache, the arrow cache, m_Time - and a peer row has | //| no local bar. Sharing | //| the path would mean inventing values for all of those, which is | //| how another instrument's outcomes end up inside this chart's IS | //| accuracy and the operating point gets fitted to them. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::TrainPoolStep(const int poolIdx) { //--- Only the 3-class direction head is poolable. Guarded rather than assumed: a head change //--- would otherwise silently train peer rows against a target of the wrong width. if(m_outputNeuronsCount != 3) return; int w = NetInputWidth(); TempData.Clear(); for(int k = 0; k < w; k++) TempData.Add(m_trainPoolReader.At(poolIdx, k)); if(TempData.Total() < w || !Net.feedForward(TempData)) return; //--- Slot order is buy / sell / neutral, identical to the local branch below, and the same label //--- smoothing - a literal 1.0 target the sigmoid only approaches asymptotically grows weights //--- toward the MAX_WEIGHT clamp. int label = m_trainPoolReader.LabelAt(poolIdx); TempData.Clear(); TempData.Add(label == 0 ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add(label == 1 ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add(label == 2 ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); Net.backProp(TempData); m_netDirty = true; } //+------------------------------------------------------------------+ //| PASS 2: the second sweep over the in-sample span. | //| | //| Ordering matters here in a way it does not in pass 1 - see | //| dOosForecast's declaration comment for why the recursion makes | //| this pass's direction load-bearing. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::RunPass2(STrainEra &era) { if(!era.stop && era.addLoop && !m_isPass2Done) { if(!m_isPass2Active) { m_isPass2Active = true; m_isTrainCursor = 0; //--- MINI-BATCH ON, for pass 2 only (2026-08-09 audit, F4). Switched back off where pass 2 //--- completes. Net.SetBatchSize(TRAIN_BATCH_SIZE); //--- 2026-07-28: a "replay-only optimizer override" was removed from here. It arrived with //--- the DFA change set and was never part of any validated run. The optimizer the user //--- selects is now the optimizer that runs. //--- CROSS-INSTRUMENT ROWS join the queue HERE, before the shuffle, so peer samples are //--- interleaved with this chart's rather than trained in a block at one end - a block would //--- be a curriculum, and the last thing the optimizer saw would decide where it landed. //--- They ride as NEGATIVE sentinels (-(poolIndex+1)); the loop below dispatches on the sign. //--- The purge cutoff is the OLDEST OOS BAR'S TIME: a peer row whose label resolved at or //--- after that instant carries information from a window this model is about to be judged //--- on. Bar indices cannot be compared across instruments - each has its own calendar - so //--- the key is wall-clock on both sides. if(TrainPoolEnabled()) { m_trainPoolWriter.Begin(BuildModelFingerprint(), m_symbol.Name(), (int)m_period); long cutoffMs = (long)m_Time.GetData(era.oosCutoff) * 1000; int adopted = m_trainPoolReader.Adopt(BuildModelFingerprint(), m_symbol.Name(), (int)m_period, cutoffMs, NetInputWidth()); if(adopted > 0) { int base = m_isTrainQueueCount; ArrayResize(m_isTrainQueue, base + adopted, 16384); for(int p = 0; p < adopted; p++) m_isTrainQueue[base + p] = -(p + 1); m_isTrainQueueCount += adopted; } //--- No print here. CTrainPoolReader::Adopt reports its own verdict - adopted, alone, or //--- every peer rejected and why - and reports it once per CHANGE rather than once per //--- era, because an era on a warm feature cache is a fraction of a second long. } for(int sIdx = m_isTrainQueueCount - 1; sIdx > 0; sIdx--) { //--- ShuffleRandomIndex, NOT MathRand()%: the queue routinely exceeds MathRand()'s 15-bit //--- range on a full-history window, which silently biased this shuffle - see the helper. int sJ = ShuffleRandomIndex(sIdx + 1); int sTmp = m_isTrainQueue[sIdx]; m_isTrainQueue[sIdx] = m_isTrainQueue[sJ]; m_isTrainQueue[sJ] = sTmp; } } for(; m_isTrainCursor < m_isTrainQueueCount; m_isTrainCursor++) { int qi = m_isTrainQueue[m_isTrainCursor]; TrainHeartbeat("pass 2 (shuffled backprop), sample", m_isTrainCursor + 1, m_isTrainQueueCount, "training"); //--- A PEER ROW contributes GRADIENT ONLY and then leaves. Everything below this point is //--- keyed on a LOCAL bar index - the excursion head, the chart arrows, m_labelCache, and //--- the IS accuracy counters - and a peer bar has none of those. Letting one through would //--- not crash; it would quietly pollute m_cumIsCorrect and the operating-point fit with //--- another instrument's outcomes, and the IS-vs-OOS gap is read as THE overfitting signal. if(qi < 0) { TrainPoolStep(-qi - 1); continue; } ulong hbT = GetMicrosecondCount(); bool qWindowOk = BuildFeatureWindow(qi); //--- Contribute this row to the pool while the window is still in TempData and before the //--- forward pass overwrites it. if(qWindowOk && TrainPoolEnabled() && m_labelCacheHasValue[qi]) m_trainPoolWriter.Add(TempData, m_labelCacheBuy[qi] ? 0 : (m_labelCacheSell[qi] ? 1 : 2), TrainPoolResolvedMs(qi)); m_passFeatUs += GetMicrosecondCount() - hbT; //--- A failed forward pass must NOT be followed by backProp() further down this block: the //--- output layer would still hold the PREVIOUS sample's activations, so the update would be //--- this bar's label against another bar's prediction - training on pure noise while every //--- accuracy counter kept reporting normally. hbT = GetMicrosecondCount(); bool qForwardOk = (qWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbT; if(qWindowOk && !qForwardOk && !era.forwardFailureReported) { era.forwardFailureReported = true; Print(__FUNCTION__ + ": CNet::feedForward FAILED at era " + IntegerToString((int)m_eraCount) + " - this era's remaining samples are being skipped, not trained. A layer is refusing to" " accept its own output (check the preceding BufferWrite/BufferRead lines for which" " buffer, and see NormalizeHost in AI\\NeuronBatchNorm.mqh for the batch-norm case)."); } if(qForwardOk) { Net.getResults(TempData); // Must go through ApplyClassificationSoftmax() (3-output case) before reading the // per-class values below - Net.getResults() returns each output neuron's own independent // SIGMOID activation (each already in [0,1] but NOT summing to 1 across the three), not a // true class-conditional probability distribution; ApplyClassificationSoftmax() is what // turns that into one (and is also what pass 1/3's displayNeuron0/1/2 already go through). double qPrevSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; double pt0 = (TempData.Total() > 0) ? TempData[0] : 0.0; double pt1 = (TempData.Total() > 1) ? TempData[1] : 0.0; double pt2 = (TempData.Total() > 2) ? TempData[2] : 0.0; bool qBuy = m_labelCacheHasValue[qi] ? m_labelCacheBuy[qi] : false; bool qSell = m_labelCacheHasValue[qi] ? m_labelCacheSell[qi] : false; ENUM_SIGNAL qTrueSignal = qBuy ? Buy : (qSell ? Sell : Neutral); UpdateTrainingStatusLabel( StringFormat("Training bar %d of %d -> %.2f%% (shuffled)", m_isTrainCursor + 1, m_isTrainQueueCount, (double)(m_isTrainCursor + 1.0) / MathMax(m_isTrainQueueCount, 1) * 100), pt0, pt1, pt2, qPrevSignal); //--- Predicted-signal tally, chart marker, and IS-accuracy stat that pass 1 used to //--- compute from its own (now-removed) redundant feedForward on this same bar - see //--- pass 1's wouldQueue comment. switch(DoubleToSignal(qPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } datetime qBarTime = m_Time.GetData(qi); // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(qi < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[qi] = qPrevSignal; } else if(DoubleToSignal(qPrevSignal) == Neutral) DeleteObject(qBarTime); else DrawObject(qBarTime, qPrevSignal, m_Close.GetData(qi)); bool qClassified = (DoubleToSignal(qPrevSignal) == Buy || DoubleToSignal(qPrevSignal) == Sell || DoubleToSignal(qPrevSignal) == Neutral); if(qClassified) { bool isHit = (DoubleToSignal(qPrevSignal) == qTrueSignal); if(isHit) dForecast += (100 - dForecast) / Net.recentAverageSmoothingFactor; else dForecast -= dForecast / Net.recentAverageSmoothingFactor; dUndefine -= dUndefine / Net.recentAverageSmoothingFactor; //--- Compounded, persistent DIRECTIONAL win-rate: count only bars the model actually //--- called Buy or Sell (a Neutral "no trade" call is neither a win nor a loss), so //--- this tracks the accuracy of its directional signals rather than the Neutral- //--- inflated all-class rate. ENUM_SIGNAL qPred = DoubleToSignal(qPrevSignal); //--- Directional calls scored on label agreement, identically to the OOS side, //--- because the IS and OOS rates are read side by side as the overfitting signal. if(qPred == Buy || qPred == Sell) { m_cumIsTotal++; if(isHit) m_cumIsCorrect++; } //--- THE OPERATING-POINT FIT NO LONGER HARVESTS HERE. It moved to the held-out //--- calibration walk below; DIR_CONF_CALIB_PCT_OF_IS carries the measured IS-vs-OOS //--- divergence that forced the move. } else if(qBuy && qSell) dUndefine += (100 - dUndefine) / Net.recentAverageSmoothingFactor; TempData.Clear(); if(m_outputNeuronsCount == 1) TempData.Add(qBuy && !qSell ? 1 : !qBuy && qSell ? -1 : 0); else if(m_outputNeuronsCount == 3) { TempData.Add(qBuy ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add(qSell ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add((!qBuy && !qSell) ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); } //--- FOCAL-LOSS MODULATION REMOVED 2026-07-31. It multiplied this weight by //--- (1-pt)^gamma, a second correction on the same axis as the logit adjustment - the //--- stacking failure Buda et al. ulong hbBp = GetMicrosecondCount(); //--- No per-sample weight: the imbalance correction is analytic (logit-adjusted loss), so //--- backProp's own sampleWeight default of 1.0 is the shipped behaviour. Net.backProp(TempData); m_netDirty = true; m_passNetUs += GetMicrosecondCount() - hbBp; } //--- YIELD ON TIME **OR** ON A STOP REQUEST. The time budget bounds THROUGHPUT; it does //--- not bound LATENCY to an unload. Pass 1 has checked IsStopped() all along; passes 2, //--- 2.5 and 3 never did, and they are the ones that grow with history. if(m_isTrainCursor + 1 < m_isTrainQueueCount && (IsStopped() || era.BudgetSpent())) { //--- yield: save enough to resume PASS 2 mid-queue on the next call - m_isPass2Active //--- and m_isTrainCursor (both members) carry the actual resume position; bars/oosCutoff/ //--- add_loop are stashed the same way pass 1 already does, since era-end logic just //--- below still needs them once pass 2 finishes. StashEraResume(era.bars, era.totalIter, era.oosCutoff, era.addLoop, era.i); return; } } //--- Apply whatever the final (usually short) batch of this era accumulated, and return the //--- net to per-sample updates. FlushBatch scales by the REAL sample count, so a short //--- trailing batch still takes a correctly-sized step. Net.FlushBatch(); Net.SetBatchSize(1); m_isPass2Active = false; m_isPass2Done = true; //--- Publish this chart's rows once the era's queue is fully walked, so a peer never reads a //--- partial sweep. The buffer is refilled from scratch each era rather than accumulated: one //--- era already covers the whole in-sample span, so accumulating would republish the same //--- bars N times and inflate this instrument's weight in every peer's pool. if(TrainPoolEnabled() && m_trainPoolWriter.Count() > 0) m_trainPoolWriter.Publish(); } } //+------------------------------------------------------------------+ //| CALIBRATION PASS: fit the directional confidence threshold on a | //| PURGED held-out slice. | //| | //| Separate from pass 2 because it must not see bars the weights | //| were fitted on - a threshold fitted on memorised bars is the | //| single easiest way to manufacture an edge that does not exist. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::RunCalibrationPass(STrainEra &era) { if(!era.stop && era.addLoop && !m_isCalibDone) { int calibLo = CalibLoIndex(era.oosCutoff); int calibHi = CalibHiIndex(era.totalIter, era.oosCutoff); if(!m_isCalibActive) { m_isCalibActive = true; Net.SetBatchNormFrozen(true); ResetDirConfHistogram(); //--- Same upper clamp pass 3 applies: a bar needs m_historyBars of older bars behind it to //--- build a window at all, so the band is trimmed to what is actually scoreable. m_calibStartIndex = (int)MathMin(calibHi - 1, era.bars - MathMax(m_historyBars, 0) - 2); m_calibIndex = m_calibStartIndex; } for(; m_calibIndex >= calibLo; m_calibIndex--) { int ci = m_calibIndex; //--- Same eligibility test pass 1 gates labelling on (its line reads //--- `i < bars-historyBars-1 && i > 1 && Time[i] > dtStudied`), so this walk can only score bars //--- pass 1 actually produced a label for. Pass 3 applies the identical test on its own window. if(!(ci < (int)(era.bars - MathMax(m_historyBars, 0) - 1) && ci > 1 && m_Time.GetData(ci) > dtStudied)) continue; TrainHeartbeat("pass 2.5 (calibration), bar", m_calibStartIndex - m_calibIndex + 1, m_calibStartIndex - calibLo + 1, "calibrating"); ulong hbC = GetMicrosecondCount(); bool cWindowOk = BuildFeatureWindow(ci); m_passFeatUs += GetMicrosecondCount() - hbC; hbC = GetMicrosecondCount(); bool cForwardOk = (cWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbC; if(cForwardOk) { Net.getResults(TempData); //--- RAW argmax softmax, NOT AdjustedSignalFromSoftmax(): feeding the fit its own //--- already- thresholded decisions would make the threshold a fixed point of itself, //--- able only to ratchet upward. double cSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; ENUM_SIGNAL cPred = DoubleToSignal(cSignal); //--- Scored on LABEL AGREEMENT - the same currency every other verdict in the run uses. //--- An unresolved bar has no label to agree with, so it joins neither the numerator nor //--- the denominator of the fit. bool cBuy = (m_labelCacheHasValue[ci] && m_labelCacheBuy[ci]); bool cSell = (m_labelCacheHasValue[ci] && m_labelCacheSell[ci]); bool cHit = (cPred == Buy) ? cBuy : ((cPred == Sell) ? cSell : false); //--- isPrimaryBar is unconditionally true: this walk visits each bar once in chronological //--- order, so there is no oversampled replay to correct for here. if(m_labelCacheHasValue[ci]) AccumulateDirConfSample(DirectionalMargin(), cHit, true); //--- Predicted-class tally and chart marker for the calibration band, which pass 1 used //--- to compute from its own (now-removed) redundant feedForward on this same bar. switch(cPred) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } datetime cBarTime = m_Time.GetData(ci); if(ci > 0) { // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(ci < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[ci] = cSignal; } else if(cPred == Neutral) DeleteObject(cBarTime); else DrawObject(cBarTime, cSignal, m_Close.GetData(ci)); } } //--- Time OR stop - see pass 2's matching comment. if(m_calibIndex - 1 >= calibLo && (IsStopped() || era.BudgetSpent())) { //--- yield: m_isCalibActive + m_calibIndex carry the resume position, same as passes 1-3. StashEraResume(era.bars, era.totalIter, era.oosCutoff, era.addLoop, era.i); return; } } Net.SetBatchNormFrozen(false); //--- An empty band (era too short to carve one - see CalibBandBars) means there is no measurement //--- this era, which is not the same as a measurement that says "trade everything". Leave the //--- operating point exactly where the last successful fit put it rather than refitting on nothing. if(calibHi > calibLo) FitDirConfThreshold(); m_isCalibActive = false; m_isCalibDone = true; } } //+------------------------------------------------------------------+ //| PASS 3: SCORE THE OUT-OF-SAMPLE SPAN. | //| | //| Walks the OOS bars with batch-norm frozen (scoring must not move | //| the running statistics - live adaptation is untouched), then runs | //| the reports that read the walk it just finished: exit-policy | //| simulation, candidate geometry, excursions, cluster pruning, tier | //| re-ranking, drift and the Alglib baselines. | //| | //| It MEASURES. Nothing here decides anything - the recall gate, the | //| plateau ladder, the deploy gate and the era checkpoint all read | //| these numbers afterwards, back in Train(). | //| | //| Guarded on era.addLoop: a chunk that ran out of wall-clock budget | //| mid-era has no complete era to score. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::RunOosPass(STrainEra &era) { if(!era.stop && era.addLoop) { if(!m_isPass3Active) { m_isPass3Active = true; //--- Freeze batch-norm running statistics for the whole scoring walk (2026-08-09 audit, //--- F5). Same reasoning (and same mechanism) as ValidateCpuInference. Live/online //--- adaptation is untouched - only scoring is frozen. Net.SetBatchNormFrozen(true); m_oosScoreStartIndex = (int)MathMin(era.oosCutoff - 1, era.bars - MathMax(m_historyBars, 0) - 2); m_oosScoreIndex = m_oosScoreStartIndex; for(int rn = 0; rn < 3; rn++) { m_oosOutMin[rn] = DBL_MAX; m_oosOutMax[rn] = -DBL_MAX; } m_oosOutSpreadSum = 0.0; m_oosOutCount = 0; m_oosNeutralStrict = 0; m_oosNeutralTie = 0; m_oosTieBuySell = 0; m_oosRailBars = 0; } for(; m_oosScoreIndex >= 2; m_oosScoreIndex--) { int oi = m_oosScoreIndex; if(!(oi < (int)(era.bars - MathMax(m_historyBars, 0) - 1) && m_Time.GetData(oi) > dtStudied)) continue; TrainHeartbeat("pass 3 (OOS scoring), bar", m_oosScoreStartIndex - m_oosScoreIndex + 1, m_oosScoreStartIndex + 1, "scoring"); ulong hbT = GetMicrosecondCount(); bool oWindowOk = BuildFeatureWindow(oi); m_passFeatUs += GetMicrosecondCount() - hbT; //--- Same guard as pass 2, and it matters more here: OOS accuracy is what checkpoint selection //--- and the plateau ladder's auto-deploy both rank on, so scoring a stale forward pass would //--- not just be wrong, it would be wrong in the one number that decides which model ships. //--- A skipped bar simply isn't counted; it never becomes a hit or a miss. hbT = GetMicrosecondCount(); bool oForwardOk = (oWindowOk && TempData.Total() >= (int)m_historyBars * m_neuronsCount && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbT; if(oWindowOk && !oForwardOk && !era.forwardFailureReported) { era.forwardFailureReported = true; Print(__FUNCTION__ + ": CNet::feedForward FAILED during OOS scoring at era " + IntegerToString((int)m_eraCount) + " - affected bars are excluded from the OOS" " accuracy rather than scored against a stale prediction."); } if(oForwardOk) { Net.getResults(TempData); // Raw output stats MUST be captured here, before ApplyClassificationSoftmax() overwrites // TempData[0..2] in place with the softmax probabilities - see m_oosOutMin's declaration // comment for what these feed. if(m_outputNeuronsCount == 3 && TempData.Total() >= 3) { double rawHi = -DBL_MAX, rawLo = DBL_MAX; for(int rn = 0; rn < 3; rn++) { double rv = TempData.At(rn); if(rv < m_oosOutMin[rn]) m_oosOutMin[rn] = rv; if(rv > m_oosOutMax[rn]) m_oosOutMax[rn] = rv; rawHi = MathMax(rawHi, rv); rawLo = MathMin(rawLo, rv); } m_oosOutSpreadSum += rawHi - rawLo; m_oosOutCount++; //--- WHY Neutral won, split into its two causes - see m_oosNeutralStrict's //--- declaration. double rB = TempData.At(0), rS = TempData.At(1), rN = TempData.At(2); bool strictB = (rB > rS && rB > rN); bool strictS = (rS > rB && rS > rN); bool strictN = (rN > rB && rN > rS); if(strictN) m_oosNeutralStrict++; else if(!strictB && !strictS) { //--- No class holds a strict majority, so the top two are EXACTLY equal and //--- ApplyClassificationSoftmax() returns Neutral by the tie rule, not by choice. m_oosNeutralTie++; //--- The expensive subset: Buy and Sell tied AT the top (either a 2-way tie above //--- Neutral, or a 3-way). The net had a directional reading and float equality //--- threw it away. if(rB == rS && rB >= rN) m_oosTieBuySell++; } //--- Sigmoid rails. The head is SIGMOID (Topology.mqh), so 0 and 1 are its //--- asymptotes; a raw value sitting ON one in float32 is the saturation that MAKES //--- exact ties possible. if(rawLo <= 1e-6 || rawHi >= 1.0 - 1e-6) m_oosRailBars++; } double oPrevSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; double oDeploySignal = oPrevSignal; if(m_outputNeuronsCount == 3) oDeploySignal = AdjustedSignalFromSoftmax(); double oNeuron0 = (TempData.Total() > 0) ? TempData[0] : 0.0; double oNeuron1 = (TempData.Total() > 1) ? TempData[1] : 0.0; double oNeuron2 = (TempData.Total() > 2) ? TempData[2] : 0.0; bool oLabeled = (oi < ArraySize(m_labelCacheHasValue) && m_labelCacheHasValue[oi]); bool oBuy = oLabeled ? m_labelCacheBuy[oi] : false; bool oSell = oLabeled ? m_labelCacheSell[oi] : false; ENUM_SIGNAL oTrueSignal = oBuy ? Buy : (oSell ? Sell : Neutral); //--- ONE CURRENCY, and it is the live one. oEnsembleVote is the signed vote this member //--- would have cast on this bar - m_weight x its tier's pattern weight, the same //--- number CExpertSignalCustom::Direction() sums and the same 0-100 win-rate scale //--- Signal_ThresholdOpen/Signal_ThresholdClose are expressed in. double oEnsembleVote = LiveVoteContribution(oDeploySignal); //--- The divisor term that goes with it - the member's weight WHENEVER it evaluated the //--- bar, Neutral included, because consensus arithmetic (2026-08-19) has abstention //--- dilute. Was zeroed on abstention under union semantics. //--- //--- ReconstructionWeight(), not ModuleWeight(): it carries the SKILL test that now keeps //--- a no-skill member out of the live divisor, without the converged-run test that would //--- zero every weight here (this runs DURING training, where m_trainingComplete is //--- false). Using ModuleWeight() would leave the scorer certifying a vote with a dead //--- member still in the denominator while live excluded it - certified != traded. //--- //--- The skill test reads the PREVIOUS era's measurement, which is the honest choice: //--- gating this era's vote on this era's own outcome would be circular. double oEnsembleWeight = ReconstructionWeight(); if(m_ensembleMember && oLabeled) EnsembleOosContribute(oi, oEnsembleVote, oEnsembleWeight, oBuy, oSell, (oBuy || oSell)); UpdateTrainingStatusLabel( StringFormat("Scoring OOS bar %d of %d -> %.2f%% (post-training)", m_oosScoreStartIndex - m_oosScoreIndex + 1, m_oosScoreStartIndex + 1, (double)(m_oosScoreStartIndex - m_oosScoreIndex + 1.0) / MathMax(m_oosScoreStartIndex + 1, 1) * 100), oNeuron0, oNeuron1, oNeuron2, oDeploySignal); // Held-out bar: score the model's freshly-trained-this-era forecast against the actual // label without learning from it - keeps the OOS accuracy an honest overfitting signal. // An UNRESOLVED bar (no committed pivot pair yet) has no label to score against, so it // is excluded from every tally rather than counted as a true Neutral it may not be. bool oClassified = oLabeled && (DoubleToSignal(oPrevSignal) == Buy || DoubleToSignal(oPrevSignal) == Sell || DoubleToSignal(oPrevSignal) == Neutral); if(oClassified) { m_oosSamples++; m_oos.confidenceSum += MathAbs(oPrevSignal); if(dOosError < 0) dOosError = 0; bool hit = (DoubleToSignal(oPrevSignal) == oTrueSignal); ENUM_SIGNAL oPred = DoubleToSignal(oPrevSignal); //--- Compounded, persistent DIRECTIONAL precision: count only bars the model actually //--- called Buy or Sell (Neutral "no trade" calls are neither right nor wrong here). if(oPred == Buy || oPred == Sell) { m_cumOosTotal++; if(hit) m_cumOosCorrect++; } // Per-class confusion counts, used for the Buy/Sell recall convergence gate below switch(oTrueSignal) { case Buy: m_oos.buyTotal++; if(hit) m_oos.buyHits++; break; case Sell: m_oos.sellTotal++; if(hit) m_oos.sellHits++; break; default: m_oos.neutralTotal++; if(hit) m_oos.neutralHits++; break; } //--- DECLUSTERED count: of the calls that would actually become POSITIONS, how many //--- were right. This pair is the one that answers "what would I have made". if(m_signalClusterWindow > 0) { //--- oDeploySignal, NOT oPrevSignal: live NMS runs downstream of the confidence //--- threshold (RefreshLatestSignal feeds NmsLiveAccept the ADJUSTED decision), so //--- replaying it on the raw argmax declusters a different, strictly larger stream //--- than the EA ever sees - different survivors, not just more of them, because rule 1 //--- collapses runs and rule 3 alternates over whatever sequence it is given. Bars the //--- threshold rejects must not consume a cluster slot or set the alternation state. ENUM_SIGNAL nmsDir = DoubleToSignal(oDeploySignal); if(nmsDir == Buy || nmsDir == Sell) { //--- Confidence for rule 2's cross-direction resolution comes from the same adjusted //--- decision, matching NmsLiveAccept's input exactly. double nmsConf = MathAbs(oDeploySignal); int lastSame = (nmsDir == Buy) ? m_oosNmsLastBuyIdx : m_oosNmsLastSellIdx; //--- 1) same-direction contiguous collapse; last-seen advances either way so a whole //--- run collapses to its first bar. bool cont = (lastSame >= 0 && (lastSame - oi) <= m_signalClusterWindow); if(nmsDir == Buy) m_oosNmsLastBuyIdx = oi; else m_oosNmsLastSellIdx = oi; bool keep = !cont; //--- 2) cross-direction resolution against the last KEPT opposite signal: flicker at //--- one turn zone resolves to the more confident side. if(keep && m_oosNmsKeptIdx >= 0 && m_oosNmsKeptDir != nmsDir && m_oosNmsKeptDir != Neutral && (m_oosNmsKeptIdx - oi) <= m_signalClusterWindow) keep = (nmsConf > m_oosNmsKeptConf); //--- 3) ALTERNATION, identical to NmsLiveAccept's rule 3. if(keep && BothDirectionsTradeable() && m_oosNmsKeptIdx >= 0 && m_oosNmsKeptDir == nmsDir) keep = false; if(keep) { m_oosNmsKeptIdx = oi; m_oosNmsKeptDir = nmsDir; m_oosNmsKeptConf = nmsConf; m_oosNmsFired++; //--- Label agreement of the surviving (position-becoming) calls. bool nmsHit = (nmsDir == Buy) ? oBuy : oSell; if(nmsHit) m_oosNmsHits++; } } } // Same confusion counts keyed by what the model actually PREDICTED this bar, not the // true label - see m_oos.buyPredicted's declaration comment for why recall alone can // hide an over-firing class. switch(DoubleToSignal(oPrevSignal)) { case Buy: m_oos.buyPredicted++; if(hit) m_oos.buyPredictedHits++; break; case Sell: m_oos.sellPredicted++; if(hit) m_oos.sellPredictedHits++; break; default: m_oos.neutralPredicted++; if(hit) m_oos.neutralPredictedHits++; break; } //--- Live-decision precision: scores the bars on which the deployed EA would //--- actually cast a directional vote, using the prior-corrected (logit-adjusted) //--- posterior - see AdjustedSignalFromSoftmax()/RefreshLatestSignal(). if(m_outputNeuronsCount == 3) { double adjSig = AdjustedSignalFromSoftmax(); ENUM_SIGNAL adjEnum = DoubleToSignal(adjSig); if(adjEnum != Neutral) { //--- Same currency as everywhere else in this block: label agreement. bool fireHit = (adjEnum == Buy) ? oBuy : oSell; //--- Bucket the same fire by confidence tier - see m_oosTierFired. It does //--- not. ConfidenceTier() reads dPrevSignal, and dPrevSignal is assigned in //--- PASS 1 only (the in-sample pass) - never anywhere in this OOS scan. int fireTier = ConfidenceTierFor(adjSig); if(fireTier >= 0 && fireTier < 4) { m_oosTierFired[fireTier]++; if(fireHit) m_oosTierHits[fireTier]++; } if(adjEnum == Buy) { m_oos.buyFired++; if(fireHit) m_oos.buyFiredHits++; } else { m_oos.sellFired++; if(fireHit) m_oos.sellFiredHits++; } } } if(hit) { dOosForecast += (100 - dOosForecast) / Net.recentAverageSmoothingFactor; dOosError -= dOosError / Net.recentAverageSmoothingFactor; } else { dOosForecast -= dOosForecast / Net.recentAverageSmoothingFactor; dOosError += (100 - dOosError) / Net.recentAverageSmoothingFactor; } } //--- Predicted-class tally for the OOS window, which pass 1 used to compute from its //--- own (now-removed) redundant feedForward on this same bar - see the //--- laterPassForwards comment there. switch(DoubleToSignal(oPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } // Chart annotation for this (OOS) bar, using post-training weights - pass 1 no longer // draws these at all (it used to, from a pre-training snapshot that this then overwrote). m_lastBarTime = m_Time.GetData(oi); if(oi > 0) { // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(oi < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[oi] = oDeploySignal; } else if(DoubleToSignal(oDeploySignal) == Neutral) DeleteObject(m_lastBarTime); else DrawObject(m_lastBarTime, oDeploySignal, m_Close.GetData(oi)); } } //--- Time OR stop - see pass 2's matching comment. if(m_oosScoreIndex - 1 >= 2 && (IsStopped() || era.BudgetSpent())) { //--- yield: save enough to resume PASS 3 mid-walk on the next call - m_isPass3Active and //--- m_oosScoreIndex (both members) carry the actual resume position. StashEraResume(era.bars, era.totalIter, era.oosCutoff, era.addLoop, era.i); return; } } m_isPass3Active = false; //--- Scoring finished - resume the normal always-adapting statistics (see the freeze at pass-3 //--- start) before anything else runs a forward pass. Net.SetBatchNormFrozen(false); //--- Pass 3 done => every scored bar's prediction is now in m_arrowSignalCache. Collapse each //--- same-direction cluster to its earliest bar so the chart shows one arrow per real turn. PruneDirectionalClusters(era.bars); //--- ...and now that this era's per-tier outcomes are complete, turn them into the vote weights //--- the NEXT era (and live trading) will use. See RankTiersFromOos(). RankTiersFromOos(); //--- ...and, once per run and only if asked, put two completely different learners on this exact //--- matrix so "the net is flat" can be told apart from "the matrix is flat". See CBaselineComparator. m_baselines.RunBaselineComparison(era.bars, era.totalIter, era.oosCutoff); } } //+------------------------------------------------------------------+ //| Shared preamble for the three exclusive walks. | //| | //| Each takes a whole Train() call, and each has to tell TWO | //| watchdogs the same thing: the stall reporter which branch is | //| running, and the era-barrier watchdog that this member is BUSY | //| rather than stuck. Written out three times, it was three chances | //| for a new walk to be added with only one of them. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ClaimCallForWalk(const string branch) { ReportTrainStall(branch); NoteBarrierProgress(); } //+------------------------------------------------------------------+ //| Why this member is idle at the ensemble era barrier. | //| | //| Resetting the stall watchdog was once the only thing the hold | //| branch did, so a held member left no record anywhere. It reports | //| on a cadence rather than on entry: a brief hold every era is the | //| DESIGN - the fast member waits a few seconds here every era - | //| and printing on entry logged ~950 lines per member per day. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportBarrierHold(void) { //--- AND SAY SO IN THE JOURNAL. Resetting the watchdog above is right (a held member is idle, //--- not stalled) but it was the ONLY thing this branch did, so a held member left no record //--- anywhere. uint nowTick = GetTickCount(); //--- ARM SILENTLY, REPORT ONLY WHEN THE HOLD OUTLASTS THE INTERVAL (2026-08-19). Printing on //--- entry logged ~950 lines/member/day, because a brief hold at the barrier is the DESIGN - //--- the fast member waits a few seconds here every era. bool justHeld = (m_barrierHoldReportTick == 0); if(justHeld) m_barrierHoldReportTick = nowTick; if((VerboseMode && justHeld) || nowTick - m_barrierHoldReportTick >= ENSEMBLE_BARRIER_REPORT_MS) { m_barrierHoldReportTick = nowTick; long minEra = EnsembleMinTrainingEra(); string blockers = ""; for(int bi = 0; bi < ArraySize(g_warriorEnsemble); bi++) { CExpertSignalAIBase *bm = g_warriorEnsemble[bi]; if(CheckPointer(bm) == POINTER_INVALID) continue; if(bm.m_trainingComplete || bm.m_trainingStopRequested || bm.m_trainingPaused || !bm.m_isInitialized || bm.m_barrierExcluded) continue; if(bm.m_eraCount <= minEra) blockers += (blockers == "" ? "" : ", ") + bm.ID; } if(EnsembleLeadCapHolds()) PrintFormat("%s: HELD BY THE ENSEMBLE LEAD CAP - this member is at era %d, the slowest member" " on the chart is at era %d, and %d eras is as far ahead as any member may get." " That slower member has ALREADY been dropped from the barrier for not advancing," " so nothing will resolve this on its own: diagnose it. Until it catches up the" " combined vote cannot be scored and no joint checkpoint can be taken, so training" " past this point would produce weights no gate could ever certify.", ID, (int)m_eraCount, (int)EnsembleMinEraAnyMember(), ENSEMBLE_MAX_ERA_LEAD); else PrintFormat("%s: HELD AT THE ERA BARRIER - this member is at era %d and the ensemble minimum is" " %d, so it is idle until [%s] catch up. It is NOT stalled and its weights are" " untouched. If this line keeps repeating, the member(s) named are the ones to" " diagnose - after %d minutes with no era AND no preparation-phase progress they" " are dropped from the barrier and this member resumes, up to %d eras ahead.", ID, (int)m_eraCount, (int)minEra, blockers == "" ? "(none - resolving)" : blockers, (int)(ENSEMBLE_BARRIER_STUCK_MS / 60000), ENSEMBLE_MAX_ERA_LEAD); } } //+------------------------------------------------------------------+ //| Does this call belong to training at all? | //| | //| Six ways it does not: paused, stopping, deploying an approved | //| ensemble checkpoint, held at the era barrier, or occupied by one | //| of the three exclusive walks. Each answers for the WHOLE call. | //| | //| None of this is training, which is why it is no longer inside | //| Train(). What is left there now reads as the era lifecycle it | //| always was, instead of opening with a hundred and twenty lines | //| of reasons not to run. | //| | //| Writes era.stop, which the caller needs either way. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::TrainCallPreempted(STrainEra &era) { //--- //--- Never block the calling thread while paused/stopped - just decline this call (or finalize a //--- run that just got stopped) and let the next scheduled call check again, so Pause/Resume/Stop //--- and everything else on the control panel stays responsive instead of Sleep()-ing the one //--- MQL5 thread this chart has. if(m_trainingPaused && !IsStopped() && !m_trainingStopRequested) return true; era.stop = IsStopped() || m_trainingStopRequested; if(era.stop) { if(m_trainRunActive) FinalizeTrainRun(); m_onlineLearning.AbortSimIfActive(); return true; } //--- ENSEMBLE DEPLOY, approved by the ensemble gate on some member's era end (see //--- EnsembleEraVerdict). Each member restores its own half of that checkpoint, so the quartet //--- that goes live is the one the vote was measured on. if(m_ensembleMember && g_ensDeployApproved && !m_trainingComplete && m_haveOosCheckpoint && m_checkpointEra == g_ensBestEra) { m_trainingComplete = true; Print(ID + ": ENSEMBLE DEPLOY - restoring this model's weights from the joint checkpoint at era " + IntegerToString((int)g_ensBestEra) + " and switching to live inference. The combined vote," " not this model alone, is what cleared the gate."); if(m_trainRunActive) FinalizeTrainRun(); //--- Same one-shot pattern-database backfill the solo path arms at its era end, and for the //--- same reason (see StartPatternDatabaseBackfill): a deployed model has to be RANKED the //--- instant it goes live, not an hour of real trades later. StartPatternDatabaseBackfill(m_resumeBars, m_resumeTotalIter, m_resumeOosCutoff); return true; } //--- ENSEMBLE ERA BARRIER (user request 2026-08-16): members advance era by era TOGETHER, //--- because the number that matters - the combined-vote OOS score - is only well-defined when //--- every member's pass 3 describes the same era, and because live trading is the members //--- voting together, not four models drifting apart in training age. BarrierEraHeartbeat(); if(EnsembleEraBarrierHolds()) { //--- deliberate idleness, not a stall - keep the stall watchdog's era clock current and say //--- what is happening on the member's panel line instead of freezing its last progress text m_lastEraCompleteTick = GetTickCount(); ReportBarrierHold(); PublishStatus(StringFormat("Waiting at era %d for slower ensemble members (min era %d) - donating its compute until they catch up", (int)m_eraCount, (int)EnsembleMinTrainingEra())); return true; } m_barrierHoldReportTick = 0; //--- Evaluation-only continual-learning OOS simulation walk in progress (see //--- StartOosContinualSimulation): give it exclusive occupancy of this call, same chunked budget //--- as the real era loop below, so a large OOS window can't freeze the UI in one shot. if(m_onlineLearning.SimRunActive()) { ClaimCallForWalk("OOS continual-learning simulation walk"); AdvanceOosSimulationChunk(); return true; } //--- One-shot pattern-database backfill in progress (see StartPatternDatabaseBackfill) - same //--- exclusive-occupancy/chunking treatment as the simulation walk above. if(m_onlineLearning.BackfillActive()) { ClaimCallForWalk("pattern-database backfill walk"); AdvancePatternDatabaseBackfill(); return true; } //--- Eager label-cache pre-build in progress (see StartLabelCachePrebuild/AdvanceLabelCachePrebuild) - //--- same exclusive-occupancy/chunking treatment as the OOS simulation walk above, so it can't freeze //--- the UI on a large study window either. m_trainRunActive stays false for its whole duration, so //--- once it completes, Train() falls through to the normal !m_trainRunActive setup below and era 0 //--- starts from the measured class distribution it just seeded. if(m_labelPrebuildActive) { ClaimCallForWalk("label-cache prebuild scan"); AdvanceLabelCachePrebuild(); return true; } //--- Nothing claimed this call: it is a training call. return false; } //+------------------------------------------------------------------+ //| EVERYTHING AN ERA DOES AFTER ITS LAST PASS SCORES. | //| | //| Calibrate confidence, read the recalls, run the deploy gate, | //| fill the telemetry, rank this era against the best so far, | //| capture or restore a checkpoint, advance the learning-rate and | //| plateau ladders, test stability, and persist. One era's verdict. | //| | //| Lifted whole rather than split, and deliberately so: its parts | //| share thirty-odd locals - the recalls, the gate verdict, the | //| better/worse flags - and threading those through three signatures | //| would recreate the eight-locals-across-four-passes problem that | //| STrainEra was built to end. Splitting this further needs an | //| era-outcome object first, not more parameters. | //| | //| Nothing here returns early, which is why it could move at all: | //| Train() still runs the era line, the finalizer and the backfill | //| after it on every path. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::CompleteEra(STrainEra &era, SEraTelemetry &tel) { const int STABILITY_WINDOW = 3; // consecutive eras the OOS accuracy must hold steady for const double STABILITY_TOLERANCE = 2.0; // max spread (percentage points) across that window if(!era.stop) { dError = Net.getRecentAverageError(); if(era.addLoop) { if(m_oosSamples > 0) { // Confidence calibration (classification head only - see m_confidenceCalScale's // declaration comment): compare this era's actual OOS accuracy against the average // confidence magnitude the model claimed, EMA-blend the resulting scale into // m_confidenceCalScale so SignedAIConfidence() reports something closer to a real // probability instead of the raw, uncalibrated softmax value. if(m_outputNeuronsCount == 3 && m_oos.confidenceSum > 0.0) { //--- accuracy / mean claimed confidence. Both terms would divide by the same sample //--- count, so it cancels - which matters, because m_oosSamples is RUN-level while //--- these tallies are per-era. Writing the ratio directly removes the chance that //--- someone later logs or gates on one half and gets a number that decays with era //--- count. If either term is ever needed alone, it needs a per-era denominator. double eraScale = MathMax(0.3, MathMin(1.5, (double)m_oos.Hits() / m_oos.confidenceSum)); m_confidenceCalScale += (eraScale - m_confidenceCalScale) / Net.recentAverageSmoothingFactor; } // Per-class recall, reported and fed to the gate's two-sidedness test. A class with // FEWER than MIN_OOS_CLASS_SAMPLES_FOR_GATE true OOS samples this era doesn't block // (recallPct == -1 => treated as passing) so a thin OOS window doesn't deadlock // convergence early in a run. int buyRecallPct = m_oos.BuyRecallPct(MIN_OOS_CLASS_SAMPLES_FOR_GATE); int sellRecallPct = m_oos.SellRecallPct(MIN_OOS_CLASS_SAMPLES_FOR_GATE); int neutralRecallPct = m_oos.NeutralRecallPct(MIN_OOS_CLASS_SAMPLES_FOR_GATE); tel.buyRecall = buyRecallPct; tel.sellRecall = sellRecallPct; tel.neutralRecall = neutralRecallPct; m_lastBuyRecallPct = buyRecallPct; m_lastSellRecallPct = sellRecallPct; //--- Predicted-rate (share of ALL OOS bars this era the model called this class, //--- regardless of whether that call was right) and precision (of just those calls, //--- how many were right) - see tel.buyPred's declaration comment above for why //--- this is worth logging alongside recall. int oosEraBars = m_oos.Bars(); //--- ONE denominator for all of these, so the log's columns are directly comparable //--- rather than nearly so; -1 = nothing to divide by. Neutral is the RESIDUAL on both //--- layers: every OOS bar gets exactly one call, so what is not Buy and not Sell is //--- Neutral by construction. tel.buyTrue = m_oos.PctOfBars(m_oos.buyTotal); tel.sellTrue = m_oos.PctOfBars(m_oos.sellTotal); tel.neutralTrue = m_oos.PctOfBars(m_oos.neutralTotal); tel.neutralPred = m_oos.NeutralPredictedShare(); //--- The traded layer: candidates that cleared m_dirConfThreshold. A bar the operating //--- point rejects is a bar the model sits out. tel.buyFired = m_oos.PctOfBars(m_oos.buyFired); tel.sellFired = m_oos.PctOfBars(m_oos.sellFired); tel.neutralFired = m_oos.NeutralFiredShare(); tel.buyPred = m_oos.PctOfBars(m_oos.buyPredicted); tel.sellPred = m_oos.PctOfBars(m_oos.sellPredicted); tel.buyPrec = SOosTally::Pct(m_oos.buyPredictedHits, m_oos.buyPredicted); tel.sellPrec = SOosTally::Pct(m_oos.sellPredictedHits, m_oos.sellPredicted); //--- Live-fired precision (what actually trades - see m_oos.buyFired): of the directional //--- calls that cleared the confidence floor under the live/prior-corrected rule this era, //--- how many were right. Cached for the panel/log; -1 = the model fired none this era. tel.buyFiredPrec = SOosTally::Pct(m_oos.buyFiredHits, m_oos.buyFired); tel.sellFiredPrec = SOosTally::Pct(m_oos.sellFiredHits, m_oos.sellFired); m_lastBuyFiredPrecPct = tel.buyFiredPrec; m_lastSellFiredPrecPct = tel.sellFiredPrec; m_lastBuyFired = m_oos.buyFired; m_lastSellFired = m_oos.sellFired; //--- SELECTION METRIC. Ranking moved off balanced accuracy (macro-recall) 2026-07-30 //--- because that metric is maximized by exactly the model this system must never //--- deploy. //--- THE WHOLE DEPLOY DECISION, evaluated off the tally in one call. Tradeability, //--- the bar it had to clear and the ranking key all come out together, because they //--- are one decision - see Training\DeployGate.mqh for why splitting them was wrong. SDeployVerdict gate; gate.Evaluate(m_oos, EffectiveSampleSize((double)m_oos.DirCalls()), buyRecallPct, sellRecallPct); int oosDirCalls = m_oos.DirCalls(); int oosDirHits = m_oos.DirHits(); int oosDirTrue = m_oos.DirTrue(); bool coverageMeasurable = gate.measurable; double coveragePct = gate.coveragePct; double dirPrecPct = gate.precPct; double chancePrecPct = gate.chancePct; tel.coverage = (int)MathRound(coveragePct); tel.dirPrec = (int)MathRound(dirPrecPct); tel.chancePrec = (chancePrecPct >= 0.0) ? (int)MathRound(chancePrecPct) : -1; //--- Deployability. Replaces the per-class recall floor as the gate the checkpoint //--- selection and the plateau ladder's "is there anything safe to deploy" test //--- read. Observed 2026-08-01: the perceptron deployed at edge +0pp. double precSE = gate.precSE; double edgeFloorPct = gate.edgeFloorPct; //--- PUBLISHED so the era line can state the bar instead of leaving it implicit. //--- Nothing will ever clear an unreachable bar, and until this line printed it the //--- symptom was indistinguishable from "the models are close but not quite". m_lastEdgeFloorPct = edgeFloorPct; m_lastPrecSE = precSE; m_lastEffN = gate.effN; //--- CONTRIBUTE THIS ERA'S EVIDENCE TO THE CROSS-INSTRUMENT POOL, then read the pool //--- back. See PooledGate.mqh. if(coverageMeasurable && dirPrecPct >= 0.0 && chancePrecPct > 0.0) { PublishPoolRecord(chancePrecPct, dirPrecPct, m_lastEffN); m_lastPoolPasses = PooledGatePasses(m_lastPoolReport); } //--- BOTH sides must still be alive - see DEPLOY_MIN_SIDE_RECALL_PCT. A negative recall //--- means "not measurable this era" (no true bars of that class in the OOS window), and //--- that must not be read as a dead side, so it passes. bool bothSidesLive = gate.twoSided; //--- Tradeability is ALSO the lexicographic ranking key (isBetterEra) and the g_eta- //--- decay trigger, not merely a deploy-time check - see DeployGate.mqh. bool tradeableOK = gate.tradeable; double selectionScore = gate.selectionScore; //--- Under precision ranking the degenerate era is the one that called NOTHING //--- directional (precision undefined, nothing to trade), not one whose per-class //--- recall touched zero - a sparse high-precision model legitimately has low recall. //--- The old per-class recall FLOOR is gone with the rest of the anti-collapse devices: //--- the logit-adjusted loss is the one imbalance mechanism, and the gate's twoSided //--- test already refuses a one-class model at deploy time. bool isFullyCollapsedEra = gate.degenerate; //--- N for the family-wise deployment gate. Every era that COULD have won is //--- counted, whether it did or not - that is precisely the set the maximum was //--- taken over. if(coverageMeasurable && !isFullyCollapsedEra) m_deployCandidateEras++; //--- Lexicographic "better than the best-so-far" ordering: passing the directional //--- recall floor always outranks not passing it, regardless of blended //--- dOosForecast; only WITHIN the same pass/fail category does blended accuracy //--- break the tie. //--- tradeableOK / selectionScore, not directionalRecallOK / balancedOosEra - see //--- the SELECTION METRIC note above. //--- SAME NOISE BAND as the ensemble's isBetter (see PLATEAU_NEW_BEST_SIGMAS). A solo //--- run plateaus on exactly the same mechanism, so it must ratchet on exactly the same //--- rule - the two orderings are documented as identical and have to stay that way. double newBestBandEra = (m_bestSelectionScore >= 0.0) ? PLATEAU_NEW_BEST_SIGMAS * gate.scoreSE : 0.0; bool isBetterEra = (tradeableOK && !m_bestPassedRecall) || (tradeableOK == m_bestPassedRecall && bothSidesLive && !m_bestBothSidesLive) || (tradeableOK == m_bestPassedRecall && bothSidesLive == m_bestBothSidesLive && !isFullyCollapsedEra && selectionScore > m_bestSelectionScore + newBestBandEra); //--- The recall-pass-loss clause used to fire on ANY drop out of a full 3-way recall //--- pass, even a near-miss on one class at unchanged accuracy (e.g. observed: //--- Buy:56% Sell:41% Neutral:34% - Neutral alone missing the 40% floor by a few //--- points) - treating that identically to a total collapse back to Neutral-only. bool isWorseEra = selectionScore < m_bestSelectionScore - ETA_DECAY_REGRESSION_PCT; //--- ENSEMBLE: ranking and checkpointing belong to the ensemble as a unit (see //--- EnsembleCommitJointCheckpoint). The g_eta recovery bump still applies: that is //--- this net's own learning-rate dynamics, not a deployment decision. if(isBetterEra && m_ensembleMember) g_eta = MathMin(m_etaCeiling, g_eta / ETA_DECAY_FACTOR); if(isBetterEra && !m_ensembleMember) { //--- Snapshot BOTH scores at the checkpoint: m_bestSelectionScore is what ranking //--- compares against next era; m_bestOosForecast keeps the blended value //--- FinalizeTrainRun() and the restore branch reset dOosForecast to. m_bestOosForecast = dOosForecast; m_bestSelectionScore = selectionScore; m_bestPassedRecall = tradeableOK; m_bestBothSidesLive = bothSidesLive; //--- Raw significance inputs for the family-wise gate, taken at the same instant as the //--- weight snapshot below so the test always describes the weights that would ship. //--- selectionScore cannot substitute: it is precision x coverage credit, and the test //--- needs the unweighted precision plus the n that sets its standard error. m_bestDirPrecPct = dirPrecPct; m_bestChancePrecPct = chancePrecPct; m_bestDirCalls = oosDirCalls; //--- The operating point is part of the model, not of the run: these OOS numbers were //--- produced by these weights UNDER this threshold, and restoring one without the //--- other would deploy a model whose coverage and precision are not the ones the gate //--- cleared. Captured at the same instant as the weight snapshot below. m_bestDirConfThreshold = m_dirConfThreshold; //--- eval candidates are throwaway - track the score (above) but never write a //--- checkpoint file; m_haveOosCheckpoint=false then also skips the worse-era //--- RestoreWeights() restore. m_haveOosCheckpoint = Net.CaptureWeights(); //--- Recovery bump: ETA_DECAY_FACTOR-only ever shrinks g_eta, and previously //--- nothing ever grew it back - a losing streak early in a run (even a since- //--- corrected one) would permanently cap how fast every later era could learn //--- for the rest of the run, all the way down to ETA_MIN with no way back. g_eta = MathMin(m_etaCeiling, g_eta / ETA_DECAY_FACTOR); } else if(isWorseEra && m_bestOosForecast > 0) { // Decaying g_eta alone only softens FUTURE steps - it does nothing to undo the // regression this era already baked into the weights, so a run could (and in // practice did) spend 15+ eras compounding forward from one bad era's damage, // each new era fighting the last one's overshoot instead of building on the best // state found so far. Restore the last checkpointed-good weights before continuing // (mirrors what FinalizeTrainRun() does at the END of a run, just applied live so // the oscillation can't compound within a single run) - this is what actually turns // "reduce LR on regression" into "step back, then retry slower", not just "drift // slower". // // BOTH the restore AND the g_eta decay below are gated on m_bestPassedRecall: before // ANY era has ever cleared the per-class recall floor, isBetterEra's own // lexicographic ordering degrades to a pure blended-accuracy tiebreak // (directionalRecallOK==false on both sides of the comparison), so "best checkpoint" // during that phase just means "called Neutral most confidently so far" - restoring // it would actively defend the majority-class collapse against any era that trades // some accuracy for real Buy/Sell recall, which is exactly the bias this whole // recall-gate mechanism exists to prevent (see isBetterEra's own comment above). // Observed in practice: era 1-3 all "improved" on accuracy alone // (24.9%->41.4%->52.3%) while Buy/Sell recall stayed at a flat 0% the entire time - // restoring pre-pass would have locked training into that trajectory instead of // letting it explore past it. Decaying g_eta has the same bias one step removed: // every regression relative to a Neutral-collapse "best" shrinks g_eta a little more, // steadily strangling the exploration needed to escape that collapse until g_eta // bottoms out at ETA_MIN with no real solution ever found and no checkpoint to fall // back on either - observed in practice as a run whose best-ever blended accuracy // kept landing on 0%/0%/100% Buy/Sell/Neutral recall eras, each one triggering // another decay on the very next era, until g_eta floored out around era 20 and the // remaining eras just oscillated between collapse states with no way to make a // large-enough move to escape and no way to reset. Once m_bestPassedRecall is true, // there IS a genuinely good state worth protecting, and both restoring the // checkpoint and decaying g_eta on regression are safe/correct again. // Defend the checkpoint only once it sits clearly above the zero-skill rate it was // measured against - restoring (and decaying g_eta toward) a best that is itself // chance-level strangles the exploration needed to escape it. bool bestWorthDefending = (m_bestChancePrecPct > 0.0 && m_bestSelectionScore > m_bestChancePrecPct + WORTH_DEFENDING_MARGIN_PCT); //--- PATIENCE (see ETA_DECAY_PATIENCE_ERAS). The loop is self-sustaining and //--- cannot discover anything, because rolling the weights back is precisely //--- what removes the exploration that would end it. m_consecutiveRegressions++; if((m_bestPassedRecall || bestWorthDefending) && m_consecutiveRegressions >= ETA_DECAY_PATIENCE_ERAS) { m_consecutiveRegressions = 0; if(m_haveOosCheckpoint && Net.RestoreWeights()) { m_netDirty = true; // an in-memory weight swap is a mutation like any other dOosForecast = m_bestOosForecast; //--- The operating point goes back with the weights it was fitted for. //--- Leaving the current one in place would pair restored weights with a //--- threshold chosen for the rejected ones - see m_bestDirConfThreshold. m_dirConfThreshold = m_bestDirConfThreshold; //--- 2026-08-09 audit, F3: the snapshot restores WEIGHTS only, so without //--- this the Adam moments still encode the just-rejected trajectory and //--- the first updates after the restore push straight back toward the //--- state that was rolled back - the restore -> regress-again -> restore //--- oscillation. A restore is a new starting point; it gets a fresh //--- optimizer. Net.ResetOptimizerState(); } if(g_eta > ETA_MIN) g_eta = MathMax(ETA_MIN, g_eta * ETA_DECAY_FACTOR); Print(ID + ": OOS selection score (coverage-weighted dir-precision) regressed from best " + DeployScoreText(m_bestSelectionScore) + " to " + DeployScoreText(selectionScore) + " (blended " + DoubleToString(m_bestOosForecast, 1) + "%->" + DoubleToString(dOosForecast, 1) + "%) - restoring best checkpoint and decaying learning rate to " + DoubleToString(g_eta, 6)); } else //--- THROTTLED (2026-08-19): this no-action branch repeated ~600x/day while //--- noise wandered below a best it was never going to displace. The acting //--- branch above (restore + g_eta decay) still always prints - it changes state. if(TrainLogDue()) Print(ID + ": OOS selection score (coverage-weighted dir-precision) regressed from best " + DeployScoreText(m_bestSelectionScore) + " to " + DeployScoreText(selectionScore) + " (blended " + DoubleToString(m_bestOosForecast, 1) + "%->" + DoubleToString(dOosForecast, 1) + "%) - best so far is still within " + DoubleToString(WORTH_DEFENDING_MARGIN_PCT, 1) + "pp of its own chance rate, so there is nothing worth" + " restoring yet - continuing to explore without decaying the learning rate (still " + DoubleToString(g_eta, 6) + ")"); } //=== IN-SAMPLE ERROR PLATEAU: THE HONEST EARLY STOP ==================================== //--- The ladder below stops on the OOS SELECTION score. This stop reads the TRAINING //--- error instead, which the gate never looks at. Both stops exist; only this one //--- buys a lower bar. if(dError >= 0.0 && MathIsValidNumber(dError)) { //--- Relative improvement, so this does not depend on the loss's absolute scale. if(m_bestIsError < 0.0 || dError < m_bestIsError * (1.0 - IS_ERROR_IMPROVE_FRAC)) { m_bestIsError = dError; m_erasSinceBestIsError = 0; } else { m_erasSinceBestIsError++; //--- Deliberately more patient than the OOS ladder: training error is noisy //--- per era (mini-batch order alone moves it), and ending a run that is still //--- learning is far more expensive than a few wasted eras. if(m_erasSinceBestIsError >= TrainPlateauPatienceEras() * IS_ERROR_PATIENCE_MULT && m_haveOosCheckpoint && !m_isErrorPlateaued) { Print(ID + ": IN-SAMPLE ERROR PLATEAU - training error has not improved by " + DoubleToString(100.0 * IS_ERROR_IMPROVE_FRAC, 1) + "% in " + IntegerToString(m_erasSinceBestIsError) + " eras (best " + DoubleToString(m_bestIsError, 4) + ", now " + DoubleToString(dError, 4) + "). The optimiser has stopped learning from the data it CAN see, so further" " eras cannot find a better model - they would only add candidates to the" " family the deploy gate corrects over, raising the bar the winner has to" " clear. Ending the search and deploying the best checkpoint. This stop" " never read an out-of-sample number, which is what makes the smaller" " family legitimate rather than a peek."); //--- LATCH FIRST, and let the LATCH - not m_plateauStage - be what the deploy //--- conditions read. m_plateauStage is mirrored from the shared ensemble ladder //--- on every era (EnsembleEraVerdict), so writing the decision there meant it //--- survived until the next verdict and no longer. See m_isErrorPlateaued. m_isErrorPlateaued = true; m_plateauStage = PLATEAU_STAGE_DEPLOY; } } } //=== PLATEAU LADDER ==================================================================== //--- Neither branch above fires in the dead zone between "new best" and "regressed //--- by more than ETA_DECAY_REGRESSION_PCT". This is the response to sitting in it: //--- count eras since the last new best and escalate. if(m_ensembleMember) { EnsembleStashEraStats(dirPrecPct, chancePrecPct, oosDirCalls, tradeableOK, bothSidesLive, selectionScore, dOosForecast); //--- m_eraCount was already incremented at the top of this block, so the era that just //--- finished - the one the vote buffer is stamped with - is m_eraCount - 1. EnsembleOosPassComplete(m_eraCount - 1, g_eta); } else if(isBetterEra) { //--- Moving again: retire the ladder AND the restart boost. The checkpoint just //--- snapshotted this era regardless. The normal per-era g_eta schedule takes //--- over. if(m_plateauStage > 0) Print(ID + ": new best selection score " + DeployScoreText(m_bestSelectionScore) + "% - plateau escape worked, clearing plateau stage " + IntegerToString(m_plateauStage)); m_erasSinceBest = 0; m_plateauStage = 0; m_restartBoostErasLeft = 0; //--- Patience is about CONSECUTIVE regressions - an era that improves clears it, so a //--- run that alternates improve/regress never accumulates its way into a decay. m_consecutiveRegressions = 0; } else { m_erasSinceBest++; int dueStage = m_erasSinceBest / TrainPlateauPatienceEras(); if(dueStage > m_plateauStage) { m_plateauStage = dueStage; string stageNote = IntegerToString(m_erasSinceBest) + " eras with no new best selection score (best " + DeployScoreText(m_bestSelectionScore) + ")"; if(m_plateauStage == PLATEAU_STAGE_RESTART || m_plateauStage == PLATEAU_STAGE_ANNEAL) { //--- BOOSTED WARM RESTART: a plateau needs a bigger step to climb out of //--- its basin, not a smaller one - and "back to the ceiling" was a NO-OP //--- whenever the run plateaued without ever tripping the regression decay, //--- because g_eta was still AT the ceiling (2026-08-09 audit, F2). double etaBefore = g_eta; g_eta = m_etaCeiling * PLATEAU_RESTART_BOOST; m_restartBoostErasLeft = TrainPlateauPatienceEras(); //--- A restart is a new schedule: replaying the plateau's own accumulated //--- Adam momentum at 5x the rate would retrace the same basin, harder. Net.ResetOptimizerState(); //--- The focal-gamma anneal that used to accompany this went with focal //--- loss on 2026-07-31. Print(ID + ": PLATEAU stage " + IntegerToString(m_plateauStage) + " - " + stageNote + ". Boosted warm restart: learning rate " + DoubleToString(etaBefore, 6) + "->" + DoubleToString(g_eta, 6) + " (annealing back to " + DoubleToString(m_etaCeiling, 6) + " over " + IntegerToString(TrainPlateauPatienceEras()) + " eras), optimizer momentum reset. Best checkpoint is safe - this only changes how the NEXT eras train."); } else if(m_plateauStage >= PLATEAU_STAGE_DEPLOY) { //--- Exhausted: both escapes were tried and neither found a better //--- model, so this IS the best this configuration reaches. Safety: only //--- ever auto-deploys a checkpoint that CLEARED the per-class recall //--- floor (m_bestPassedRecall). double zBest = 0.0, pFam = 1.0; int nTried = 0; bool survivesSelection = BestCheckpointSurvivesSelection(zBest, pFam, nTried); string selectionNote = " | best-of-" + IntegerToString(nTried) + " test: edge " + DoubleToString(m_bestDirPrecPct - m_bestChancePrecPct, 1) + "pp on " + IntegerToString(m_bestDirCalls) + " calls = " + DoubleToString(zBest, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ")"; if(m_bestPassedRecall && m_haveOosCheckpoint && survivesSelection) Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + " across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " warm restarts. Training has converged on what this" + " configuration can reach - deploying the best checkpoint (dir-precision " + DeployScoreText(m_bestSelectionScore) + ", blended " + DoubleToString(m_bestOosForecast, 1) + "%)." + selectionNote + " - CLEARS."); else if(m_bestPassedRecall && m_haveOosCheckpoint) { //--- Passed the per-era floor but not the selection correction: //--- this is a maximum that a pure-noise search of this length //--- produces routinely. Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + ". The best checkpoint clears the per-era deployability floor but DOES NOT clear the" + " null of the MAXIMUM over the eras it was chosen from" + selectionNote + ". A best-of-N this large happens routinely when every era is a noise draw, so the" + " ranking carries no evidence of an edge and this model is not safe to trade." + " Restarting the plateau ladder and continuing to train; the " + IntegerToString(m_maxErasPerRun) + "-era cap remains the backstop."); m_erasSinceBest = 0; m_plateauStage = 0; } else { Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + ", but no checkpoint has ever cleared the deployability floor (directional calls on" + " at least a quarter as many bars as actually swing, at a precision above that base rate, with BOTH Buy and Sell" + " recall at or above " + DoubleToString(DEPLOY_MIN_SIDE_RECALL_PCT, 0) + "%), so there is nothing safe to" + " deploy. Restarting the plateau ladder and continuing to train rather than deploying a" + " one-class model; the " + IntegerToString(m_maxErasPerRun) + "-era cap remains the backstop."); m_erasSinceBest = 0; m_plateauStage = 0; } } } } //--- Restart-boost anneal (see PLATEAU_RESTART_BOOST): walk g_eta geometrically from //--- boost x ceiling back down to the ceiling over PLATEAU_PATIENCE_ERAS eras, one //--- step per completed era - the SGDR-style decaying half of the cycle, which is //--- what makes the boost a bounded kick instead of a new permanent rate. if(m_restartBoostErasLeft > 0) { g_eta = MathMax(m_etaCeiling, g_eta * MathPow(PLATEAU_RESTART_BOOST, -1.0 / TrainPlateauPatienceEras())); m_restartBoostErasLeft--; } m_oosWindow.Add(dOosForecast); while(m_oosWindow.Total() > STABILITY_WINDOW) m_oosWindow.Delete(0); m_oosStable = false; if(m_oosWindow.Total() >= STABILITY_WINDOW) { double oosMin = m_oosWindow.At(0), oosMax = m_oosWindow.At(0); for(int w = 1; w < m_oosWindow.Total(); w++) { oosMin = MathMin(oosMin, m_oosWindow.At(w)); oosMax = MathMax(oosMax, m_oosWindow.At(w)); } m_oosStable = (oosMax - oosMin) <= STABILITY_TOLERANCE; } //--- The dError<0.1 RMS-error floor is meaningful for the single-neuron regression //--- head (m_outputNeuronsCount==1), where it's the only convergence signal //--- available. bool errorGateOK = (m_outputNeuronsCount == 3) ? true : (dError < 0.1); //--- Convergence (unlike isBetterEra's ranking) FINALIZES the model, so both //--- directional classes must have actually been MEASURED this era. bool directionalRecallMeasured = (m_outputNeuronsCount != 3) || (buyRecallPct >= 0 && sellRecallPct >= 0); //--- VALIDITY of this era's model, no longer "did it hit a target accuracy". Neither //--- is what "train to the best result" means. m_objectiveMet = errorGateOK && directionalRecallMeasured; } //--- Only mark the persisted model "complete" once it actually converged this era - an //--- interruption (stop) or an ordinary in-progress era must stay flagged incomplete so //--- a restart resumes training instead of quietly treating a partial run as done. //--- ...and the family-wise selection gate, for the same reason the deploy branch applies it: //--- these two conditions MUST stay identical or the flag persisted into the .nnw disagrees //--- with the decision to stop, and a reload would run inference on a model the ladder had //--- refused to deploy. Cheap enough to re-evaluate per era (one normal-tail evaluation). double zConv = 0.0, pConv = 1.0; int nConv = 0; //--- ENSEMBLE: the verdict is the ensemble's, so the flag persisted into this member's //--- .nnw has to be the ensemble's too - otherwise a reload would run one member live //--- against three still training, which is not the model that was measured. m_trainingComplete = m_ensembleMember ? (g_ensDeployApproved && m_haveOosCheckpoint && m_checkpointEra == g_ensBestEra) : ((m_plateauStage >= PLATEAU_STAGE_DEPLOY || m_isErrorPlateaued) && m_bestPassedRecall && m_haveOosCheckpoint && BestCheckpointSurvivesSelection(zConv, pConv, nConv)); double currentIndicatorParams[]; m_indicatorTuner.Flatten(currentIndicatorParams); if(!Net.Save(m_activeFileName + ".nnw", dError, dUndefine, dForecast, dtStudied, m_activeFileCommon, m_eraCount, m_trainingComplete, currentIndicatorParams)) Print(__FUNCTION__ + ": ERROR - era-end Net.Save failed for " + m_activeFileName + ".nnw (era " + IntegerToString(m_eraCount) + "). Training continues but this era's checkpoint was NOT persisted - a crash/restart now would resume from an older era."); else m_netDirty = false; // disk now holds exactly these weights - see PersistWeightsOnShutdown if(!SaveModelStats(m_activeFileName, m_activeFileCommon)) // keep calibration state paired with the just-saved weights Print(__FUNCTION__ + ": ERROR - SaveModelStats failed for " + m_activeFileName + " (era " + IntegerToString(m_eraCount) + "). Calibration/online-learning state not persisted this era."); SaveShadowNet(currentIndicatorParams); } } } //+------------------------------------------------------------------+ //| START OF A TRAINING RUN - everything that happens once, before | //| era 0 rather than before every era. | //| | //| Waits (bounded, non-blocking across calls) for the terminal to | //| finish syncing history, sizes the study window, and arms the | //| one-shot preparatory walks. Several of its branches DEFER: they | //| decline this call and let the next scheduled one try again, which | //| is why this reports whether Train() should return rather than | //| falling through. | //| | //| True = this call is spent. False = the run is live, carry on. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::BeginTrainRun(STrainEra &era, const datetime startTrainBar) { if(!m_trainRunActive) { //--- Wait (briefly, bounded, non-blocking across calls) for the terminal to finish syncing //--- this symbol/period's history from the broker before computing the training window. if(!SeriesInfoInteger(m_symbol.Name(), PERIOD_CURRENT, SERIES_SYNCHRONIZED)) { uint syncNowTick = GetTickCount(); if(m_syncWaitStartTick == 0) m_syncWaitStartTick = syncNowTick; if(syncNowTick - m_syncWaitStartTick < 5000) { ReportTrainStall("waiting for history sync"); return true; // retry on the next scheduled call instead of blocking here } Print(ID + ": WARNING - history for " + m_symbol.Name() + " " + EnumToString(PERIOD_CURRENT) + " did not finish syncing after 5s; training window may still grow as more history arrives"); } m_syncWaitStartTick = 0; //--- 3 no-op passes before the era loop ever runs for a fresh start (see m_warmupPassesRemaining's //--- declaration comment) - each is its own separately-scheduled Train() call (this whole method //--- just returns, deferring to the next "New Bar"/timer-driven call), giving MT5's history sync //--- several real, wall-clock-separated chances to settle on top of the 5s soft wait just above, //--- before training commits to a bar count and starts populating the label cache below. if(m_warmupPassesRemaining > 0) { ReportTrainStall("history-settle warm-up pass"); m_warmupPassesRemaining--; PrintVerbose(ID + ": warm-up pass " + IntegerToString(3 - m_warmupPassesRemaining) + " of 3 (letting history sync settle before training starts)"); return true; } //--- ALL available history, floored by MinTrainYear - see TrainWindowStart(). dtStudied = TrainWindowStart(startTrainBar); //--- OOS-based objective + stability tracking: training only "converges" once the objective //--- is met AND OOS accuracy has held inside a tight band for the last few eras, so a single //--- lucky era can't get locked in as the final model. m_oosWindow.Clear(); m_bestOosForecast = -1; m_bestSelectionScore = -1; m_bestPassedRecall = false; m_bestBothSidesLive = false; m_haveOosCheckpoint = false; m_checkpointEra = -1; // the joint-checkpoint era stamp goes with the snapshot it describes m_oosStable = false; m_objectiveMet = false; //--- Family-wise deployment gate state, reset with the checkpoint tracking it describes: N counts //--- the eras THIS run selects a maximum over, so carrying it across runs would test the winner //--- against a search that never happened. m_bestDirPrecPct = -1.0; m_bestChancePrecPct = -1.0; m_bestDirCalls = 0; m_deployCandidateEras = 0; m_erasSinceCooldown = 0; m_eraResumePending = false; //--- Plateau ladder starts fresh with this run, so it re-walks the escalation from its own //--- starting point. (The focal-gamma anneal that used to reset here went with focal loss on //--- 2026-07-31 - the ladder's real escape is the learning-rate warm restart.) m_erasSinceBest = 0; m_plateauStage = 0; m_restartBoostErasLeft = 0; //--- Per-RUN like the ladder above, and for the same reason: a resumed run restarts the search, so //--- carrying a previous run's best training error would let it early-stop on the first era. m_bestIsError = -1.0; m_erasSinceBestIsError = 0; m_isErrorPlateaued = false; //--- ENSEMBLE: the shared gate state is per-RUN for the same reason the per-member state //--- above is - N must count the eras THIS run's maximum was taken over, so carrying it //--- across runs would test the winner against a search that never happened. bool ensembleRunAlreadyOpen = false; if(m_ensembleMember) for(int mi = 0; mi < ArraySize(g_warriorEnsemble); mi++) { CExpertSignalAIBase *mm = g_warriorEnsemble[mi]; if(CheckPointer(mm) != POINTER_INVALID && mm != GetPointer(this) && mm.m_trainRunActive) { ensembleRunAlreadyOpen = true; break; } } if(m_ensembleMember && !ensembleRunAlreadyOpen) { g_ensLastVerdictEra = -1; g_ensBestScore = -1.0; g_ensBestTradeable = false; g_ensBestTwoSided = false; g_ensBestPrecPct = -1.0; g_ensBestChancePct = -1.0; g_ensBestCalls = 0; g_ensBestEra = -1; g_ensBestCoveragePct = -1.0; g_ensBestMinCoverPct = -1.0; g_ensBestEdgeFloorPct = -1.0; //--- The pin belongs to the checkpoint, so clearing the checkpoint releases it. The next //--- scoring era takes the checkpoint unconditionally and re-pins from its own measurement. g_ensDerivedThreshold = -1.0; g_ensGateTestedEra = -1; // no gate test belongs to a run that has not happened yet g_ensCandidateEras = 0; g_ensErasSinceBest = 0; g_ensPlateauStage = 0; g_ensIsPlateauAnnounced = false; g_ensDeployApproved = false; } //--- One-time eager pre-scan for a fresh start (see m_labelCachePrebuilt's declaration //--- comment) - kick it off and defer era 0 until it's done, so era 0 can start with a real //--- class-balance oversampling ratio instead of the reps=1 fallback. if(!m_labelCachePrebuilt) { ReportTrainStall("arming the first label-cache prebuild"); StartLabelCachePrebuild(); return true; } m_trainRunActive = true; } //--- Set up and ready to run this call's chunk. return false; } //+------------------------------------------------------------------+ //| START OF ONE ERA, or resumption of one that yielded mid-chunk. | //| | //| Fresh era: re-measure the class priors, clear every per-era tally | //| together, size the IS/OOS split, and reset the four passes. The | //| resume arm restores the bar cursor exactly where the last chunk | //| left it - the two are one decision and stay in one place. | //| | //| Defers on the same contract as BeginTrainRun: true = this call is | //| spent, false = the era is ready to run passes. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::BeginEra(STrainEra &era) { if(!m_eraResumePending) { //--- COLD-INDICATOR BACKOFF (2026-08-13). Give them a few quiet seconds instead; the stall //--- reporter stays the loud diagnosis if it persists. if(m_coldSweepTick != 0) { if(GetTickCount() - m_coldSweepTick < 5000) { ReportTrainStall("cold-indicator backoff (all windows failed on a transient cause)"); return true; } m_coldSweepTick = 0; } int barsNow = (int)MathMin(Bars(m_symbol.Name(), PERIOD_CURRENT, dtStudied, TimeCurrent()) + m_historyBars, Bars(m_symbol.Name(), PERIOD_CURRENT)); //--- PRIME, THEN SETTLE, THEN SWEEP. if(!ResizeBuffers(barsNow) || !RefreshData()) { PrintFormat("%s: era start ABORTED - price/indicator buffers would not prepare for %d bars" " (priming ResizeBuffers/RefreshData failed); ending this training run, it re-arms" " on the next scheduled call", ID, barsNow); FinalizeTrainRun(); return true; } //--- Now wait out the depth rather than snapshotting it. Returns 0 while the count is still moving. int settled = SettledBars(barsNow, "training sweep"); if(settled <= 0) { ReportTrainStall("priming indicator history (holding the sweep until the calculated depth" " stops changing)"); return true; } //--- The floor is the one piece of policy that stays here: below TRAIN_MIN_CLAMPED_BARS a settled //--- depth is too thin to train anything worth measuring, so the run holds and the stall reporter //--- stays the loud diagnosis rather than producing a meaningless era. if(settled < barsNow) { if(settled < TRAIN_MIN_CLAMPED_BARS) { ReportTrainStall(StringFormat("indicator depth settled at %d bars, below the %d-bar floor" " for a trainable era", settled, TRAIN_MIN_CLAMPED_BARS)); return true; } barsNow = settled; //--- Re-prepare at the clamped depth, and ONLY when it actually changed: the primer above //--- already left every buffer refreshed at the full depth, so an unconditional second pass //--- would be a wasted CopyBuffer over every buffer, every era, on the charts that need none. if(!ResizeBuffers(barsNow) || !RefreshData()) { //--- The ONLY exit from Train() that tears down the whole run, and it used to be completely //--- silent - a transient buffer/history hiccup ended the run, FinalizeTrainRun() pushed //--- dtStudied to the last scanned bar, and the next era simply never started. Indistinguishable //--- from a hang while it was quiet, so it says so (2026-08-10). PrintFormat("%s: era start ABORTED - price/indicator buffers would not prepare for the" " settled depth of %d bars (ResizeBuffers/RefreshData failed); ending this" " training run, it re-arms on the next scheduled call", ID, barsNow); FinalizeTrainRun(); return true; } } era.bars = barsNow; ReportEraWindow(barsNow); //--- Cross-asset panel is indexed against exactly this bar grid, so it is (re)built wherever //--- the grid is - never per bar. Non-fatal on failure; see BuildCrossAssetPanel(). BuildCrossAssetPanel(barsNow); EnsureSpreadSeries(barsNow); era.addLoop = false; //--- Label/feature cache invalidation: MQL5 timeseries indices are always relative to "now" //--- (index 0 = current bar), so every new closed candle shifts every older bar's index - a //--- cache keyed by index would silently misalign the moment that happens. int barsBefore = m_labelCacheBars; datetime anchorBefore = m_labelCacheAnchorTime; datetime anchorNow = m_Time.GetData(0); if(EnsureBarCachesCapacity(era.bars) && m_labelCachePrebuilt) { //--- The failure mode: the era and the prebuild disagree about `bars`, or about which bar //--- is index 0, and re-arm each other forever - caches wiped, relabelled, wiped again, no //--- era ever runs. string sizeKey = (era.bars != barsBefore) ? StringFormat("SIZE CHANGED %d -> %d", barsBefore, era.bars) : "size unchanged"; string anchorKey = (anchorNow != anchorBefore) ? StringFormat("ANCHOR MOVED %s -> %s", TimeToString(anchorBefore), TimeToString(anchorNow)) : "anchor unchanged"; ReportTrainStall(StringFormat("cache invalidated at era start - %s, %s (era sized %d bars, cache" " held %d). An anchor that moves EVERY era with the size steady is" " a new candle each pass or a Time buffer that is not being" " refreshed; a size that moves is the era/prebuild disagreement.", sizeKey, anchorKey, era.bars, barsBefore)); StartLabelCachePrebuild(); return true; } //--- freeze the just-finished era's true class totals for this new era's priors (see //--- m_prevEraTrueBuyCount's declaration comment) before resetting the live counters below - EXCEPT //--- right after StartLabelCachePrebuild()/AdvanceLabelCachePrebuild() seeded them for era 0: the //--- live m_trueBuyCount/Sell/Neutral tally is still all-zero at that point (nothing trained yet), //--- so copying it here would silently stomp the real upfront tally back to an empty distribution. if(m_prebuildSeedPending) m_prebuildSeedPending = false; else { m_prevEraTrueBuyCount = m_trueBuyCount; m_prevEraTrueSellCount = m_trueSellCount; m_prevEraTrueNeutralCount = m_trueNeutralCount; } //--- Natural class base rates for the live logit-adjusted decision (see //--- AdjustedSignalFromSoftmax): derived from the same just-finished-era true class totals //--- the oversampling ratio uses, so live calibrates to exactly the distribution the model //--- was measured against. UpdateClassPriors(m_prevEraTrueBuyCount, m_prevEraTrueSellCount, m_prevEraTrueNeutralCount); //--- Re-install the training-time logit offsets from the priors just measured, so this //--- era's gradient tracks the distribution the era is scored against. ApplyLogitAdjustment(); m_countBuySignals = 0; m_countSellSignals = 0; m_countNeutralSignals = 0; m_trueBuyCount = 0; m_trueSellCount = 0; m_trueNeutralCount = 0; //--- All the OOS confusion counts at once. They are read together at era end, so they //--- must be cleared together - see 7452bd1 for what a partial reset of a tally group costs. m_oos.Reset(); //--- Declustered tally + its replay cursors. -1 / Neutral is "nothing seen yet this era", which is //--- what makes the first directional call of an era always survive rule 1. m_oosNmsFired = 0; m_oosNmsHits = 0; m_oosNmsLastBuyIdx = -1; m_oosNmsLastSellIdx = -1; m_oosNmsKeptIdx = -1; m_oosNmsKeptConf = 0.0; m_oosNmsKeptDir = Neutral; ArrayInitialize(m_oosTierFired, 0); ArrayInitialize(m_oosTierHits, 0); // Nearest-to-present slice of this era's bars is held out as OOS and never backprop'd on; // the rest (older bars) is the IS/training slice. era.totalIter = (int)MathMax(era.bars - MathMax(m_historyBars, 0), 0); era.oosCutoff = (int)(MathMax(0, MathMin(100, m_oosSplitPct)) / 100.0 * era.totalIter); era.i = (int)(era.bars - MathMax(m_historyBars, 0) - 1); //--- Fresh era: reset pass 2's shuffled-backprop queue (see m_isTrainQueue's declaration //--- comment). ArrayResize(m_isTrainQueue, era.totalIter * 4); m_isTrainQueueCount = 0; //--- Heartbeat baseline for this era - see the member declarations for why this exists. m_eraStartTick = GetTickCount(); m_passFeatUs = 0; m_passNetUs = 0; m_passWindowOk = 0; m_passWindowFail = 0; m_passHeartbeatPrints = 0; m_lastHeartbeatTick = 0; m_isTrainCursor = 0; m_isPass2Active = false; m_isPass2Done = false; m_isCalibActive = false; m_isCalibDone = false; m_isPass3Active = false; //--- Fresh per-era predicted-signal cache for the end-of-era NMS sweep (see PruneDirectionalClusters). //--- -2 = "not scored this era" so stale bars from a longer prior era can't draw phantom arrows. if(m_signalClusterWindow > 0) { ArrayResize(m_arrowSignalCache, era.bars); ArrayInitialize(m_arrowSignalCache, -2.0); } } else { //--- resuming a chunk that yielded mid-bar-loop last call - pick up exactly where it left off era.bars = m_resumeBars; era.totalIter = m_resumeTotalIter; era.oosCutoff = m_resumeOosCutoff; era.addLoop = m_resumeAddLoop; era.i = m_resumeBarIndex; m_eraResumePending = false; } //--- Set up and ready to run this call's chunk. return false; } //+------------------------------------------------------------------+ //| WHAT PASS 1 FOUND, said out loud. Reporting only - it decides | //| nothing and the era proceeds identically either way. | //| | //| Two outcomes worth a line. A sweep that produced NO usable | //| window at all gets a full autopsy naming the lookback slot, the | //| guard that rejected it and the per-indicator depth, because the | //| era is discarded and restarts and the symptom is otherwise a | //| silent loop - the backoff this arms was dead until 2026-08-17, | //| which is why the USDJPY/XAUUSD stall never recovered. | //| | //| A HEALTHY first sweep is the one moment the assembled feature | //| vector is known readable and not yet trained on, and the first | //| point at which the bar grid, the barrier geometry and the label | //| lifespan are all real numbers rather than defaults - so the | //| block-level autopsy and the detectability report belong here and | //| nowhere else. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportPass1Outcome(STrainEra &era) { //--- PASS 1 IS OVER (the yield above is the only other way out of that loop). This is the //--- point that decides whether the era does any work at all, and until now it said nothing. //--- add_loop is exactly "m_passWindowOk > 0". if(!era.stop) { if(!era.addLoop) { //--- SELF-HEAL BEFORE RESTARTING. A sweep that produced no usable window at all will //--- produce exactly the same result next time unless something changes, because every //--- bar it touched is now answered from the feature cache. ArrayInitialize(m_featureCacheHasValue, false); //--- Routed through ReportTrainStall rather than printed directly: a discarded era //--- restarts immediately, so this condition repeats as fast as pass 1 can sweep, and //--- an unthrottled line would bury the journal. string whyLine; if(m_windowFailSlot == -2) whyLine = "no window has been attempted yet this run (m_windowFailSlot unset) - the" " failure is upstream of BuildFeatureWindow"; else if(m_windowFailSlot < 0) whyLine = StringFormat("every lookback bar was ACCEPTED and the window was still" " short: %d of %d values. A feature block emitted fewer values" " than m_neuronsCount promises", m_windowFailTotal, (int)m_historyBars * m_neuronsCount); else //--- THE BLOCK, NOT JUST THE SLOT. A total failure (ok=0) is itself evidence: it //--- means the newest anchors failed too, which no depth shortfall can cause. { string byBlock = m_featureFailBlock; if(byBlock == "") byBlock = "(no guard recorded - the rejection came from a TempData.Add failure," " not a data guard)"; whyLine = StringFormat("lookback slot %d of %d REJECTED the bar at series index %d" " (window had %d of %d values). REJECTED BY: %s. Slot 0 is the" " DEEPEST lookback of the window, so with ok=0 the newest anchors" " failed as well - which rules out a plain history-edge read and" " points at a buffer that is unreadable at every index." " Per-indicator depth:%s", m_windowFailSlot, (int)m_historyBars, m_featureFailIdx, m_windowFailTotal, (int)m_historyBars * m_neuronsCount, byBlock, IndicatorDepthReport()); } ReportTrainStall(StringFormat("pass 1 finished but NOT ONE of %d scanned bars produced a" " usable feature window, so the era is discarded and restarts" " from scratch (feature cache dropped so the next sweep" " recomputes) - windows ok=%d failed=%d, BuildFeatureWindow" " needs %d values per bar (historyBars=%d x featuresPerBar=%d)" " over %d bars | LAST FAILURE: %s", era.totalIter, m_passWindowOk, m_passWindowFail, (int)m_historyBars * m_neuronsCount, (int)m_historyBars, m_neuronsCount, era.bars, whyLine)); //--- transient cause (cold indicator) -> arm the era-start backoff instead of //--- resweeping at full speed; see the backoff block at the top of the fresh-era //--- branch. The mechanism was right; nothing reached it. THIS BACKOFF WAS DEAD UNTIL //--- 2026-08-17 and that is why the USDJPY/XAUUSD stall never recovered. m_coldSweepTick = GetTickCount(); } else { //--- Healthy pass 1. FIRST HEALTHY SWEEP is the only moment the assembled feature //--- vector is known to be readable and not yet been trained on - so it is where the //--- block-level autopsy belongs. ReportFeatureHealth(era.bars); //--- Same moment, same reason: the first sweep that produced usable windows is the //--- first point at which the era's bar grid and the measured label lifespan are real //--- numbers rather than defaults. ReportDetectability(era.oosCutoff); const uint PASS1_LOUD_AFTER_MS = 10000; //--- The calibration band is reported here, beside the queue count it is subtracted from, so //--- the two are read together: a run where the band silently came out empty (see //--- CalibBandBars) is one whose operating point is no longer being refitted at all, and the //--- only place that is visible is next to the number it should have reduced. string pass1Line = StringFormat("%s: era %d pass 1 done in %.0fs - %d of %d bars usable" " (%d failed, normal over the oldest bars), %d queued for" " backprop | %d bars held out to calibrate the operating" " point (+2x%d purged around it)", ID, (int)m_eraCount, (GetTickCount() - m_eraStartTick) / 1000.0, m_passWindowOk, m_passWindowOk + m_passWindowFail, m_passWindowFail, m_isTrainQueueCount, CalibBandBars(era.totalIter, era.oosCutoff), CalibPurgeBars()); if(GetTickCount() - m_eraStartTick >= PASS1_LOUD_AFTER_MS) Print(pass1Line); else PrintVerbose(pass1Line); } } } //+------------------------------------------------------------------+ //| THE ERA COMPLETED. Count it, age the shadow net, and decide | //| whether the RUN ends here. | //| | //| Two ways it does. The plateau ladder (or the ensemble gate) says | //| there is something worth deploying - the normal, wanted ending. | //| Or the era cap is reached, which is not a verdict about the model | //| at all: it asks the operator, and a "deploy anyway" is an explicit| //| choice that the automatic ladder would have refused, so it says | //| so plainly rather than letting the deploy read as a clean pass. | //| | //| Self-guarding on era.addLoop: an era that produced no usable bars | //| is not an era and must not advance the counter, or the cap and | //| the plateau ladder both measure work that never happened. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::AdvanceEra(STrainEra &era, SEraTelemetry &tel) { //--- era complete (ran out of bars) or a stop was requested mid-era if(era.addLoop) { m_eraCount++; m_erasSinceCooldown++; //--- EMA shadow-weight deployment: blend the shadow a small step (SHADOW_WEIGHT_TAU) //--- toward Net's just-updated weights, every era - see COnlineLearning's class comment. EnsureShadowNet(); m_onlineLearning.BlendTowardNet(); //--- Status-label progress is invisible with no chart (headless/optimization runs), and //--- even in visual mode a long training run can otherwise look "stuck" for a long time //--- with no Journal output at all - log progress at most every ~5s (real wall-clock, not //--- simulated time) so an operator can tell it's actively working, not hung. uint nowTick = GetTickCount(); tel.shouldLog = (nowTick - m_lastProgressLogTick >= 5000); if(tel.shouldLog) m_lastProgressLogTick = nowTick; //--- Era cap. There used to be a second, much smaller cap here for throwaway auto-tune //--- candidates; the filter tuner does not train candidates at all, so only the real one remains. int effectiveEraCap = m_maxErasPerRun; //--- PLATEAU LADDER, terminal stage: training stopped improving and both escape attempts //--- (two learning-rate warm restarts) failed to find anything better - see the ladder in //--- the era-end block below, which is what raised m_plateauStage this far and already //--- logged why. bool deployNow = m_ensembleMember ? (g_ensDeployApproved && m_haveOosCheckpoint) : ((m_plateauStage >= PLATEAU_STAGE_DEPLOY || m_isErrorPlateaued) && m_bestPassedRecall && m_haveOosCheckpoint); if(deployNow) { era.stop = true; m_trainingComplete = true; } else if(effectiveEraCap > 0 && m_erasSinceCooldown >= effectiveEraCap) { //--- Era cap reached without converging: ask the operator whether to keep training or //--- deploy the best checkpoint and stop (see PromptContinuePastEraCap / m_maxErasPerRun). if(PromptContinuePastEraCap(dOosForecast)) { m_erasSinceCooldown = 0; // keep training - reset the cap window Print(ID + ": hit the " + IntegerToString(m_maxErasPerRun) + "-era cap (best score " + DeployScoreText(m_bestSelectionScore) + ", blended OOS " + DoubleToString(dOosForecast, 1) + "%) - CONTINUING training by operator choice."); } else { //--- stop: end THIS era loop now; FinalizeTrainRun (reached via the stop path below, //--- because stop==true) deploys the best checkpoint. See OnlineLearnStep()'s gate. era.stop = true; //--- Operator DELIBERATELY chose to deploy this best checkpoint as the final model. //--- Note the m_trainingComplete=(m_objectiveMet&&m_oosStable) line below is inside //--- if(!stop), so it can't clobber this back to false on this path. m_trainingComplete = true; Print(ID + ": hit the " + IntegerToString(m_maxErasPerRun) + "-era cap before the plateau ladder finished (best score " + DeployScoreText(m_bestSelectionScore) + ", blended OOS " + DoubleToString(dOosForecast, 1) + "%) - operator chose to DEPLOY the best checkpoint as final (marked complete; reloads will run inference, not retrain). Reaching this cap now means the run was still finding new bests, or never cleared the deployability floor - raise the era cap for the former."); //--- NOT blocked - this branch is an explicit operator decision and stays one. But the //--- automatic ladder would refuse this model, so say so plainly rather than letting the //--- deploy read as a clean pass. See DEPLOY_FAMILY_WISE_ALPHA. ReportSelectionGateVerdict("era-cap deploy"); } } } } //+------------------------------------------------------------------+ void CExpertSignalAIBase::Train(datetime StartTrainBar = 0) { //--- THIS CALL'S WORKING STATE. One object rather than eight locals threaded through four //--- passes - see STrainEra for why the passes could not be separated while it was eight. STrainEra era; //--- Max wall-clock work per call before yielding - see m_trainRunActive's declaration comment //--- for why chunking exists at all. //--- ENSEMBLE: the members share the one chart thread and their chunks queue back-to-back within //--- one timer event, so the TOTAL across members - not the per-member share - is the worst-case //--- latency between a panel click and a free thread. 480ms total (4 x 120) was the documented //--- "drags stickily, buttons miss clicks" regime (2026-08-15); 120ms total was fully reactive //--- but left the chart thread idle 76% of every 500ms timer period - a 4x dilution on top of //--- everything else, on a box where training is the bottleneck. 300ms total (2026-08-25, //--- operator prioritised training throughput over UI smoothness) sits between the two: ~60% //--- duty cycle, ~2.5x the old throughput, click latency bounded at ~300ms while training runs. era.budgetMs = m_ensembleMember ? (uint)(300 / MathMax(EnsembleActiveTrainers(), 1)) : 300; if(TrainCallPreempted(era)) return; if(BeginTrainRun(era, StartTrainBar)) return; if(BeginEra(era)) return; // Restore this model's own learning-rate trajectory into the shared global right before this // chunk's backProp() calls touch it - see m_modelEta's declaration comment. g_eta = m_modelEta; era.chunkStartTick = GetTickCount(); // Iterate over the bars - skipped entirely when resuming straight into pass 2, OR when resuming // into a still-unfinished pass 3 (see m_isPass2Done's declaration comment for why checking // m_isPass2Active alone isn't enough to detect the latter case): pass 1 already fully completed // in an earlier call either way. //--- EVERY PASS BELOW CAN YIELD MID-CHUNK, and a yield means THIS CALL is spent, not that the pass //--- finished. Each one signals it by going through StashEraResume - the single writer of //--- m_eraResumePending - and then returning from its own method. Those returns used to be returns //--- from Train() itself; when the four passes were extracted into methods (08c2cec) they became //--- returns from a void helper, and this function carried on regardless: it reported pass 1 as //--- "done" after one 120ms budget, trained pass 2 on the sliver pass 1 had queued so far, scored //--- an OOS slice of it and let AdvanceEra count an era. Measured 2026-08-24 on SP500 H4: ~1,400 //--- "eras" in ten minutes, each one a ~1,200-bar chunk of a 16,264-bar window, and the oldest 90% //--- of the history never reached at all. if(!m_isPass2Active && !m_isPass2Done) { RunPass1(era); //--- Before ReportPass1Outcome, deliberately: that report decides whether the era did any work //--- at all, and a yielded pass 1 has no answer to give yet. if(m_eraResumePending) return; ReportPass1Outcome(era); } //--- Pass 2: replay the bars pass 1 queued into m_isTrainQueue for backProp, in a freshly //--- shuffled order - see m_isTrainQueue's declaration comment for the full rationale. RunPass2(era); if(m_eraResumePending) return; //--- Pass 2.5: the CALIBRATION walk. RunCalibrationPass(era); if(m_eraResumePending) return; //--- Pass 3: OOS scoring, chronological, AFTER pass 2 has actually trained on this era's IS data - //--- see m_isPass3Active's declaration comment for why this can no longer happen inline during //--- pass 1's scan. RunOosPass(era); if(m_eraResumePending) return; //--- What this era has to report, filled in below and rendered by ReportEraProgress. Every //--- field starts at -1 = not measured, which is what era 0 and any stopped era report. SEraTelemetry tel; AdvanceEra(era, tel); CompleteEra(era, tel); ReportEraProgress(tel); //--- Genuine convergence THIS era (not a stale m_trainingComplete carried over from a //--- previous run) - (re)start the evaluation-only continual-learning OOS walk. if(!era.stop && m_trainingComplete) { Print(ID + ": training CONVERGED at era " + IntegerToString(m_eraCount) + " - this is the best this configuration reached: dir-precision " + DeployScoreText(m_bestSelectionScore) + ", blended OOS " + DoubleToString(dOosForecast, 1) + "%, IS error " + DoubleToString(dError, 2) + ". No new best for " + IntegerToString(m_erasSinceBest) + " eras across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " learning-rate warm restarts." + " Weights saved, switching to live inference."); StartOosContinualSimulation(era.bars, era.oosCutoff); } if(era.stop || m_trainingComplete) FinalizeTrainRun(); //--- Deliberately AFTER FinalizeTrainRun(): that call restores the DEPLOYED checkpoint's weights //--- (which may differ from the last era's, if the plateau ladder's best era wasn't the last one //--- run), and this backfill must score with exactly what is about to trade live. if(!era.stop && m_trainingComplete) StartPatternDatabaseBackfill(era.bars, era.totalIter, era.oosCutoff); //--- else: this era is done but the run continues - the next Train() call (re-triggered via //--- ScheduleTrainingIfNeeded()'s custom event, same mechanism as always) starts the next era //--- fresh, since m_eraResumePending is false while m_trainRunActive stays true //--- Save this model's own learning-rate trajectory back out of the shared global before //--- returning - see m_modelEta's declaration comment. Covers every path that reaches here //--- (natural era completion, whether or not the run itself just finalized). m_modelEta = g_eta; } //+------------------------------------------------------------------+ //| Ends the current Train() run: restores the best-scoring era's | //| checkpointed weights (if any beat the era the loop happened to | //| end on), persists final state, and clears the resumable-run | //| flags. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::PromptContinuePastEraCap(double bestOos) { //--- No GUI in the Strategy Tester/optimizer - MessageBox() is unavailable there and would just //--- stall a headless run, so deploy the best checkpoint found so far and stop (the safe default). if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD)) return false; //--- Reaching this cap is now the UNUSUAL outcome: a run normally ends itself when the plateau //--- ladder runs out of escapes (see the PLATEAU_* constants), which is a statement about the //--- run having stopped improving rather than about any accuracy number. string neutralNote = (m_priorNeutral > 0.0) ? ("inflated by the ~" + IntegerToString((int)MathRound(m_priorNeutral * 100.0)) + "% Neutral base rate") : "inflated by the dominant Neutral class"; string reasons = ""; if(!m_bestPassedRecall) reasons += " - No era has ever cleared the deployability floor, so there is no model safe to\n" + " auto-deploy yet (a one-sided or chance-level model must never ship)\n"; else reasons += " - Still improving: " + IntegerToString(m_erasSinceBest) + " eras since the last new best, plateau stage " + IntegerToString(m_plateauStage) + " of " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " (the run ends itself at stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + ")\n"; if(!m_objectiveMet) reasons += " - The latest era did not produce a valid model (per-class recall not measured)\n"; string balancedStr = (m_bestSelectionScore > 0.0) ? ("\nBest directional precision, coverage-weighted (the metric the deployed\ncheckpoint is chosen on): " + DeployScoreText(m_bestSelectionScore) + "\nBest blended OOS accuracy: " + DoubleToString(bestOos, 1) + "% (" + neutralNote + ")\n") : ""; string msg = ID + ": training reached the " + IntegerToString(m_maxErasPerRun) + "-era cap before it finished on its own.\n\n" + "Training now runs until it stops improving, then deploys its best model. Status:\n" + reasons + balancedStr + "\nContinue training?\n\n" + "Yes = keep training for another " + IntegerToString(m_maxErasPerRun) + " eras\n" + "No = deploy the best checkpoint so far and stop training"; int res = MessageBox(msg, "Warrior EA - training", MB_YESNO | MB_ICONQUESTION); return (res == IDYES); } //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| See the declaration comment - the single deploy-persistence path. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::PersistDeployedModel(void) { if(CheckPointer(Net) == POINTER_INVALID) return; double currentIndicatorParams[]; m_indicatorTuner.Flatten(currentIndicatorParams); if(!Net.Save(m_activeFileName + ".nnw", dError, dUndefine, dForecast, dtStudied, m_activeFileCommon, m_eraCount, m_trainingComplete, currentIndicatorParams)) Print(__FUNCTION__ + ": ERROR - Net.Save failed for " + m_activeFileName + ".nnw. The deployed model was NOT persisted to disk."); else m_netDirty = false; // disk now holds exactly these weights - see PersistWeightsOnShutdown //--- Deploy-time gate: does this model's pure-MQL5 forward pass match the backend? If so, an //--- inference-only backtest can run DLL-free (see ValidateCpuInference / CNet::SetCpuInference). //--- Persisted into the .stats written next. Chart-only; safe-false everywhere else. m_mqlInferenceValidated = ValidateCpuInference(); if(!SaveModelStats(m_activeFileName, m_activeFileCommon)) // keep calibration state paired with the just-saved weights Print(__FUNCTION__ + ": ERROR - SaveModelStats failed for " + m_activeFileName + ". Calibration state not persisted."); SaveShadowNet(currentIndicatorParams); } //+------------------------------------------------------------------+ void CExpertSignalAIBase::FinalizeTrainRun(void) { //--- A run stopped mid-pass-2.5 or mid-pass-3 never reached that pass's own unfreeze, so lift //--- the scoring freeze here before anything else touches the net - the deployed model must //--- adapt live (see the freeze at pass-3 start, and the identical one the calibration walk //--- takes for the same reason). if(CheckPointer(Net) != POINTER_INVALID) { Net.SetBatchNormFrozen(false); Net.FlushBatch(); Net.SetBatchSize(1); } //--- deploy the most stable/best-scoring era's weights rather than whatever the run happened to //--- end on (which may reflect drift after the objective was first hit, or an aborted run). if(m_haveOosCheckpoint) { if(Net.RestoreWeights()) { //--- The in-memory swap makes the live net differ from the last save until //--- PersistDeployedModel (below) or the shutdown save writes it back down. m_netDirty = true; dOosForecast = m_bestOosForecast; //--- Deploy the checkpoint's operating point alongside its weights - the OOS coverage and //--- precision this run is about to report were measured with this pair together. m_dirConfThreshold = m_bestDirConfThreshold; //--- Same F3 reset as the mid-run restore: the deployed weights are the checkpoint's, so the //--- optimizer state that continues from here (online continual learning backprops on this //--- same net - see OnlineLearnStep) must not be the dead run's momentum. Net.ResetOptimizerState(); RefreshLatestSignal(); //--- NOT during shutdown. RestoreWeights() above is an in-MEMORY swap, so the best //--- checkpoint is already the live net by this line - and OnDeinit's //--- PersistWeightsOnShutdown() is about to write exactly those weights anyway. if(!m_shutdownInProgress) PersistDeployedModel(); } } //--- Clean up any legacy on-disk checkpoint from an older (file-based) build so it can't linger. int checkpointFlags = m_activeFileCommon ? FILE_COMMON : 0; if(FileIsExist(m_activeFileName + "_ckpt.tmp", checkpointFlags)) FileDelete(m_activeFileName + "_ckpt.tmp", checkpointFlags); //--- (dtStudied used to be held back while scoring a throwaway candidate - that marker belongs //--- to the DEPLOYED model's "studied up to" state; a candidate eval must leave it untouched. The //--- checkpoint block above is already inert in eval mode (m_haveOosCheckpoint stays false). if(m_eraCount > 0) dtStudied = m_lastBarTime; m_trainRunActive = false; m_eraResumePending = false; m_haveOosCheckpoint = false; m_checkpointEra = -1; // the joint-checkpoint era stamp goes with the snapshot it describes //--- Persist the arrows now drawn on the chart so a deploy/stop survives a later re- //--- add/recompile without a retrain (durable even if the terminal never gets a clean OnDeinit). SaveChartSignals(!m_trainingStopRequested); } #endif // WARRIOR_AIBASE_TRAINING_MQH