//+------------------------------------------------------------------+ //| Warrior_EA | //| AnimateDread | //| | //| Era loop, plateau ladder, checkpoint selection, deploy/finalise.| //| | //| PARTIAL IMPLEMENTATION FILE - not standalone. | //| This holds CExpertSignalAIBase method BODIES only. The class | //| declaration lives in Expert\ExpertSignalAIBase.mqh, which | //| #includes this file at the bottom, after the declaration. Do not | //| include it anywhere else and do not compile it on its own. | //| | //| Split out purely to make the 8216-line original navigable; the | //| code inside was moved verbatim, not rewritten. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AIBASE_TRAINING_MQH #define WARRIOR_AIBASE_TRAINING_MQH //+------------------------------------------------------------------+ //| Upper tail of the standard normal - see the declaration. | //+------------------------------------------------------------------+ double CExpertSignalAIBase::NormalUpperTail(double z) { if(!MathIsValidNumber(z)) return 1.0; // unusable input reads as "not significant" if(z < 0.0) return 1.0 - NormalUpperTail(-z); //--- ntB* / ntP, not the b1..b5 / p the reference prints: AI\Network.mqh line 79 does //--- "#define b1 AdamBeta1" (and b2 likewise), so a local named b1 here is macro-expanded into the //--- Adam beta INPUT and the compiler warns that it hides a global. Renamed rather than un-defining //--- the macro, which the whole Adam path reads. const double ntP = 0.2316419; const double ntB1 = 0.319381530, ntB2 = -0.356563782, ntB3 = 1.781477937; const double ntB4 = -1.821255978, ntB5 = 1.330274429; double t = 1.0 / (1.0 + ntP * z); double pdf = MathExp(-0.5 * z * z) / MathSqrt(2.0 * M_PI); double poly = t * (ntB1 + t * (ntB2 + t * (ntB3 + t * (ntB4 + t * ntB5)))); return MathMax(0.0, MathMin(1.0, pdf * poly)); } //+------------------------------------------------------------------+ //| Does the checkpoint about to deploy survive having been CHOSEN? | //| | //| The per-era test (EDGE_MIN_SIGMAS, see tradeableOK) asks "is this | //| era's edge more than 2 standard errors above chance". Asked once, | //| that is a fair question. Asked of every era in a run, and then | //| answered with the best one, it is the null-of-the-maximum error | //| this project has now found in four separate places - and this is | //| the instance that ships a model to a live account. | //| | //| Same shape as ReportBarrierGeometryScan's winner test and the | //| indicator tuner's Sidak correction, applied to the era search: | //| z = (precision - chance) / SE, SE = sqrt(p0(1-p0)/n) | //| p_single = P(Z >= z) | //| p_family = 1 - (1 - p_single)^N | //| and deployment needs p_family <= DEPLOY_FAMILY_WISE_ALPHA. | //| | //| Uses the checkpoint's OWN snapshotted precision/chance/call count, | //| not the latest era's, because the model that deploys is the one | //| that has to clear the bar. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::BestCheckpointSurvivesSelection(double &zObs, double &pFamily, int &nTried) { zObs = 0.0; pFamily = 1.0; nTried = MathMax(m_deployCandidateEras, 1); //--- No ranked era yet, or a degenerate chance rate: nothing to test, so nothing to deploy. if(m_bestDirCalls <= 0 || m_bestDirPrecPct < 0.0 || m_bestChancePrecPct <= 0.0 || m_bestChancePrecPct >= 100.0) return false; double p0 = m_bestChancePrecPct / 100.0; double se = 100.0 * MathSqrt(p0 * (1.0 - p0) / m_bestDirCalls); if(se <= 0.0) return false; zObs = (m_bestDirPrecPct - m_bestChancePrecPct) / se; double pSingle = NormalUpperTail(zObs); //--- 1-(1-p)^N directly. At the magnitudes in play (p ~ 1e-4..1e-2, N ~ 10..1000) double precision is //--- ample; no need for the log1p/expm1 form MQL5 would not give us anyway. pFamily = 1.0 - MathPow(1.0 - pSingle, (double)nTried); return (pFamily <= DEPLOY_FAMILY_WISE_ALPHA); } //+------------------------------------------------------------------+ //| ENSEMBLE GATE - the same test as above, asked of the VOTE. | //| See the ENSEMBLE DEPLOY GATE block in ExpertSignalAIBase.mqh for | //| why the vote rather than the member is the thing being gated. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::EnsembleSurvivesSelection(double &zObs, double &pFamily, int &nTried) { zObs = 0.0; pFamily = 1.0; nTried = MathMax(g_ensCandidateEras, 1); if(g_ensBestCalls <= 0 || g_ensBestPrecPct < 0.0 || g_ensBestChancePct <= 0.0 || g_ensBestChancePct >= 100.0) return false; double p0 = g_ensBestChancePct / 100.0; double se = 100.0 * MathSqrt(p0 * (1.0 - p0) / g_ensBestCalls); if(se <= 0.0) return false; zObs = (g_ensBestPrecPct - g_ensBestChancePct) / se; pFamily = 1.0 - MathPow(1.0 - NormalUpperTail(zObs), (double)nTried); return (pFamily <= DEPLOY_FAMILY_WISE_ALPHA); } //+------------------------------------------------------------------+ //| JOINT CHECKPOINT: snapshot EVERY member's weights, at this one | //| era, and commit each member's own era statistics as the stats its | //| best checkpoint is described by. | //| | //| Correct only because of the era barrier - see the ENSEMBLE DEPLOY | //| GATE block. Every member has finished era `votedEra` and is held | //| before era votedEra+1, so all four sets of weights belong to the | //| same measured instant. Without the barrier this would snapshot | //| whatever era each member happened to be mid-way through. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleCommitJointCheckpoint(const long votedEra) { int captured = 0, members = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; members++; //--- The member's OWN figures at the winning era. They describe this member's contribution to a //--- checkpoint the ENSEMBLE selected, which is why they are committed from the stash rather //--- than from a per-member ranking: no member "won" this era, the vote did. mm.m_bestOosForecast = mm.m_eraStatBlended; mm.m_bestBalancedOos = mm.m_eraStatScore; mm.m_bestPassedRecall = mm.m_eraStatTradeable; mm.m_bestBothSidesLive = mm.m_eraStatTwoSided; mm.m_bestDirPrecPct = mm.m_eraStatPrecPct; mm.m_bestChancePrecPct = mm.m_eraStatChancePct; mm.m_bestDirCalls = mm.m_eraStatCalls; mm.m_bestDirConfThreshold = mm.m_eraStatThreshold; //--- In-memory snapshot, same primitive the solo path uses. A member whose capture fails keeps //--- m_haveOosCheckpoint false and is reported - it would otherwise deploy whatever weights it //--- happens to hold at the end of the run, silently breaking the "deploy what was measured" //--- guarantee this whole mechanism exists for. if(CheckPointer(mm.Net) != POINTER_INVALID && mm.Net.CaptureWeights()) { mm.m_haveOosCheckpoint = true; mm.m_checkpointEra = votedEra; // the deploy gate cross-checks this against the winning era captured++; } else Print(mm.ID + ": WARNING - joint ensemble checkpoint capture FAILED at era " + IntegerToString((int)votedEra) + ". This member cannot contribute the weights the vote" " was measured with; the ensemble will not deploy a checkpoint it cannot reproduce."); //--- a new joint best retires the shared ladder for everyone mm.m_erasSinceBestBalanced = 0; mm.m_plateauStage = 0; mm.m_restartBoostErasLeft = 0; mm.m_consecutiveRegressions = 0; } //--- PARTIAL CAPTURE IS NOT A CHECKPOINT. Retire this era as the best rather than leaving the //--- record pointing at a quartet that cannot be reproduced - the era-stamp test would refuse to //--- deploy it anyway, and leaving a high score in place would block every later era from winning, //--- freezing the search behind a checkpoint that does not exist. Rolling it back lets the run //--- carry on and simply find its best again. if(captured < members) { g_ensBestScore = -1.0; g_ensBestTradeable = false; g_ensBestTwoSided = false; g_ensBestCalls = 0; g_ensBestEra = -1; Print("AI ensemble: joint checkpoint INCOMPLETE at era " + IntegerToString((int)votedEra) + " (" + IntegerToString(captured) + " of " + IntegerToString(members) + " members captured)" " - discarding this era as the best; the search continues from no joint checkpoint."); } } //+------------------------------------------------------------------+ //| Once per era, on the LAST still-training member to finish its | //| pass-3 scan: score the combined vote, rank the era, checkpoint, | //| advance the shared plateau ladder, and decide deployment. | //| | //| Every statistic mirrors the per-member gate exactly (coverage | //| floor, chance reference, EDGE_MIN_SIGMAS margin, Sidak | //| correction); only the population differs - the bars the VOTE | //| fired on rather than the bars one member called. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleEraVerdict(const int needMask, const long votedEra, double &etaLocal) { int members = EnsembleBitCount(needMask); //--- One trainer left (the others deployed, paused or stopped) is not an ensemble read: the //--- "vote" would be that member's own signal and the gate would silently become the solo gate //--- under an ensemble label. Members keep training; nothing is ranked or deployed from here. if(members < 2) return; //--- SHARED BARS ONLY. A bar one member skipped (feature-window failure) has an average over a //--- different membership, which is a different quantity - averaging it in would make the score //--- depend on which member happened to fail where. int shared = 0, fired = 0, wins = 0, firedLong = 0, firedShort = 0; int dirLabelBars = 0, alwaysLongWins = 0, alwaysShortWins = 0; for(int r = 0; r < g_ensVoteRows; r++) { if((g_ensVoteMask[r] & needMask) != needMask) continue; shared++; if(g_ensVoteDirLabel[r]) dirLabelBars++; //--- zero-skill reference, measured over EVERY shared bar (see chancePrecPct's derivation in //--- the era-end block): what always-long and always-short would have collected here if(g_ensVoteWinLong[r]) alwaysLongWins++; if(g_ensVoteWinShort[r]) alwaysShortWins++; //--- THE LIVE AGGREGATION, reproduced exactly (CExpertSignalCustom::Direction(), pass 2 plus the //--- `result /= number` normalization): sum the members' signed votes, divide by how many of them //--- ACTUALLY VOTED, and compare the magnitude against Min_Vote_Open on the same 0..100 scale the //--- tier weights already live on. No x100 any more - the contributions are pattern weights now, //--- not confidences (see the oEnsembleVote comment in the pass-3 scan). //--- //--- Dividing by `members` was the old behaviour and it is NOT what live does: an abstaining //--- member was pulling the average toward zero here while live simply left it out, so the gate //--- fired on a strictly smaller, more agreement-heavy set of bars than the EA trades. //--- //--- CONSENSUS (2026-08-19): the divisor is the weight of every member that EVALUATED the bar, //--- Neutral included, so agreement is what the magnitude measures - full agreement reads the //--- members' weighted mean win rate (the ceiling), one-of-four reads a quarter of it, splits //--- net out. Union semantics (voters-only divisor) made the magnitude near-constant at the //--- pooled win rate once tiers self-ranked, and the threshold a step function around it. int voters = EnsembleBitCount(g_ensVoteVoterMask[r] & needMask); if(voters <= 0 || g_ensVoteWeightSum[r] <= 0.0) continue; // every member abstained: no vote, no trade, not a fired bar double net = g_ensVoteSum[r] / g_ensVoteWeightSum[r]; if(MathAbs(net) < g_ensembleVoteThreshold) continue; fired++; if(net > 0.0) { firedLong++; if(g_ensVoteWinLong[r]) wins++; } else { firedShort++; if(g_ensVoteWinShort[r]) wins++; } } double slBe = 0.0, tpBe = 0.0; BarrierMultiples(slBe, tpBe); int bePct = (slBe > 0.0 && tpBe > 0.0) ? (int)MathRound(100.0 * slBe / (slBe + tpBe)) : -1; bool measurable = (shared > 0 && dirLabelBars > 0); double votePrecPct = (fired > 0) ? 100.0 * wins / fired : -1.0; double coveragePct = measurable ? 100.0 * fired / shared : -1.0; double baseRatePct = measurable ? 100.0 * dirLabelBars / shared : -1.0; double minCoverPct = measurable ? baseRatePct * MIN_COVERAGE_FRACTION_OF_BASE_RATE : -1.0; double chancePct = measurable ? 100.0 * MathMax(alwaysLongWins, alwaysShortWins) / shared : -1.0; double chanceP = (chancePct >= 0.0) ? chancePct / 100.0 : 0.0; //--- EFFECTIVE sample, not the raw fire count - the vote's outcomes are overlapping triple-barrier //--- labels exactly as the member gate's are. See EffectiveSampleSize(); the two gates have to apply //--- the identical correction or the ensemble becomes the easier one to clear. double precSE = (fired > 0 && chanceP > 0.0 && chanceP < 1.0) ? 100.0 * MathSqrt(chanceP * (1.0 - chanceP) / EffectiveSampleSize((double)fired)) : 0.0; double edgeFloorPct = chancePct + EDGE_MIN_SIGMAS * precSE; //--- Anti-degenerate pair, same intent as the member gate's coverage floor + bothSidesLive: a vote //--- that fires on almost nothing, or only ever one way, is not a tradeable ensemble however high //--- its win rate reads. (One-sidedness here is the vote's, not a class-recall measure - a vote //--- that never goes short IS the always-long model the chance reference already prices in.) bool twoSided = (firedLong > 0 && firedShort > 0); bool tradeableOK = measurable && votePrecPct >= 0.0 && twoSided && coveragePct >= minCoverPct && votePrecPct > edgeFloorPct; double coverCredit = 1.0; if(minCoverPct > 0.0 && coveragePct >= 0.0) coverCredit = MathMin(1.0, coveragePct / minCoverPct); double score = (votePrecPct >= 0.0) ? votePrecPct * coverCredit : 0.0; //--- N for the family-wise correction: every era that COULD have won, mirroring the member gate's //--- exclusion of eras with nothing to trade. bool degenerate = (fired <= 0); if(measurable && !degenerate) g_ensCandidateEras++; //--- Same lexicographic ordering as isBetterEra: deployable outranks two-sided outranks score. bool isBetter = (tradeableOK && !g_ensBestTradeable) || (tradeableOK == g_ensBestTradeable && twoSided && !g_ensBestTwoSided) || (tradeableOK == g_ensBestTradeable && twoSided == g_ensBestTwoSided && !degenerate && score > g_ensBestScore); if(isBetter) { g_ensBestScore = score; g_ensBestTradeable = tradeableOK; g_ensBestTwoSided = twoSided; g_ensBestPrecPct = votePrecPct; g_ensBestChancePct = chancePct; g_ensBestCalls = fired; g_ensBestEra = votedEra; g_ensErasSinceBest = 0; g_ensPlateauStage = 0; EnsembleCommitJointCheckpoint(votedEra); } else g_ensErasSinceBest++; //--- SHARED PLATEAU LADDER. One counter, one stage, applied to every member at the same era, so //--- the four nets escalate and finish together instead of drifting into different stages of //--- different searches. Mechanics per member are unchanged (boosted warm restart + optimizer //--- reset); only the trigger is collective. //--- MEMBERS PUBLISH, THE ORCHESTRATOR COMBINES - the same rule the vote and the live confidence //--- follow. Each member latches m_isErrorPlateaued when its OWN training error stops improving; this //--- is the only place allowed to turn that into a collective decision, and it needs UNANIMITY: one //--- member still learning can still move the combined vote, and the vote is what the gate certifies. //--- Without this the IS stop was inert for every ensemble member. It wrote m_plateauStage, which the //--- mirror at the end of this function overwrites every era - so on 2026-08-18 SP500 ConvLSTM //--- announced the plateau 1,299 times and trained to era 1,398 anyway, adding every one of those eras //--- to the family the deploy gate must correct over. The stop exists to SHRINK that family. int plateauedMembers = 0, learningMembers = 0; for(int pi = 0; pi < ArraySize(g_warriorEnsemble); pi++) { CExpertSignalAIBase *pm = g_warriorEnsemble[pi]; if(CheckPointer(pm) == POINTER_INVALID || pm.m_ensembleIndex < 0) continue; //--- Same participation test the barrier uses: a member that has finished or been stopped is not //--- something the rest should wait on, and must not veto the collective stop either. if(pm.m_trainingComplete || pm.m_trainingStopRequested || !pm.m_isInitialized || pm.m_barrierExcluded) continue; if(pm.m_isErrorPlateaued) plateauedMembers++; else learningMembers++; } bool allIsPlateaued = (plateauedMembers > 0 && learningMembers == 0); if(allIsPlateaued && !g_ensIsPlateauAnnounced) { g_ensIsPlateauAnnounced = true; PrintFormat("AI ensemble: EVERY member's IN-SAMPLE error has plateaued (%d participating members)." " No member is still learning from the data it can see, so more eras cannot find a" " better vote - they would only enlarge the family the deploy gate corrects over." " Ending the search on the joint checkpoint at the next era that does not improve it." " This stop never read an out-of-sample number, which is what makes the smaller family" " legitimate rather than a peek.", plateauedMembers); } string ladderNote = ""; if(!isBetter) { int dueStage = g_ensErasSinceBest / TrainPlateauPatienceEras(); //--- FED IN AS A DUE STAGE rather than written straight to g_ensPlateauStage, and the difference is //--- the whole fix: the block that actually ends the run sits under `dueStage > g_ensPlateauStage`, //--- so assigning the stage directly makes that test FALSE and the deploy never happens. Raising //--- dueStage lets the existing ladder carry it through its own path - warm restarts skipped, the //--- family-wise vote test, the measurement screen, the joint checkpoint - unchanged. //--- Left inside `if(!isBetter)` deliberately: an era that just produced a better vote produced a //--- better checkpoint, and ending on the next non-improving era costs one era and keeps it. if(allIsPlateaued) dueStage = PLATEAU_STAGE_DEPLOY; if(dueStage > g_ensPlateauStage) { g_ensPlateauStage = dueStage; if(g_ensPlateauStage == PLATEAU_STAGE_RESTART || g_ensPlateauStage == PLATEAU_STAGE_ANNEAL) { for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || !mm.m_isInitialized) continue; mm.m_modelEta = mm.m_etaCeiling * PLATEAU_RESTART_BOOST; mm.m_restartBoostErasLeft = TrainPlateauPatienceEras(); mm.m_plateauStage = g_ensPlateauStage; if(CheckPointer(mm.Net) != POINTER_INVALID) mm.Net.ResetOptimizerState(); //--- THIS member is the one still inside Train(), holding eta in a local that would //--- overwrite m_modelEta on the way out - so its restart has to reach the local too. if(mm == GetPointer(this)) etaLocal = mm.m_modelEta; } ladderNote = StringFormat(" | PLATEAU stage %d: %d eras with no better vote - boosted warm" " restart on all %d models (learning rate x%.1f, optimizer momentum" " reset). The joint checkpoint is safe.", g_ensPlateauStage, g_ensErasSinceBest, members, PLATEAU_RESTART_BOOST); } else if(g_ensPlateauStage >= PLATEAU_STAGE_DEPLOY) { //--- EXHAUSTED. Both escapes tried, nothing better found: this is the best vote this //--- ensemble reaches. Now the gate that matters - has the best-of-N vote survived //--- having been chosen? (See DEPLOY_FAMILY_WISE_ALPHA: a per-era floor alone opens on //--- noise with probability 1-(1-a)^N, which is near-certain by era 100.) double zBest = 0.0, pFam = 1.0; int nTried = 0; bool survives = EnsembleSurvivesSelection(zBest, pFam, nTried); bool haveJoint = true; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; //--- Era-stamped, not just present: a snapshot from an EARLIER era would make the //--- deployed quartet one that was never measured together (see m_checkpointEra). if(!mm.m_haveOosCheckpoint || mm.m_checkpointEra != g_ensBestEra) haveJoint = false; } string testNote = StringFormat(" best-of-%d test on the VOTE: edge %.1fpp (%.1f%% vs chance" " %.1f%%) on %d fired bars = %.2f sigma, family-wise p=%.4f" " (need <=%.2f)", nTried, g_ensBestPrecPct - g_ensBestChancePct, g_ensBestPrecPct, g_ensBestChancePct, g_ensBestCalls, zBest, pFam, DEPLOY_FAMILY_WISE_ALPHA); //--- THE MEASUREMENT SCREEN, applied to the ensemble exactly as to a solo model. The MI //--- suite runs ONCE per chart and its outcome is shared (see the tune/MI sharing), so //--- every member on this chart carries the same verdict - checking this member's flag is //--- checking the chart's. Four models finding nothing between them is not four chances at //--- an edge; it is four fits to the same absent information. if(g_ensBestTradeable && haveJoint && survives && !m_dirEvidence) Print("AI ensemble: DEPLOY REFUSED BY THE MEASUREMENT SCREEN - the combined vote cleared" " its statistical gate, but neither the feature/label mutual information nor the" " normalised excursion asymmetry cleared a permutation null on this chart's" " feature set. The vote is a best-of-N maximum over a search that had no measured" " directional information to find; clearing the gate on top of that is the" " family-wise trap this project has hit four times. Checkpoints kept, training" " untouched - this refuses to go LIVE, nothing else."); if(g_ensBestTradeable && haveJoint && survives && m_dirEvidence) { g_ensDeployApproved = true; Print("AI ensemble: PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - no better vote for " + IntegerToString(g_ensErasSinceBest) + " eras across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " warm restarts." + testNote + " - CLEARS. Deploying the JOINT checkpoint from era " + IntegerToString((int)g_ensBestEra) + ": every model reverts to the weights it held" " at the era whose combined vote scored best, so the ensemble that trades is" " exactly the one that was measured."); ladderNote = " | ENSEMBLE DEPLOY APPROVED"; } else { //--- Restart the ladder and keep training, exactly as the solo gate does on a //--- failed selection test. The era cap stays the backstop. Print("AI ensemble: PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + (!g_ensBestTradeable ? "no era's combined vote ever cleared the deployability floor (fires on at least a" " quarter as many bars as actually swing, both directions alive, at a win rate" " above the always-one-way reference by 2 sigma), so there is nothing safe to deploy." : (!haveJoint ? "the joint checkpoint is incomplete - at least one model has no snapshot of the" " winning era, so the measured ensemble cannot be reproduced." : "the best combined vote clears the per-era floor but DOES NOT clear the null of" " the MAXIMUM over the eras it was chosen from." + testNote + " A best-of-N this large happens routinely when every era is a noise draw.")) + " Restarting the ladder and continuing to train; the era cap remains the backstop."); g_ensErasSinceBest = 0; g_ensPlateauStage = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) != POINTER_INVALID && mm.m_ensembleIndex >= 0) mm.m_plateauStage = 0; } ladderNote = " | ladder restarted (gate not cleared)"; } } } } //--- Mirror the shared ladder onto every member. Their private counters no longer advance (the //--- ensemble's do), so without this each model's own era line would report "0 eras since best, //--- stage 0" forever while the ensemble was three stages into its search - a status display that //--- contradicts the mechanism actually running. for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; mm.m_erasSinceBestBalanced = g_ensErasSinceBest; mm.m_plateauStage = g_ensPlateauStage; } //--- LIFETIME ACCUMULATION - same cadence as a solo model's m_cumOosTotal (see its increment sites): //--- every scored era adds the bars the vote fired on and how many paid, monotonically, never reset //--- per era. `wins`/`fired` above are this era's OOS rows only; the panel reads the running total. g_ensCumOosTotal += fired; g_ensCumOosCorrect += wins; //--- PANEL + JOURNAL. The panel line uses the SAME label and the SAME measurement as a solo model's //--- ComputeCompoundedAccuracyLine (Expert\AIBase\ChartUI.mqh) - a persisted win-rate over every //--- called bar, not this era's fired-bar percentage alone - so the two panels read consistently //--- (user request 2026-08-16: same label, same measurement for both). string ensAccLine; if(g_ensCumOosTotal > 0) { int winPctLifetime = (int)MathRound(g_ensCumOosCorrect * 100.0 / g_ensCumOosTotal); //--- THIS ERA alongside the lifetime figure - same reason and same fix as the solo panel's //--- ComputeCompoundedAccuracyLine (see its "THIS ERA" comment): the lifetime average is diluted //--- by every fired bar from every prior era, so a real swing this era barely moves it. `wins`/ //--- `fired` above are this era's combined-vote rows only. string thisEra = (fired > 0) ? StringFormat(", this era %d%%", (int)MathRound(votePrecPct)) : ""; ensAccLine = StringFormat("Buy/Sell calls correct: %d%% (unseen data%s%s)", winPctLifetime, (bePct >= 0 ? StringFormat(", need %d%%", bePct) : ""), thisEra); } else ensAccLine = (g_ensCandidateEras > 0) ? "Buy/Sell calls correct: no directional calls yet" : "Buy/Sell calls correct: measuring..."; g_ensembleVoteLine = StringFormat("%s (era %d, %d models%s)", ensAccLine, (int)votedEra, members, (g_ensDeployApproved ? ", DEPLOYING" : (tradeableOK ? ", deployable" : ""))); Print(StringFormat("AI ensemble: combined-vote era %d - %d models, %d shared OOS bars, %d fired at" " vote>=%.0f%% (%.1f%% coverage, floor %.1f%%), win %s vs chance %.1f%% (needs" " >%.1f%% at %d sigma)%s -> score %.1f%s%s. The vote that actually trades: each" " member's DB-ranked tier weight x module weight, averaged over the members that" " VOTED (abstentions excluded, as live), graded on target-before-stop.", (int)votedEra, members, shared, fired, g_ensembleVoteThreshold, coveragePct, minCoverPct, (fired > 0 ? StringFormat("%.1f%%", votePrecPct) : "n/a"), chancePct, edgeFloorPct, (int)EDGE_MIN_SIGMAS, (tradeableOK ? " DEPLOYABLE" : ""), score, (isBetter ? StringFormat(" <-- NEW BEST, joint checkpoint captured (era %d)", (int)votedEra) : StringFormat(" (best %.1f at era %d, %d eras ago)", g_ensBestScore, (int)g_ensBestEra, g_ensErasSinceBest)), ladderNote)); } //+------------------------------------------------------------------+ //| Per-member era-end hook: mark this member done for the era and, | //| when it is the last one, run the verdict above. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnsembleOosPassComplete(const long votedEra, double &etaLocal) { if(!m_ensembleMember || m_ensembleIndex < 0) return; //--- The rows were stamped during pass 3, BEFORE this member incremented its era counter, so the //--- buffer's era is the era that just finished. A mismatch means this member contributed nothing //--- to the current buffer (no OOS bars scored this era) - it cannot be counted as having read the //--- vote, or the verdict would be taken on a subset that silently excludes it. if(g_ensVoteEra != votedEra) return; g_ensVoteDoneMask |= (1 << m_ensembleIndex); int need = 0; for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) == POINTER_INVALID || mm.m_ensembleIndex < 0) continue; if(mm.m_trainingComplete || mm.m_trainingStopRequested || mm.m_trainingPaused || !mm.m_isInitialized) continue; need |= (1 << mm.m_ensembleIndex); } if(need == 0 || (g_ensVoteDoneMask & need) != need) return; //--- Idempotence: one verdict per era, whatever order the members arrive in. if(g_ensLastVerdictEra == votedEra) return; g_ensLastVerdictEra = votedEra; EnsembleEraVerdict(need, votedEra, etaLocal); } //+------------------------------------------------------------------+ //| Log the selection-gate verdict for a deploy the gate does NOT | //| block - the era-cap path and the panel's Deploy button, both of | //| which are explicit operator decisions and stay that way. The point | //| is that "I chose to ship this" and "this cleared the bar" should | //| never be confusable in the log afterwards. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportSelectionGateVerdict(string context) { double z = 0.0, pFam = 1.0; int nTried = 0; //--- ENSEMBLE: report the gate that actually governs this model. Quoting the member's own //--- best-of-N here would answer a question nobody asked - the member never deploys alone, and a //--- member-level "CLEARS" next to a vote that did not is precisely the confusion this function //--- exists to prevent. if(m_ensembleMember) { bool okEns = EnsembleSurvivesSelection(z, pFam, nTried); if(g_ensBestCalls <= 0) { Print(ID + ": " + context + " - the ENSEMBLE selection gate cannot be evaluated (no era's" " combined vote has been ranked yet). Treat this ensemble as unvalidated."); return; } Print(ID + ": " + context + " - ENSEMBLE best-of-" + IntegerToString(nTried) + " test on the" " combined VOTE: edge " + DoubleToString(g_ensBestPrecPct - g_ensBestChancePct, 1) + "pp (" + DoubleToString(g_ensBestPrecPct, 1) + "% vs chance " + DoubleToString(g_ensBestChancePct, 1) + "%) on " + IntegerToString(g_ensBestCalls) + " fired bars = " + DoubleToString(z, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ") - " + (okEns ? "CLEARS." : "DOES NOT CLEAR. A maximum this size arises routinely when every era is a noise" " draw, so this ensemble is being deployed on operator authority, NOT on measured" " evidence of an edge.")); return; } bool ok = BestCheckpointSurvivesSelection(z, pFam, nTried); if(m_bestDirCalls <= 0) { Print(ID + ": " + context + " - selection gate cannot be evaluated (no ranked checkpoint with" " directional calls). Treat this model as unvalidated."); return; } Print(ID + ": " + context + " - best-of-" + IntegerToString(nTried) + " selection test: edge " + DoubleToString(m_bestDirPrecPct - m_bestChancePrecPct, 1) + "pp (" + DoubleToString(m_bestDirPrecPct, 1) + "% vs chance " + DoubleToString(m_bestChancePrecPct, 1) + "%) on " + IntegerToString(m_bestDirCalls) + " directional calls = " + DoubleToString(z, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ") - " + (ok ? "CLEARS." : "DOES NOT CLEAR. A maximum this size arises routinely when every era is a noise draw, so" " this model is being deployed on operator authority, NOT on measured evidence of an edge.")); } //+------------------------------------------------------------------+ //| Training and Signal Methods | //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| Where the TRAINING window starts: ALL available history, floored | //| by MinTrainYear. The StudyPeriods input this replaced could only | //| ever throw data away: the signal is weak and the directional | //| classes are rare, so every extra year is more of the minority | //| class, and the honest generalization read comes from the OOS | //| holdout rather than from withholding history. MinTrainYear | //| survives because it answers a different question - excluding a | //| broker's dubious pre-history - not "how much". | //| Ordering: SERIES_FIRSTDATE is the floor of what EXISTS, | //| MinTrainYear the floor of what is TRUSTED; the window starts at | //| whichever is later. | //| Shared by Train()'s era start and StartLabelCachePrebuild(), so | //| the pre-scan and the era loop can never disagree about what "the | //| window" means - the saved dtStudied watermark is NOT an input | //| here, which is the point (see the call site in Train()). | //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| Names the Train() branch being taken while no era has completed | //| for a long time. Silent on a healthy run (an era ends, the clock | //| resets); at most one line per 60s per signal once stalled. | //| See m_lastEraCompleteTick for the incident that forced this. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ReportTrainStall(const string branch) { const uint STALL_AFTER_MS = 180000; // 3 min: ~2x the slowest healthy era seen on this config const uint STALL_REPORT_INTERVAL = 60000; uint nowTick = GetTickCount(); //--- First call ever: adopt now as the baseline rather than reporting instantly against tick 0. if(m_lastEraCompleteTick == 0) { m_lastEraCompleteTick = nowTick; return; } uint since = nowTick - m_lastEraCompleteTick; if(since < STALL_AFTER_MS) return; if(m_lastStallReportTick != 0 && nowTick - m_lastStallReportTick < STALL_REPORT_INTERVAL) return; m_lastStallReportTick = nowTick; PrintFormat("%s: TRAIN STALL - no era has completed for %.0fs and Train() is taking the '%s' branch" " | era %d | runActive=%s prebuildActive=%s cachePrebuilt=%s simOos=%s eraResume=%s" " paused=%s stopReq=%s | labelCacheBars=%d anchor=%s dtStudied=%s", ID, since / 1000.0, branch, (int)m_eraCount, m_trainRunActive ? "Y" : "N", m_labelPrebuildActive ? "Y" : "N", m_labelCachePrebuilt ? "Y" : "N", m_simOosRunActive ? "Y" : "N", m_eraResumePending ? "Y" : "N", m_trainingPaused ? "Y" : "N", m_trainingStopRequested ? "Y" : "N", m_labelCacheBars, TimeToString(m_labelCacheAnchorTime), TimeToString(dtStudied)); } //+------------------------------------------------------------------+ //| Speaks ONLY when an era is genuinely slow: nothing for the first | //| 60 seconds of an era, at most 6 lines after that, one per 4096 | //| processed items. Reports where the time actually went, split into | //| the two candidate costs and the remainder, because "the era is | //| slow" without the split is exactly the undiagnosable state the | //| 2026-08-10 restart produced (see the member declarations). | //+------------------------------------------------------------------+ void CExpertSignalAIBase::TrainHeartbeat(const string tag, int done, int total, const string shortLabel) { //--- Panel progress is published on EVERY call, before the 4096-item gate below: the gate exists to //--- keep the JOURNAL quiet, and applying it to the panel too would leave the display frozen between //--- boundaries. Two assignments, no formatting - cheap enough for a per-item path. m_passLabel = shortLabel; m_passProgressPct = (total > 0) ? (int)MathMin(100.0, 100.0 * done / total) : 0; //--- TIME-gated, not item-gated. The first version fired only on 4096-item boundaries once the era //--- had already run 60s - and those boundaries are all crossed in the first few chunks, so a run //--- that got slow AFTER them printed nothing at all. That is exactly what happened on 2026-08-10: //--- 20 minutes, four pegged cores, zero heartbeats, and the silence was read as "the era loop is //--- never reached" when it may simply have been past its last boundary. A diagnostic whose //--- trigger can be outrun by the condition it watches for is worse than none - it produces //--- confident wrong conclusions. The 255-item mask only keeps GetTickCount() off the hot path. if((done & 255) != 0) return; uint nowTick = GetTickCount(); uint elapsedMs = nowTick - m_eraStartTick; if(elapsedMs < 60000 || m_passHeartbeatPrints >= 12) return; if(m_lastHeartbeatTick != 0 && nowTick - m_lastHeartbeatTick < 30000) return; m_lastHeartbeatTick = nowTick; m_passHeartbeatPrints++; double featS = (double)m_passFeatUs / 1000000.0; double netS = (double)m_passNetUs / 1000000.0; PrintFormat("%s: SLOW ERA heartbeat - %s %d of %d after %.0fs | feature windows %.1fs | net fwd/back %.1fs | everything else %.1fs", ID, tag, done, total, elapsedMs / 1000.0, featS, netS, MathMax(elapsedMs / 1000.0 - featS - netS, 0.0)); } //+------------------------------------------------------------------+ datetime CExpertSignalAIBase::TrainWindowStart(datetime startTrainBar) { datetime firstAvailableBar = (datetime)SeriesInfoInteger(m_symbol.Name(), PERIOD_CURRENT, SERIES_FIRSTDATE); MqlDateTime floor_time; TimeCurrent(floor_time); floor_time.year = m_minTrainYear; floor_time.mon = 1; floor_time.day = 1; floor_time.hour = 0; floor_time.min = 0; floor_time.sec = 0; datetime st_time = StructToTime(floor_time); if(firstAvailableBar > st_time) st_time = firstAvailableBar; return MathMax(startTrainBar, st_time); } //+------------------------------------------------------------------+ void CExpertSignalAIBase::Train(datetime StartTrainBar = 0) { //--- One-shot latch so a failing forward pass reports itself ONCE per call instead of once per //--- sample. CNet::feedForward's return value used to be discarded at all three call sites below, //--- which is how the 2026-08-02 run spent a whole era backpropagating against a batch-norm layer //--- whose device-side output had frozen: the only trace was 13,776 identical BufferWrite lines //--- from three frames deeper, and nothing said training was still running on top of them. bool forwardFailureReported = false; const int STABILITY_WINDOW = 3; // consecutive eras the OOS accuracy must hold steady for const double STABILITY_TOLERANCE = 2.0; // max spread (percentage points) across that window // Max wall-clock work per call before yielding - see m_trainRunActive's declaration comment for // why chunking exists at all. TuneIndicatorsAndTrain()/Train() only run ONCE per dispatched // "New Bar" custom chart event (see OnChartEventHandler - this instance's m_studyEventId calls it // exactly once, then // clears bEventStudy so ScheduleTrainingIfNeeded() can arm the next one), so the real throughput // ceiling in practice is however fast MT5 itself pumps/dispatches that custom event - NOT this // constant. Raising the OnTimer interval (5s->250ms) had ~zero effect for exactly that reason: // ticks/chart events were already redispatching far more often than the timer alone would. Since // per-event dispatch overhead is roughly fixed, doing more compute per event (fewer, larger // chunks) cuts wall-clock training time roughly in proportion, but MT5 has only this one thread - // the panel/chart can only respond to input in the gap between chunks, so 500ms made it feel // unresponsive unless clicks landed in that narrow window. Lowered back to 120ms to keep the UI // reactive. Raised to 200ms 2026-07-26 (throughput became the bigger complaint, as flagged above) - // a deliberate middle ground between the reactive-but-slow 120ms and the previously-rejected 500ms, // not a return to that. Watch panel drag/click feel after this change; back off toward 120ms if it // regresses, or raise further only in small steps if it doesn't. // 2026-07-30: it regressed, exactly as that warning anticipated - the panel drags stickily and // buttons miss clicks under load, because 200ms is the worst-case latency between a click landing // and this thread being free to notice it. Backing off to the documented 120ms. The throughput this // costs is a far smaller sacrifice than it was when the note above was written: the derived topology // cut the network from ~292k weights to ~29k (see ComputeFirstLayerWidth), so an era is a fraction // of the work it used to be and the fixed per-dispatch overhead the note worried about is now a // correspondingly smaller share of it. Responsiveness is worth more than the remainder. //--- ENSEMBLE: four members share the one chart thread and their chunks queue back-to-back, so the //--- worst-case latency between a panel click and a free thread is members x budget - 4 x 120ms = //--- 480ms, which is exactly the "drags stickily, buttons miss clicks" regime the 200ms note above //--- documents (user-reported on the first ensemble runs, 2026-08-15). Divide the budget instead: //--- an ensemble chart's UI latency returns to the solo chart's (~4 x 30ms) at the cost of a little //--- more per-chunk dispatch overhead, which the derived ~29k-weight topology can afford. //--- 2026-08-16: the divided budget alone did NOT restore responsiveness, because all members shared //--- one custom-event id and CExpertCustom broadcasts events to every filter - each posted event ran //--- a chunk in ALL N members, N*N chunks per round, and the thread never idled (panel completely //--- dead, not just sticky). Fixed with per-instance study-event ids (see STUDY_EVENT_ID_BASE); the //--- divided budget below is what makes the FIXED dispatch behave as the note above intends. //--- Divided by the ACTIVE trainer count, not the member count: a member waiting at the era barrier //--- (or deployed/paused) consumes no chunks, so its share is donated to the members still working - //--- one laggard left gets the full 120ms - while the chart thread's UI headroom stays constant. const uint TRAIN_TIME_BUDGET_MS = m_ensembleMember ? (uint)(120 / MathMax(EnsembleActiveTrainers(), 1)) : 120; //--- //--- Never block the calling thread while paused/stopped - just decline this call (or finalize a //--- run that just got stopped) and let the next scheduled call check again, so Pause/Resume/Stop //--- and everything else on the control panel stays responsive instead of Sleep()-ing the one //--- MQL5 thread this chart has. if(m_trainingPaused && !IsStopped() && !m_trainingStopRequested) return; bool stop = IsStopped() || m_trainingStopRequested; if(stop) { if(m_trainRunActive) FinalizeTrainRun(); if(m_simOosRunActive) { delete m_simOosNet; m_simOosNet = NULL; m_simOosRunActive = false; } return; } //--- ENSEMBLE DEPLOY, approved by the ensemble gate on some member's era end (see //--- EnsembleEraVerdict). Acted on HERE, at the next call, rather than at the next era end: the //--- decision is already made, and training one more era would only produce weights that //--- FinalizeTrainRun immediately discards in favour of the joint checkpoint. Each member restores //--- its own half of that checkpoint, so the quartet that goes live is the one the vote was //--- measured on. A member without a snapshot keeps training rather than deploying weights nothing //--- measured - the verdict refuses to approve in that case, so this is belt and braces. if(m_ensembleMember && g_ensDeployApproved && !m_trainingComplete && m_haveOosCheckpoint && m_checkpointEra == g_ensBestEra) { m_trainingComplete = true; Print(ID + ": ENSEMBLE DEPLOY - restoring this model's weights from the joint checkpoint at era " + IntegerToString((int)g_ensBestEra) + " and switching to live inference. The combined vote," " not this model alone, is what cleared the gate."); if(m_trainRunActive) FinalizeTrainRun(); //--- Same one-shot pattern-database backfill the solo path arms at its era end, and for the same //--- reason (see StartPatternDatabaseBackfill): a deployed model has to be RANKED the instant it //--- goes live, not an hour of real trades later. It has to be armed HERE as well because an //--- ensemble member never reaches that era-end block - deploy is acted on at Train() ENTRY and //--- returns immediately (that is the whole point: no wasted era), so line ~3489 is unreachable //--- for all four members and the feature was a no-op in exactly the mode it ships in. //--- m_resumeBars/m_resumeOosCutoff are the last era's own window bounds (see where the era loop //--- stamps them) - the same pair the solo call passes, just read from the members that survive //--- across chunked calls rather than from the era loop's locals, which are out of scope here. StartPatternDatabaseBackfill(m_resumeBars, m_resumeTotalIter, m_resumeOosCutoff); return; } //--- ENSEMBLE ERA BARRIER (user request 2026-08-16): members advance era by era TOGETHER, because //--- the number that matters - the combined-vote OOS score - is only well-defined when every //--- member's pass 3 describes the same era, and because live trading is the members voting //--- together, not four models drifting apart in training age. A member ahead of the slowest //--- still-training member declines the call here; it costs nothing (its chunk budget flows to the //--- laggards via EnsembleActiveTrainers above) and resumes untouched when the barrier clears. //--- Deployed, stopped and paused members are exempt from the min (see EnsembleMinTrainingEra), so //--- nothing deadlocks. Placed AFTER the pause/stop handling: a Stop must still finalize, and a //--- barred member must still respond to the panel. //--- Stamp this member's era-advance clock BEFORE the barrier test, so a member that cannot finish an //--- era eventually stops pinning the whole chart (see BarrierEraHeartbeat / ENSEMBLE_BARRIER_STUCK_MS). BarrierEraHeartbeat(); if(EnsembleEraBarrierHolds()) { //--- deliberate idleness, not a stall - keep the stall watchdog's era clock current and say //--- what is happening on the member's panel line instead of freezing its last progress text m_lastEraCompleteTick = GetTickCount(); //--- AND SAY SO IN THE JOURNAL. Resetting the watchdog above is right (a held member is idle, not //--- stalled) but it was the ONLY thing this branch did, so a held member left no record anywhere. //--- On 2026-08-17 that meant two frozen charts reported through their two BROKEN members and //--- stayed completely silent about the two healthy ones - the panel said "Waiting at era N" and //--- the journal, which is what gets read afterwards, said nothing at all. Rate-limited, and it //--- names the member being waited on so the blocker is identified from one line rather than by //--- cross-referencing every member's last era. uint nowTick = GetTickCount(); if(m_barrierHoldReportTick == 0 || nowTick - m_barrierHoldReportTick >= ENSEMBLE_BARRIER_REPORT_MS) { m_barrierHoldReportTick = nowTick; long minEra = EnsembleMinTrainingEra(); string blockers = ""; for(int bi = 0; bi < ArraySize(g_warriorEnsemble); bi++) { CExpertSignalAIBase *bm = g_warriorEnsemble[bi]; if(CheckPointer(bm) == POINTER_INVALID) continue; if(bm.m_trainingComplete || bm.m_trainingStopRequested || bm.m_trainingPaused || !bm.m_isInitialized || bm.m_barrierExcluded) continue; if(bm.m_eraCount <= minEra) blockers += (blockers == "" ? "" : ", ") + bm.ID; } PrintFormat("%s: HELD AT THE ERA BARRIER - this member is at era %d and the ensemble minimum is" " %d, so it is idle until [%s] catch up. It is NOT stalled and its weights are" " untouched. If this line keeps repeating, the member(s) named are the ones to" " diagnose - after %d minutes without an era they are dropped from the barrier and" " this member resumes on its own.", ID, (int)m_eraCount, (int)minEra, blockers == "" ? "(none - resolving)" : blockers, (int)(ENSEMBLE_BARRIER_STUCK_MS / 60000)); } PublishStatus(StringFormat("Waiting at era %d for slower ensemble members (min era %d) - donating its compute until they catch up", (int)m_eraCount, (int)EnsembleMinTrainingEra())); return; } m_barrierHoldReportTick = 0; //--- Evaluation-only continual-learning OOS simulation walk in progress (see StartOosContinualSimulation): //--- give it exclusive occupancy of this call, same chunked budget as the real era loop below, so a //--- large OOS window can't freeze the UI in one shot. While it's active no real-training //--- ResizeBuffers()/RefreshData() runs, so the price/ATR/time buffers it reads stay frozen for its //--- whole walk - it never has to worry about the label cache's shifting-index invalidation below. if(m_simOosRunActive) { ReportTrainStall("OOS continual-learning simulation walk"); AdvanceOosSimulationChunk(); return; } //--- One-shot pattern-database backfill in progress (see StartPatternDatabaseBackfill) - same //--- exclusive-occupancy/chunking treatment as the simulation walk above. if(m_dbBackfillActive) { ReportTrainStall("pattern-database backfill walk"); AdvancePatternDatabaseBackfill(); return; } //--- Eager label-cache pre-build in progress (see StartLabelCachePrebuild/AdvanceLabelCachePrebuild) - //--- same exclusive-occupancy/chunking treatment as the OOS simulation walk above, so it can't freeze //--- the UI on a large study window either. m_trainRunActive stays false for its whole duration, so //--- once it completes, Train() falls through to the normal !m_trainRunActive setup below and era 0 //--- starts from the measured class distribution it just seeded. if(m_labelPrebuildActive) { ReportTrainStall("label-cache prebuild scan"); AdvanceLabelCachePrebuild(); return; } if(!m_trainRunActive) { //--- Wait (briefly, bounded, non-blocking across calls) for the terminal to finish syncing this //--- symbol/period's history from the broker before computing the training window. Bars(symbol, //--- period) - the hard cap on how many bars the era loop below will ever process - reflects //--- whatever's synced SO FAR, not necessarily the true total; starting before sync completes //--- would let that cap (and therefore the "Bar X of Y" progress display) silently grow between //--- eras as more history trickles in. if(!SeriesInfoInteger(m_symbol.Name(), PERIOD_CURRENT, SERIES_SYNCHRONIZED)) { uint syncNowTick = GetTickCount(); if(m_syncWaitStartTick == 0) m_syncWaitStartTick = syncNowTick; if(syncNowTick - m_syncWaitStartTick < 5000) { ReportTrainStall("waiting for history sync"); return; // retry on the next scheduled call instead of blocking here } Print(ID + ": WARNING - history for " + m_symbol.Name() + " " + EnumToString(PERIOD_CURRENT) + " did not finish syncing after 5s; training window may still grow as more history arrives"); } m_syncWaitStartTick = 0; //--- 3 no-op passes before the era loop ever runs for a fresh start (see m_warmupPassesRemaining's //--- declaration comment) - each is its own separately-scheduled Train() call (this whole method //--- just returns, deferring to the next "New Bar"/timer-driven call), giving MT5's history sync //--- several real, wall-clock-separated chances to settle on top of the 5s soft wait just above, //--- before training commits to a bar count and starts populating the label cache below. if(m_warmupPassesRemaining > 0) { ReportTrainStall("history-settle warm-up pass"); m_warmupPassesRemaining--; PrintVerbose(ID + ": warm-up pass " + IntegerToString(3 - m_warmupPassesRemaining) + " of 3 (letting history sync settle before training starts)"); return; } //--- ALL available history, floored by MinTrainYear - see TrainWindowStart(). Factored out //--- (2026-08-09) because StartLabelCachePrebuild needs the SAME rule: the resumed-model //--- pre-scan used to size its window from the SAVED dtStudied instead, and a model whose //--- watermark sat at the last studied bar got Bars(dtStudied, now) = 0 - a zero-bar "complete" //--- label cache, logged as "Buy: 0 | Sell: 0 | Neutral: 0", with everything downstream //--- (horizon, geometry, the MI report) computed on nothing. dtStudied = TrainWindowStart(StartTrainBar); //--- OOS-based objective + stability tracking: training only "converges" once the objective is //--- met AND OOS accuracy has held inside a tight band for the last few eras, so a single lucky //--- era can't get locked in as the final model. The best-scoring era's weights are checkpointed //--- to an agent-local scratch file (not FILE_COMMON) and restored at the end - this works inside //--- the tester too, unlike Net.Save()/Load() which are disabled there. m_oosWindow.Clear(); m_bestOosForecast = -1; m_bestBalancedOos = -1; m_bestPassedRecall = false; m_bestBothSidesLive = false; m_haveOosCheckpoint = false; m_checkpointEra = -1; // the joint-checkpoint era stamp goes with the snapshot it describes m_oosStable = false; m_objectiveMet = false; //--- Family-wise deployment gate state, reset with the checkpoint tracking it describes: N counts //--- the eras THIS run selects a maximum over, so carrying it across runs would test the winner //--- against a search that never happened. m_bestDirPrecPct = -1.0; m_bestChancePrecPct = -1.0; m_bestDirCalls = 0; m_deployCandidateEras = 0; m_erasSinceCooldown = 0; m_eraResumePending = false; //--- Plateau ladder starts fresh with this run, so it re-walks the escalation from its own //--- starting point. (The focal-gamma anneal that used to reset here went with focal loss on //--- 2026-07-31 - the ladder's real escape is the learning-rate warm restart.) m_erasSinceBestBalanced = 0; m_plateauStage = 0; m_restartBoostErasLeft = 0; //--- Per-RUN like the ladder above, and for the same reason: a resumed run restarts the search, so //--- carrying a previous run's best training error would let it early-stop on the first era. m_bestIsError = -1.0; m_erasSinceBestIsError = 0; m_isErrorPlateaued = false; //--- ENSEMBLE: the shared gate state is per-RUN for the same reason the per-member state above is - //--- N must count the eras THIS run's maximum was taken over, so carrying it across runs would //--- test the winner against a search that never happened. //--- Reset by the FIRST member to open the run, identified as "no sibling has a run active yet". //--- Members open their runs within milliseconds of each other but not simultaneously, and a //--- late starter must not wipe state the ensemble is already accumulating - this test is exact //--- either way: the first opener sees no active sibling, every later one does, and a member //--- reopening mid-flight (its own run finalized while the others train on) correctly declines. bool ensembleRunAlreadyOpen = false; if(m_ensembleMember) for(int i = 0; i < ArraySize(g_warriorEnsemble); i++) { CExpertSignalAIBase *mm = g_warriorEnsemble[i]; if(CheckPointer(mm) != POINTER_INVALID && mm != GetPointer(this) && mm.m_trainRunActive) { ensembleRunAlreadyOpen = true; break; } } if(m_ensembleMember && !ensembleRunAlreadyOpen) { g_ensLastVerdictEra = -1; g_ensBestScore = -1.0; g_ensBestTradeable = false; g_ensBestTwoSided = false; g_ensBestPrecPct = -1.0; g_ensBestChancePct = -1.0; g_ensBestCalls = 0; g_ensBestEra = -1; g_ensCandidateEras = 0; g_ensErasSinceBest = 0; g_ensPlateauStage = 0; g_ensIsPlateauAnnounced = false; g_ensDeployApproved = false; } //--- One-time eager pre-scan for a fresh start (see m_labelCachePrebuilt's declaration comment) - //--- kick it off and defer era 0 until it's done, so era 0 can start with a real class-balance //--- oversampling ratio instead of the reps=1 fallback. Routed via the m_labelPrebuildActive gate //--- above on every subsequent call until it completes. if(!m_labelCachePrebuilt) { ReportTrainStall("arming the first label-cache prebuild"); StartLabelCachePrebuild(); return; } m_trainRunActive = true; } int bars, totalIter, oosCutoff, i; bool add_loop; if(!m_eraResumePending) { //--- COLD-INDICATOR BACKOFF (2026-08-13). When the previous era was discarded because EVERY //--- window failed on a TRANSIENT cause (an async indicator still calculating - see //--- ADIndicatorCold/the cold-ATR guard), restarting the sweep immediately is worse than //--- useless: a full-history pass 1 hammers the CPU and memory the indicator threads need to //--- finish warming, which on a memory-starved box turns "cold for a second" into "cold //--- forever" (observed 2026-08-13: a resumed META model resweeping 54k bars back-to-back for //--- 6+ minutes, indicators never warming, panel oscillating 0->100%). Give them a few quiet //--- seconds instead; the stall reporter stays the loud diagnosis if it persists. if(m_coldSweepTick != 0) { if(GetTickCount() - m_coldSweepTick < 5000) { ReportTrainStall("cold-indicator backoff (all windows failed on a transient cause)"); return; } m_coldSweepTick = 0; } int barsNow = (int)MathMin(Bars(m_symbol.Name(), PERIOD_CURRENT, dtStudied, TimeCurrent()) + m_historyBars, Bars(m_symbol.Name(), PERIOD_CURRENT)); //--- PRIME, THEN SETTLE, THEN SWEEP. //--- //--- This ResizeBuffers/RefreshData pair is the PRIMER: the CopyBuffer it issues at full depth is //--- what asks the terminal to calculate that far, and asking is the only thing that starts it. //--- What must NOT follow immediately is the sweep - a 50k-bar feature scan starves the very //--- indicator threads the request just woke, which is how the 2026-08-17 failure sustained itself //--- for 40 minutes at a stretch (discard era -> re-sweep -> discard, the panel's 0->100% loop). if(!ResizeBuffers(barsNow) || !RefreshData()) { PrintFormat("%s: era start ABORTED - price/indicator buffers would not prepare for %d bars" " (priming ResizeBuffers/RefreshData failed); ending this training run, it re-arms" " on the next scheduled call", ID, barsNow); FinalizeTrainRun(); return; } //--- Now wait out the depth rather than snapshotting it. Returns 0 while the count is still moving. int settled = SettledBars(barsNow, "training sweep"); if(settled <= 0) { ReportTrainStall("priming indicator history (holding the sweep until the calculated depth" " stops changing)"); return; } //--- The floor is the one piece of policy that stays here: below TRAIN_MIN_CLAMPED_BARS a settled //--- depth is too thin to train anything worth measuring, so the run holds and the stall reporter //--- stays the loud diagnosis rather than producing a meaningless era. if(settled < barsNow) { if(settled < TRAIN_MIN_CLAMPED_BARS) { ReportTrainStall(StringFormat("indicator depth settled at %d bars, below the %d-bar floor" " for a trainable era", settled, TRAIN_MIN_CLAMPED_BARS)); return; } barsNow = settled; //--- Re-prepare at the clamped depth, and ONLY when it actually changed: the primer above //--- already left every buffer refreshed at the full depth, so an unconditional second pass //--- would be a wasted CopyBuffer over every buffer, every era, on the charts that need none. if(!ResizeBuffers(barsNow) || !RefreshData()) { //--- The ONLY exit from Train() that tears down the whole run, and it used to be completely //--- silent - a transient buffer/history hiccup ended the run, FinalizeTrainRun() pushed //--- dtStudied to the last scanned bar, and the next era simply never started. Indistinguishable //--- from a hang while it was quiet, so it says so (2026-08-10). PrintFormat("%s: era start ABORTED - price/indicator buffers would not prepare for the" " settled depth of %d bars (ResizeBuffers/RefreshData failed); ending this" " training run, it re-arms on the next scheduled call", ID, barsNow); FinalizeTrainRun(); return; } } bars = barsNow; //--- Cross-asset panel is indexed against exactly this bar grid, so it is (re)built wherever //--- the grid is - never per bar. Non-fatal on failure; see BuildCrossAssetPanel(). BuildCrossAssetPanel(barsNow); EnsureSpreadSeries(barsNow); //--- Meta target: resolve the candidate corpus onto THIS era's bar grid before pass 1 walks it //--- (series indices shift on every closed bar, so the resolution is per-era, like the caches). //--- No candidates is not a trainable state - end the run loudly instead of scanning for nothing. if(IsMetaTarget() && !MetaPrepareEra(barsNow)) { PrintFormat("%s: era start ABORTED - no usable meta candidates on this chart (see the" " MetaCorpus lines above for the corpus/offset diagnostics); ending this training" " run, it re-arms on the next scheduled call", ID); FinalizeTrainRun(); return; } add_loop = false; //--- Label/feature cache invalidation: MQL5 timeseries indices are always relative to "now" //--- (index 0 = current bar), so every new closed candle shifts every older bar's index - a //--- cache keyed by index would silently misalign the moment that happens. See //--- EnsureBarCachesCapacity() for why `bars` + m_Time.GetData(0) are the correct/sufficient //--- invalidation keys. //--- When a wipe happens MID-RUN (a new candle closed while training was still going - e.g. the //--- market reopening after the weekend), the label cache comes back empty and the lazy per-bar //--- fallback (ComputeLabelForBar) labels everything Neutral by design (recent pivots are //--- unconfirmable) - so continuing on a wiped cache silently turns the REST OF THE RUN into //--- training AND scoring against an all-Neutral world. Observed 2026-07-19: eras 18-20 started //--- right after the Sunday session open - IS error collapsed 0.44->0.22, OOS "accuracy" soared //--- to 84.9% with Buy/Sell recall n/a and era time halved, the convergence machinery happily //--- rewarding all-Neutral predictions on 100%-Neutral relabeled truth. Re-arm the same chunked //--- eager prebuild that seeded era 0 and defer this era until it completes; its completion //--- re-seeds the class tallies (m_prebuildSeedPending) so the next era's priors reflect the //--- freshly relabeled window. //--- CAPTURED BEFORE THE CALL, and that is the whole point. EnsureBarCachesCapacity assigns BOTH //--- invalidation keys (m_labelCacheBars = bars, m_labelCacheAnchorTime = m_Time.GetData(0)) before //--- it returns true, so a message built afterwards reads the values it just overwrote: the two bar //--- counts are equal BY CONSTRUCTION and the anchor is always the live one. The line that exists //--- to name which key tripped could therefore never name it, and it printed //--- "era sized 16236 bars, cache holds 16236" for two members wedged for 77 minutes (SP500 and //--- XAUUSD LSTM, 2026-08-17 19:43 -> 21:00, stuck at era 1 while their siblings passed era 200). //--- Equal numbers were read as "not the size then", which is not something that message was ever //--- able to establish. int barsBefore = m_labelCacheBars; datetime anchorBefore = m_labelCacheAnchorTime; datetime anchorNow = m_Time.GetData(0); if(EnsureBarCachesCapacity(bars) && m_labelCachePrebuilt) { //--- The failure mode: the era and the prebuild disagree about `bars`, or about which bar is //--- index 0, and re-arm each other forever - caches wiped, relabelled, wiped again, no era ever //--- runs. Name WHICH of the two keys tripped and by how much, or the next occurrence costs //--- another session to attribute. string sizeKey = (bars != barsBefore) ? StringFormat("SIZE CHANGED %d -> %d", barsBefore, bars) : "size unchanged"; string anchorKey = (anchorNow != anchorBefore) ? StringFormat("ANCHOR MOVED %s -> %s", TimeToString(anchorBefore), TimeToString(anchorNow)) : "anchor unchanged"; ReportTrainStall(StringFormat("cache invalidated at era start - %s, %s (era sized %d bars, cache" " held %d). An anchor that moves EVERY era with the size steady is" " a new candle each pass or a Time buffer that is not being" " refreshed; a size that moves is the era/prebuild disagreement.", sizeKey, anchorKey, bars, barsBefore)); StartLabelCachePrebuild(); return; } //--- freeze the just-finished era's true class totals for this new era's priors (see //--- m_prevEraTrueBuyCount's declaration comment) before resetting the live counters below - EXCEPT //--- right after StartLabelCachePrebuild()/AdvanceLabelCachePrebuild() seeded them for era 0: the //--- live m_trueBuyCount/Sell/Neutral tally is still all-zero at that point (nothing trained yet), //--- so copying it here would silently stomp the real upfront tally back to an empty distribution. if(m_prebuildSeedPending) m_prebuildSeedPending = false; else { m_prevEraTrueBuyCount = m_trueBuyCount; m_prevEraTrueSellCount = m_trueSellCount; m_prevEraTrueNeutralCount = m_trueNeutralCount; } //--- Natural class base rates for the live logit-adjusted decision (see AdjustedSignalFromSoftmax): //--- derived from the same just-finished-era true class totals the oversampling ratio uses, so live //--- calibrates to exactly the distribution the model was measured against. Both branches above //--- leave m_prevEraTrue* holding the freshest real tally (prebuild-seeded on era 0, copied here //--- otherwise), so updating from them here covers both paths. //--- 2026-08-01: THE `if(!m_evalMode)` GUARD THAT USED TO WRAP THIS IS GONE, because it silently //--- disabled the entire imbalance correction for the whole auto-tune search. ApplyLogitAdjustment() //--- immediately below needs measured priors; without them it clears the offsets and returns. In //--- eval mode the priors were never measured, so every GA candidate - which is to say every era of //--- a run with AutoTuneIndicators on, the shipped default - trained under PLAIN cross-entropy. //--- That was invisible while the labels were near-balanced and became a total Neutral collapse the //--- moment a 2:6 barrier put the majority class at 52.5%: recall Buy 0% / Sell 0% / Neutral 100% //--- by era 5 on all four topologies, and the panel stuck on "measuring..." because a model that //--- never calls a direction never accumulates a directional tally. //--- The guard's stated fear - a search contaminating the deployed calibration - cannot happen: //--- these priors are measured from the LABEL distribution, and the tuner only perturbs indicator //--- periods (MA/RSI/MACD/Ichimoku/AD). The barrier label depends on ATR, SL_Mode and TP_Mode, none //--- of which the search touches, so every candidate sees byte-identical labels and therefore //--- identical priors. There is nothing for a candidate to contaminate. What the guard actually //--- protected against is the .stats WRITE, and that is gated separately (eval candidates never //--- checkpoint - see m_haveOosCheckpoint - and never persist). //--- Meta target trains WITHOUT the logit adjustment, deliberately: the correction exists for //--- the direction head's extreme class imbalance (directional bars were a ~6% tail), while the //--- meta label's base rate is the setup's own win rate (~40%), where plain CE is fine and the //--- operating-point fit (pass 2.5) carries the calibration. Documented deviation from the //--- design doc's "prior correction" line - the machinery is 3-class-shaped and generalizing it //--- buys nothing at this base rate. if(!IsMetaTarget()) { UpdateClassPriors(m_prevEraTrueBuyCount, m_prevEraTrueSellCount, m_prevEraTrueNeutralCount); //--- Re-install the training-time logit offsets from the priors just measured, so this //--- era's gradient tracks the distribution the era is scored against. Runs in eval mode //--- too: a GA candidate must train under the same loss as the real run or its score //--- means nothing - only the PERSISTED calibration is withheld from eval mode. ApplyLogitAdjustment(); } m_countBuySignals = 0; m_countSellSignals = 0; m_countNeutralSignals = 0; m_trueBuyCount = 0; m_trueSellCount = 0; m_trueNeutralCount = 0; m_oosBuyHits = 0; m_oosBuyTotal = 0; m_oosSellHits = 0; m_oosSellTotal = 0; m_oosNeutralHits = 0; m_oosNeutralTotal = 0; m_oosBuyPredicted = 0; m_oosBuyPredictedHits = 0; m_oosSellPredicted = 0; m_oosSellPredictedHits = 0; m_oosNeutralPredicted = 0; m_oosNeutralPredictedHits = 0; m_oosBuyPredictedWins = 0; m_oosSellPredictedWins = 0; //--- Simulated-exit accumulators, reset with the rest of the per-era OOS tallies. m_simRSum = 0.0; m_simRSumSq = 0.0; m_simTrades = 0; m_simVoteExits = 0; m_simBarrierWins = 0; m_oosWinLongTotal = 0; m_oosWinShortTotal = 0; m_oosBuyFired = 0; m_oosBuyFiredHits = 0; m_oosSellFired = 0; m_oosSellFiredHits = 0; //--- Declustered tally + its replay cursors. -1 / Neutral is "nothing seen yet this era", which is //--- what makes the first directional call of an era always survive rule 1. //--- meta per-family/per-side OOS decomposition - see the member declaration ArrayInitialize(m_metaFamCand, 0); ArrayInitialize(m_metaFamWins, 0); ArrayInitialize(m_metaFamFired, 0); ArrayInitialize(m_metaFamFiredWins, 0); ArrayInitialize(m_metaSideCand, 0); ArrayInitialize(m_metaSideWins, 0); ArrayInitialize(m_metaSideFired, 0); ArrayInitialize(m_metaSideFiredWins, 0); m_oosNmsFired = 0; m_oosNmsHits = 0; m_oosNmsLastBuyIdx = -1; m_oosNmsLastSellIdx = -1; m_oosNmsKeptIdx = -1; m_oosNmsKeptConf = 0.0; m_oosNmsKeptDir = Neutral; ArrayInitialize(m_oosTierFired, 0); ArrayInitialize(m_oosTierHits, 0); m_oosConfidenceSum = 0; // Nearest-to-present slice of this era's bars is held out as OOS and never backprop'd on; // the rest (older bars) is the IS/training slice. totalIter = (int)MathMax(bars - MathMax(m_historyBars, 0), 0); oosCutoff = (int)(MathMax(0, MathMin(100, m_oosSplitPct)) / 100.0 * totalIter); i = (int)(bars - MathMax(m_historyBars, 0) - 1); //--- Fresh era: reset pass 2's shuffled-backprop queue (see m_isTrainQueue's declaration //--- comment). Preallocated to a parity-shaped ESTIMATE, not a hard worst case: at full parity //--- all 3 classes replicate to ~the majority count, so the queue lands near 3x totalIter - //--- 4x covers that plus label drift. The queueing block below grows the arrays on demand if an //--- era ever exceeds the estimate (it used to silently DROP overflow instead - harmless at the //--- old totalIter*cap sizing, which could never fill, but real data loss now that the measured //--- ratio, not a small fixed cap, decides the replica count). ArrayResize(m_isTrainQueue, totalIter * 4); ArrayResize(m_isTrainQueueWeightScale, totalIter * 4); ArrayResize(m_isTrainQueuePrimary, totalIter * 4); ArrayResize(m_isTrainQueueCand, totalIter * 4); m_isTrainQueueCount = 0; //--- Heartbeat baseline for this era - see the member declarations for why this exists. m_eraStartTick = GetTickCount(); m_passFeatUs = 0; m_passNetUs = 0; m_passWindowOk = 0; m_passWindowFail = 0; m_passHeartbeatPrints = 0; m_lastHeartbeatTick = 0; m_isTrainCursor = 0; m_isPass2Active = false; m_isPass2Done = false; m_isCalibActive = false; m_isCalibDone = false; m_isPass3Active = false; //--- Excursion head: per-era Brier accumulators only. The base rates it is compared against are a //--- property of the data, not of the era, so they keep accumulating (see ExcursionResetEraScores). ExcursionResetEraScores(); //--- Fresh per-era predicted-signal cache for the end-of-era NMS sweep (see PruneDirectionalClusters). //--- -2 = "not scored this era" so stale bars from a longer prior era can't draw phantom arrows. if(m_signalClusterWindow > 0) { ArrayResize(m_arrowSignalCache, bars); ArrayInitialize(m_arrowSignalCache, -2.0); } } else { //--- resuming a chunk that yielded mid-bar-loop last call - pick up exactly where it left off bars = m_resumeBars; totalIter = m_resumeTotalIter; oosCutoff = m_resumeOosCutoff; add_loop = m_resumeAddLoop; i = m_resumeBarIndex; m_eraResumePending = false; } // Restore this model's own learning-rate trajectory into the shared global right before this // chunk's backProp() calls touch it - see m_modelEta's declaration comment. eta = m_modelEta; uint chunkStartTick = GetTickCount(); // Iterate over the bars - skipped entirely when resuming straight into pass 2, OR when resuming // into a still-unfinished pass 3 (see m_isPass2Done's declaration comment for why checking // m_isPass2Active alone isn't enough to detect the latter case): pass 1 already fully completed // in an earlier call either way. if(!m_isPass2Active && !m_isPass2Done) { for(; i >= 0 && !stop; i--) { //--- Build THIS bar's own feature window and feed it forward BEFORE checking/training against //--- its label - see r's declaration comment below for why the window must end AT bar i, and //--- why this must run before the label-check block rather than after: the label check needs //--- this bar's own freshly-computed prediction, not the previous iteration's (see windowOk). TempData.Clear(); //--- Window ends AT (includes) bar i itself, extending m_historyBars bars into the past - i.e. //--- "everything known as of this bar's close." Predicting label(i) - "was THIS bar the //--- reversal" - from a window that stops short of bar i itself would blind the model to the //--- most recent price action, which is exactly the information a reversal call most depends //--- on. Must match RefreshLatestSignal()'s window exactly (r=i there too - live that is //--- i=1, the newest CLOSED bar, since at the first tick after a bar opens index 0 is a //--- 1-tick forming candle no training window ever contained; see the 2026-08-11 parity //--- comment there), since that's what actually queries the deployed model live - training //--- on a different window than what gets queried at inference time would teach the wrong //--- task entirely. //--- BuildFeatureWindow() owns the Clear/Reserve/loop AND the oldest-bar-first ordering that //--- the LSTM stacks depend on - see its definition comment. int r = i; bool windowOk = false; double displayNeuron0 = 0, displayNeuron1 = 0, displayNeuron2 = 0; if(r <= bars) { ulong hbT = GetMicrosecondCount(); windowOk = BuildFeatureWindow(r); m_passFeatUs += GetMicrosecondCount() - hbT; if(windowOk) { add_loop = true; m_passWindowOk++; } else m_passWindowFail++; } TrainHeartbeat("pass 1 (scan/queue), bar", bars - MathMax(m_historyBars, 0) - i, totalIter, "scan"); //--- Determine label/queue-eligibility BEFORE running any feedForward this bar - see //--- wouldQueue's use below for why. Mirrors the label-check condition this block used to //--- gate on (moved earlier, unchanged). bool haveLabel = false, buy = false, sell = false, wouldQueue = false; //--- "some LATER pass in this same era will feed this exact bar forward anyway", which is a //--- strictly wider set than wouldQueue - see its use at the feedForward below. Declared out //--- here because the three membership tests that decide it are scoped to the label block. bool laterPassForwards = false; if(windowOk && i < (int)(bars - MathMax(m_historyBars, 0) - 1) && i > 1 && m_Time.GetData(i) > dtStudied && (m_outputNeuronsCount == 1 || m_outputNeuronsCount == 3 || IsMetaTarget())) { //--- The fractal/swing-confirmation/trend-context label at now-relative index i only depends //--- on price/ATR history, never on model state, so it's identical every era until a new bar //--- closes and shifts the index frame (see the cache invalidation check above) - cache it //--- rather than recomputing from scratch every single era. Usually already populated by //--- AdvanceLabelCachePrebuild() before era 0 ever starts - this is just a lazy fallback for //--- any index it didn't cover (e.g. bars/window drifted between prebuild and era 0's start). if(m_labelCacheHasValue[i]) { buy = m_labelCacheBuy[i]; sell = m_labelCacheSell[i]; } else { ComputeLabelForBar(i, bars, buy, sell); m_labelCacheBuy[i] = buy; m_labelCacheSell[i] = sell; //--- Kept in step with the label caches by hand here, because this fallback does not go //--- through AdvanceBarrierLabelState. ComputeLabelForBar is a stub that returns no label, //--- so "no winning direction" is the honest entry - but leaving them unwritten would mean //--- reading whatever ArrayResize left behind, under a validity flag that says otherwise. if(i < ArraySize(m_winLongCache)) { m_winLongCache[i] = false; m_winShortCache[i] = false; } m_labelCacheHasValue[i] = true; } haveLabel = true; bool isOOS = (i < oosCutoff); // Embargo: a bar's triple-barrier label is decided by the m_barrierHorizonBars bars that // follow it (see TripleBarrierLabel()). An IS bar within that distance of the OOS boundary // therefore carries a label that was only knowable using price action from inside the // held-out OOS window - purge that narrow band from backprop entirely instead of training // on it as ordinary IS. Lopez de Prado ch. 7 calls this purging, and it is the whole reason // a naive train/test split leaks on overlapping-horizon financial labels. // Was m_swingConfirmationBars + LABEL_WINDOW_BARS, which measured the ZigZag repainting // delay - the correct quantity for the old target and the wrong one for this label. int calibLo = CalibLoIndex(oosCutoff); // = oosCutoff + one purge width int calibHi = CalibHiIndex(totalIter, oosCutoff); // == calibLo when the band is empty bool isEmbargoed = (!isOOS && i < calibLo); //--- The calibration slice and its far-side purge are held out of backprop for the same //--- reason the OOS window is, and the layout is documented once at CalibLoIndex(). This is //--- the ONLY place the band is excluded from training - the walk that scores it (pass 2.5) //--- derives its bounds from the same two helpers, so the two cannot disagree about which //--- bars are held out. bool isCalib = (i >= calibLo && i < calibHi); bool isCalibPurge = (calibHi > calibLo && i >= calibHi && i < calibHi + CalibPurgeBars()); wouldQueue = (!isOOS && !isEmbargoed && !isCalib && !isCalibPurge); //--- Meta target: only bars HOSTING a candidate carry training rows, and passes 2/2.5/3 //--- forward those per-candidate themselves (the descriptor differs per candidate, so a //--- bar-level scan forward could not be reused anyway). Everything scan-side that reads a //--- forward pass is direction-display machinery, so the meta path skips it entirely. wouldQueue = wouldQueue && (!IsMetaTarget() || MetaCandFirst(i) >= 0); //--- Pass 2 re-forwards every queued bar, pass 2.5 re-forwards the whole calibration band, //--- and pass 3 re-forwards the whole OOS window - each over EXACTLY this bar set (all //--- three derive their bounds from the same helpers and apply the identical eligibility //--- test this block gates on). Only the two purge bands and the ineligible edge bars are //--- visited here and nowhere else, so those are the only ones that still need a scan-time //--- forward pass. At the shipped 30% OOS / 15% calibration split that is ~40% of all bars //--- whose forward pass was being computed twice per era and thrown away the first time. laterPassForwards = (wouldQueue || isOOS || isCalib); } //--- Only run this bar's feedForward (and the display/count/chart-draw work that depends on //--- it) when NO later pass is about to redo it anyway. A bar that a later pass revisits gets //--- a completely fresh feedForward within this same era, and that later result is strictly //--- better than this one: it is computed against weights this era has actually trained, //--- whereas the scan runs before pass 2 has taken a single step. So the scan's copy was never //--- the one that survived - it was overwritten (arrow cache, status label) or measured a //--- one-era-stale model (the predicted-class tally), and it cost a full forward pass per bar //--- to produce. The book's SGD (references\neuronetworksbook.pdf, section 1.4) is one //--- forward+backward pass per training sample, not two, and the same logic extends to the //--- held-out bars: one forward pass per SCORED bar, taken by the pass that scores it. //--- The counters and the arrow-cache write this used to perform for those bars now happen in //--- pass 2 (queued), pass 2.5 (calibration band) and pass 3 (OOS) respectively, so the //--- populations behind them are unchanged - only the weights they are measured against are, //--- and those move from pre-training to post-training, which is the honest reading. //--- Display/IS-scoring only on this path, but the same rule applies: getResults() after a //--- failed pass returns the previous bar's activations, which would be shown on the panel and //--- counted as this bar's prediction. ulong hbFwd = GetMicrosecondCount(); //--- !IsMetaTarget(): the scan-time forward exists only for the direction display/tally on //--- bars no later pass revisits; a meta forward without a candidate descriptor would be //--- width-mismatched against the meta input layer as well as meaningless. bool scanForwardOk = (windowOk && !laterPassForwards && !IsMetaTarget() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbFwd; if(scanForwardOk) { Net.getResults(TempData); if(m_outputNeuronsCount == 1) dPrevSignal = TempData[0]; else if(m_outputNeuronsCount == 3) dPrevSignal = ApplyClassificationSoftmax(); //--- Snapshot the just-computed neuron output(s) for the status label display below, before //--- the label-check block clears/refills TempData with the target label (Step A always //--- runs after this point now) - reading TempData directly for display after that would //--- show the TRUE LABEL of the bar just trained on, not the network's own prediction. if(TempData.Total() > 0) displayNeuron0 = TempData[0]; if(TempData.Total() > 1) displayNeuron1 = TempData[1]; if(TempData.Total() > 2) displayNeuron2 = TempData[2]; switch(DoubleToSignal(dPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } m_lastBarTime = m_Time.GetData(i); if(i > 0) { // NMS on: record only - the era-end sweep is the SOLE renderer, so no raw (un- // declustered) arrow is ever drawn mid-era. NMS off: draw inline as before. if(m_signalClusterWindow > 0) { if(i < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[i] = dPrevSignal; } else if(DoubleToSignal(dPrevSignal) == Neutral) DeleteObject(m_lastBarTime); else DrawObject(m_lastBarTime, dPrevSignal, m_High.GetData(i), m_Low.GetData(i)); } UpdateTrainingStatusLabel( StringFormat("Bar %d of %d -> %.2f%% (scan)", bars - i + 1, bars, (double)(bars - i + 1.0) / bars * 100), displayNeuron0, displayNeuron1, displayNeuron2, dPrevSignal); } else //--- Bars a later pass will re-forward skip the feedForward above, and they are now very //--- nearly ALL of pass 1 - the queued IS bars (~58%, processed FIRST because the loop //--- walks oldest-to-newest), plus the calibration band and the OOS slice. Painting only on //--- the forward path meant the panel sat on the idle writer's "Getting ready..." for the //--- whole IS sweep, which on a slow era reads exactly like a hang (2026-08-10: four //--- charts, 20+ minutes, no sign of life anywhere) - and after this change that would be //--- the ENTIRE scan. The label is throttled internally, so painting every bar costs //--- nothing. UpdateTrainingStatusLabel( StringFormat("Bar %d of %d -> %.2f%% (scan)", bars - i + 1, bars, (double)(bars - i + 1.0) / bars * 100), displayNeuron0, displayNeuron1, displayNeuron2, dPrevSignal); //--- META TARGET: one training row per candidate journaled at this bar (bars without a //--- candidate carry no rows - wouldQueue already required one). The win/loss label maps onto //--- the Buy/Sell class-tally slots - THE MAPPING THE WHOLE META PATH RUNS ON: win->Buy, //--- loss->Sell, Neutral unused. Under it every downstream consumer keeps its meaning with no //--- era-end changes at all: "buy recall" reads as sensitivity, "sell recall" as specificity //--- (so bothSidesLive rejects an always-call/never-call collapse), m_oosWinLongTotal/eraBars //--- becomes the base win rate - which IS the zero-skill precision of calling every candidate //--- - and coverage becomes the fraction of candidates traded. See the pass 3 meta branch. if(haveLabel && IsMetaTarget()) { for(int cd = MetaCandFirst(i); cd >= 0; cd = MetaCandNext(cd)) { if(MetaCandidateWon(cd, i)) m_trueBuyCount++; else m_trueSellCount++; if(!wouldQueue) continue; if(m_isTrainQueueCount + 1 > ArraySize(m_isTrainQueue)) { int newQueueSize = m_isTrainQueueCount + 1; ArrayResize(m_isTrainQueue, newQueueSize, 16384); ArrayResize(m_isTrainQueueWeightScale, newQueueSize, 16384); ArrayResize(m_isTrainQueuePrimary, newQueueSize, 16384); ArrayResize(m_isTrainQueueCand, newQueueSize, 16384); } m_isTrainQueue[m_isTrainQueueCount] = i; //--- no oversampling and no per-sample reweighting for the meta label (~40% base rate) m_isTrainQueueWeightScale[m_isTrainQueueCount] = 1.0; m_isTrainQueuePrimary[m_isTrainQueueCount] = true; m_isTrainQueueCand[m_isTrainQueueCount] = cd; m_isTrainQueueCount++; } } else if(haveLabel) { // True label as an ENUM_SIGNAL, derived directly from the buy/sell bools - not read // back from TempData, which no longer holds a target at this point at all (see above). ENUM_SIGNAL trueSignal = buy ? Buy : (sell ? Sell : Neutral); // Track the true class distribution this era (used below to weight IS oversampling, // and surfaced in the status label text alongside the predicted-class counts) switch(trueSignal) { case Buy: m_trueBuyCount++; break; case Sell: m_trueSellCount++; break; default: m_trueNeutralCount++; break; } // OOS scoring used to happen right here, against whatever weights this bar's earlier // feedForward (this pass) happened to be using - which for era 0 is the network's // still-untrained cold-start state (100% Neutral - see the output-layer bias seed's // declaration comment), and for every later era is last era's END-of-training state, // never THIS era's. That silently gave every era's OOS score a full one-era lag behind // its own training, and made era 0's OOS score meaningless by construction. OOS scoring // now happens in its own pass (see m_isPass3Active's declaration comment), AFTER pass 2 // has actually trained on this era's IS data, against a fresh feedForward on each OOS // bar rather than this scan's now-stale one. if(wouldQueue) { // Queue this bar for pass 2's shuffled backProp instead of training on it here, // immediately, in strict chronological order - see m_isTrainQueue's declaration // comment for the full rationale. The predicted-signal counts, the chart-marker draw, // and the dForecast/dUndefine IS-accuracy update are all computed in pass 2 instead, // against that bar's own freshly-recomputed confidence - see the matching block right // after pass 2's Net.feedForward() call. // // EVERY BAR IS QUEUED EXACTLY ONCE. Class imbalance is corrected analytically inside // the gradient by the logit-adjusted loss, not by duplicating minority bars here. // // The history is worth keeping, because it is why the data-level approach was // abandoned rather than merely re-tuned. Four successive versions of oversampling all // collapsed, in both directions: // v1 uncapped replication x an independent loss weight (up to ~4.5x total) -> // Buy-only collapse, OOS ~10%, IS error 0.37->0.57 in 4 eras. // v2 capped the ratio before splitting it between the two -> mathematically the // same total correction as pure loss weighting, which had already failed. // v3 replication alone, capped at 3x against a ~5.3x imbalance -> Neutral collapse, // Buy/Sell recall 0% for 6 straight eras (2026-07-18). // v4 replication to ~90% parity (up to 28x) -> measured across six topologies on // 2026-07-29, every model drove ONE direction to ~50% recall and abandoned the // other, and which direction was arbitrary. One era in 1,301 cleared the floor. // The through-line: replication makes Buy and Sell compete for the same replicated // capacity, and Adam's mt/sqrt(vt) normalisation (Kingma & Ba 2015) is near-invariant // to the gradient rescaling that the loss-weighted variants relied on. Per Buda, Maki // & Mazurowski 2018, stacking data-level and cost-level corrections on one axis is not // reliably additive - and the logit-adjusted loss replaces BOTH with a single // correction that is provably consistent for balanced error. // // Pass 2 still Fisher-Yates shuffles the queue: chronological order correlates // consecutive gradients, which is the same correlated-momentum overshoot documented at // AI\Network.mqh's MAX_WEIGHT_DELTA comment. That reason is independent of replication // and survives it. //--- MINORITY REPLAY REMOVED 2026-07-31. Every bar is queued exactly once; class //--- imbalance is corrected analytically in the gradient by the logit-adjusted loss //--- (Menon et al. 2021) instead of by duplicating rare bars in the data. Stacking the //--- two double-counts the same imbalance - Buda et al. 2018 - and the replay branch had //--- in fact been gated OFF for the whole shipped configuration, so this is the code //--- catching up with the behaviour rather than a change in it. Measured 2026-07-29 //--- across six topologies, replay made Buy and Sell compete for the same replicated //--- capacity: every model drove ONE direction to ~50% recall and abandoned the other, //--- and which direction was arbitrary. One era in 1,301 cleared the per-class floor. int repCount = 1; double perOccurrenceScale = 1.0; //--- grow on demand (reserve keeps this amortized-rare) - the prealloc above is an //--- estimate, and dropping overflow would silently starve exactly the minority //--- classes the replication exists to protect if(m_isTrainQueueCount + repCount > ArraySize(m_isTrainQueue)) { int newQueueSize = m_isTrainQueueCount + repCount; ArrayResize(m_isTrainQueue, newQueueSize, 16384); ArrayResize(m_isTrainQueueWeightScale, newQueueSize, 16384); ArrayResize(m_isTrainQueuePrimary, newQueueSize, 16384); ArrayResize(m_isTrainQueueCand, newQueueSize, 16384); } for(int rep = 0; rep < repCount; rep++) { m_isTrainQueue[m_isTrainQueueCount] = i; m_isTrainQueueWeightScale[m_isTrainQueueCount] = perOccurrenceScale; //--- rep 0 is this bar's single "counts once" occurrence - see m_isTrainQueuePrimary. //--- Every rep still trains; only the reported IS accuracy looks at this flag. m_isTrainQueuePrimary[m_isTrainQueueCount] = (rep == 0); m_isTrainQueueCand[m_isTrainQueueCount] = -1; m_isTrainQueueCount++; } } } stop = IsStopped() || m_trainingStopRequested; if(!stop && i > 0 && GetTickCount() - chunkStartTick >= TRAIN_TIME_BUDGET_MS) { //--- yield: save exactly enough to resume this same era, mid-bar-loop, on the next call - //--- see m_trainRunActive's declaration comment for why this must happen instead of //--- letting one era (or the whole run) process synchronously to completion m_resumeBars = bars; m_resumeTotalIter = totalIter; m_resumeOosCutoff = oosCutoff; m_resumeAddLoop = add_loop; m_resumeBarIndex = i - 1; m_eraResumePending = true; // Save this model's own learning-rate trajectory back out of the shared global before // yielding - see m_modelEta's declaration comment. m_modelEta = eta; return; } } //--- PASS 1 IS OVER (the yield above is the only other way out of that loop). This is the point //--- that decides whether the era does any work at all, and until now it said nothing. //--- add_loop is exactly "m_passWindowOk > 0". When it is false, pass 2, pass 3, the era counter, //--- the checkpoint and every log line below are ALL skipped - Train() returns having done //--- nothing, m_eraResumePending is still false, and the next call restarts the SAME era from //--- bar 0. An infinite, completely silent 0->100% "scan" loop with no journal output whatsoever, //--- which is what the panel showed on 2026-08-10 once the dispatch fix let pass 1 run at speed. //--- A PARTIAL failure is normal and must not be alarming: the loop walks oldest-to-newest and //--- the deepest bars legitimately predate the indicators' warm-up, so those windows fail and are //--- cached as misses. Only a TOTAL failure is a defect, so only that one shouts. if(!stop) { if(!add_loop) { //--- SELF-HEAL BEFORE RESTARTING. A sweep that produced no usable window at all will //--- produce exactly the same result next time unless something changes, because every bar //--- it touched is now answered from the feature cache. Dropping the cached verdicts forces //--- the next sweep to recompute against whatever the terminal has finished loading since - //--- which is the difference between recovering a few seconds later and looping forever. //--- The known cause of this state (a cold ATR read by a resumed model before its //--- indicators had calculated) is fixed at source in BufferTempData, so reaching here at //--- all now means an unknown cause; recover anyway rather than spin, and say so. ArrayInitialize(m_featureCacheHasValue, false); //--- Routed through ReportTrainStall rather than printed directly: a discarded era restarts //--- immediately, so this condition repeats as fast as pass 1 can sweep, and an unthrottled //--- line would bury the journal. That helper already rate-limits to one line a minute and //--- carries the run-state flags this needs read alongside it. //--- The CAUSE, not just the count. "0 of 54681 usable" reads identically for a cold ATR, //--- a conditionally-missing optional feature block and an out-of-range index, and telling //--- them apart by reasoning cost a whole debugging session once already. string whyLine; if(m_windowFailSlot == -2) whyLine = "no window has been attempted yet this run (m_windowFailSlot unset) - the" " failure is upstream of BuildFeatureWindow"; else if(m_windowFailSlot < 0) whyLine = StringFormat("every lookback bar was ACCEPTED and the window was still" " short: %d of %d values. A feature block emitted fewer values" " than m_neuronsCount promises", m_windowFailTotal, (int)m_historyBars * m_neuronsCount); else //--- THE BLOCK, NOT JUST THE SLOT. The old form ended at "slot 0 REJECTED (window had //--- 24 of 832 values)" plus a guess ("an indicator warm-up or a history-edge read"), //--- and that guess was read as a finding five times across 2026-08-17. m_featureFailBlock //--- is written by the guard that actually returned false, and IndicatorDepthReport() //--- prints every handle's BarsCalculated() beside it, so the next occurrence is READ //--- rather than reasoned about. A total failure (ok=0) is itself evidence: it means the //--- newest anchors failed too, which no depth shortfall can cause. { string byBlock = m_featureFailBlock; if(byBlock == "") byBlock = "(no guard recorded - the rejection came from a TempData.Add failure," " not a data guard)"; whyLine = StringFormat("lookback slot %d of %d REJECTED the bar at series index %d" " (window had %d of %d values). REJECTED BY: %s. Slot 0 is the" " DEEPEST lookback of the window, so with ok=0 the newest anchors" " failed as well - which rules out a plain history-edge read and" " points at a buffer that is unreadable at every index." " Per-indicator depth:%s", m_windowFailSlot, (int)m_historyBars, m_featureFailIdx, m_windowFailTotal, (int)m_historyBars * m_neuronsCount, byBlock, IndicatorDepthReport()); } ReportTrainStall(StringFormat("pass 1 finished but NOT ONE of %d scanned bars produced a" " usable feature window, so the era is discarded and restarts" " from scratch (feature cache dropped so the next sweep" " recomputes) - windows ok=%d failed=%d, BuildFeatureWindow" " needs %d values per bar (historyBars=%d x featuresPerBar=%d)" " over %d bars | LAST FAILURE: %s", totalIter, m_passWindowOk, m_passWindowFail, (int)m_historyBars * m_neuronsCount, (int)m_historyBars, m_neuronsCount, bars, whyLine)); //--- transient cause (cold indicator) -> arm the era-start backoff instead of resweeping //--- at full speed; see the backoff block at the top of the fresh-era branch. //--- THIS BACKOFF WAS DEAD UNTIL 2026-08-17 and that is why the USDJPY/XAUUSD stall never //--- recovered. It arms only on m_featureFailTransient, and of the guards that can reject //--- a bar only the open/ATR pair ever set that flag - the MA, RSI, MACD and Ichimoku //--- guards did not. A cold ADMovingAverage therefore looked PERMANENT, so the sweep was //--- re-run at full speed forever, and six instances doing that on a six-core box starved //--- the very indicator they were waiting on. The mechanism was right; nothing reached it. //--- //--- 2026-08-17 (second pass): the backoff is now UNCONDITIONAL on a total failure, not //--- gated on m_featureFailTransient. Naming every guard's flag correctly is a list that has //--- to stay correct forever - the same shape of fix the feature cache abandoned above for //--- the same reason - and being wrong once costs a chart. The gate is also pointless here: //--- whether the cause is transient or permanent, a sweep in which ZERO of 50,163 bars //--- produced a window will produce zero again if it restarts a millisecond later, and doing //--- so at full speed is what starved six indicator threads on a six-core box. Back off in //--- both cases. The flag is kept for what it legitimately decides - whether the miss may be //--- cached (see BufferTempData) - which is a per-bar question, not a scheduling one. m_coldSweepTick = GetTickCount(); } else { //--- Healthy pass 1. Quiet on a fast era, but an era that has already taken longer than //--- PASS1_LOUD_AFTER_MS is one somebody is watching a progress bar on, and the single most //--- useful thing to tell them is that the scan ENDED and what it handed to pass 2 - that //--- is what separates "slow but advancing" from "sweeping the same bars forever". //--- FIRST HEALTHY SWEEP is the only moment the assembled feature vector is known to be //--- readable and not yet been trained on - so it is where the block-level autopsy belongs. //--- See ReportFeatureHealth() for the two silent failures that motivated it. ReportFeatureHealth(bars); //--- Same moment, same reason: the first sweep that produced usable windows is the first //--- point at which the era's bar grid, the measured barrier geometry and the label lifespan //--- are all real numbers rather than defaults. What this one says is what the CONFIGURATION //--- can prove - published before the run spends a thousand eras chasing something the OOS //--- window could never certify. //--- oosCutoff IS the OOS count, not the IS/OOS boundary counted from the other end: //--- pass 3 grades `isOOS = (i < oosCutoff)`, so these are the bars the deploy gate will //--- ever get to see, and they are the only ones this budget may be denominated in. ReportDetectability(oosCutoff); const uint PASS1_LOUD_AFTER_MS = 10000; //--- The calibration band is reported here, beside the queue count it is subtracted from, so //--- the two are read together: a run where the band silently came out empty (see //--- CalibBandBars) is one whose operating point is no longer being refitted at all, and the //--- only place that is visible is next to the number it should have reduced. string pass1Line = StringFormat("%s: era %d pass 1 done in %.0fs - %d of %d bars usable" " (%d failed, normal over the oldest bars), %d queued for" " backprop | %d bars held out to calibrate the operating" " point (+2x%d purged around it)", ID, (int)m_eraCount, (GetTickCount() - m_eraStartTick) / 1000.0, m_passWindowOk, m_passWindowOk + m_passWindowFail, m_passWindowFail, m_isTrainQueueCount, CalibBandBars(totalIter, oosCutoff), CalibPurgeBars()); if(GetTickCount() - m_eraStartTick >= PASS1_LOUD_AFTER_MS) Print(pass1Line); else PrintVerbose(pass1Line); } } } // end if(!m_isPass2Active) - pass 1 //--- Pass 2: replay the bars pass 1 queued into m_isTrainQueue for backProp, in a freshly //--- shuffled order - see m_isTrainQueue's declaration comment for the full rationale. Runs //--- whenever pass 1 just finished (or we resumed straight into an already-active pass 2 - see //--- m_isPass2Active's declaration comment); skipped on a stopped run, an era with no valid window //--- at all (add_loop still false), or - critically - a resume into a still-unfinished pass 3 (see //--- m_isPass2Done's declaration comment): without this last check, that resume would re-shuffle //--- and replay the ENTIRE queue again from scratch every single call. if(!stop && add_loop && !m_isPass2Done) { if(!m_isPass2Active) { m_isPass2Active = true; m_isTrainCursor = 0; //--- MINI-BATCH ON, for pass 2 only (2026-08-09 audit, F4). Scoped this tightly on purpose: //--- pass 2 is the only place Net.backProp() runs during era training, and everything else //--- that ever backprops on this net - notably OnlineLearnStep, which learns from a handful of //--- newly-confirmed live bars - wants its update applied immediately rather than held back //--- waiting for a batch that may never fill. Switched back off where pass 2 completes. Net.SetBatchSize(TRAIN_BATCH_SIZE); // 2026-07-28: a "replay-only optimizer override" was removed from here. It captured every // neuron's optimizer and forced the whole net to SGD for the duration of pass 2, on the // rationale that oversampled minority bars should not "exploit the same Adam-style momentum // path as the base training pass". But pass 2 IS the base training pass - it is the only place // Net.backProp() is called during training at all (pass 1 only feeds forward and queues) - so // the override applied to 100% of weight updates, not to some replay subset. // Adam's mt/vt were therefore never updated and its bias-correction step counter never // advanced: TrainingOptimizer=ADAM was silently a no-op and the model trained purely on // SGD+momentum at Adam's learning rate. It arrived with the DFA change set and was never part // of any validated run. The optimizer the user selects is now the optimizer that runs. // Fisher-Yates shuffle - a fresh random order every era, so Adam's momentum can't keep // landing on the same contiguous same-class label run at the same point in the sequence every // single era. Barrier labels make those runs LONGER than the old exact-pivot ones (adjacent // bars share most of their forward window, so they usually resolve the same way), which makes // the shuffle matter more here, not less. m_isTrainQueueWeightScale is swapped in lockstep - each // slot's stored per-occurrence weight (see the queueing block's oversampling comment) must // stay attached to the same bar index it was computed for. for(int sIdx = m_isTrainQueueCount - 1; sIdx > 0; sIdx--) { //--- ShuffleRandomIndex, NOT MathRand()%: the queue routinely exceeds MathRand()'s 15-bit //--- range on a full-history window, which silently biased this shuffle - see the helper. int sJ = ShuffleRandomIndex(sIdx + 1); int sTmp = m_isTrainQueue[sIdx]; m_isTrainQueue[sIdx] = m_isTrainQueue[sJ]; m_isTrainQueue[sJ] = sTmp; double sScaleTmp = m_isTrainQueueWeightScale[sIdx]; m_isTrainQueueWeightScale[sIdx] = m_isTrainQueueWeightScale[sJ]; m_isTrainQueueWeightScale[sJ] = sScaleTmp; //--- the primary flag must travel with its own slot too, or the "count this bar once" //--- marker would end up attached to a different bar's occurrence - see m_isTrainQueuePrimary bool sPrimTmp = m_isTrainQueuePrimary[sIdx]; m_isTrainQueuePrimary[sIdx] = m_isTrainQueuePrimary[sJ]; m_isTrainQueuePrimary[sJ] = sPrimTmp; //--- the candidate id is the meta label's identity - it must stay attached to its slot int sCandTmp = m_isTrainQueueCand[sIdx]; m_isTrainQueueCand[sIdx] = m_isTrainQueueCand[sJ]; m_isTrainQueueCand[sJ] = sCandTmp; } } for(; m_isTrainCursor < m_isTrainQueueCount; m_isTrainCursor++) { int qi = m_isTrainQueue[m_isTrainCursor]; TrainHeartbeat("pass 2 (shuffled backprop), sample", m_isTrainCursor + 1, m_isTrainQueueCount, "training"); ulong hbT = GetMicrosecondCount(); bool qWindowOk = BuildFeatureWindow(qi); //--- Meta target: the input is window + per-candidate setup descriptor; the net's input layer //--- is sized for both (NetInputWidth), so the append must happen before EVERY forward. if(qWindowOk && IsMetaTarget()) AppendCandidateFeatures(m_isTrainQueueCand[m_isTrainCursor]); m_passFeatUs += GetMicrosecondCount() - hbT; //--- A failed forward pass must NOT be followed by backProp() further down this block: the //--- output layer would still hold the PREVIOUS sample's activations, so the update would be //--- this bar's label against another bar's prediction - training on pure noise while every //--- accuracy counter kept reporting normally. hbT = GetMicrosecondCount(); bool qForwardOk = (qWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbT; if(qWindowOk && !qForwardOk && !forwardFailureReported) { forwardFailureReported = true; Print(__FUNCTION__ + ": CNet::feedForward FAILED at era " + IntegerToString((int)m_eraCount) + " - this era's remaining samples are being skipped, not trained. A layer is refusing to" " accept its own output (check the preceding BufferWrite/BufferRead lines for which" " buffer, and see NormalizeHost in AI\\NeuronBatchNorm.mqh for the batch-norm case)."); } //--- META TARGET pass 2: binary win/loss backprop per candidate. No excursion step (that head //--- belongs to the direction models), no arrows, no 3-class softmax - just the running IS //--- stats under the win->Buy / loss->Sell mapping documented at pass 1's meta branch. if(qForwardOk && IsMetaTarget()) { int qc = m_isTrainQueueCand[m_isTrainCursor]; Net.getResults(TempData); double qPwin = MetaWinProbability(); bool qWon = MetaCandidateWon(qc, qi); //--- the argmax of a 2-class softmax IS pWin >= 0.5 - the unthresholded "call" bool qCall = (qPwin >= 0.5); bool qHit = (qCall == qWon); if(qHit) dForecast += (100 - dForecast) / Net.recentAverageSmoothingFactor; else dForecast -= dForecast / Net.recentAverageSmoothingFactor; dUndefine -= dUndefine / Net.recentAverageSmoothingFactor; if(qCall) m_countBuySignals++; else m_countSellSignals++; //--- persistent IS precision over the candidates the model would trade, in WINS - the //--- meta analogue of the direction path's m_cumIsTotal contract (compared against the OOS //--- side as the overfitting signal, so both must count the same quantity). if(qCall) { m_cumIsTotal++; if(qWon) m_cumIsCorrect++; } UpdateTrainingStatusLabel( StringFormat("Training candidate %d of %d -> %.2f%% (shuffled)", m_isTrainCursor + 1, m_isTrainQueueCount, (double)(m_isTrainCursor + 1.0) / MathMax(m_isTrainQueueCount, 1) * 100), (TempData.Total() > 0) ? TempData[0] : 0.0, (TempData.Total() > 1) ? TempData[1] : 0.0, 0.0, qPwin); TempData.Clear(); //--- slot 0 = P(win), slot 1 = P(loss); same label smoothing as the 3-class head TempData.Add(qWon ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add(qWon ? LABEL_SMOOTH_LOW : LABEL_SMOOTH_HIGH); ulong hbBpM = GetMicrosecondCount(); Net.backProp(TempData, 1.0); m_passNetUs += GetMicrosecondCount() - hbBpM; } else if(qForwardOk) { //--- EXCURSION HEAD, trained here and ONLY here in pass 2. Must run BEFORE getResults(), //--- which overwrites TempData in place with the classifier's output activations - the //--- feature window is gone after the next line. Only primary occurrences: the replay queue //--- oversamples for CLASS balance, and duplicating minority-direction bars would skew the //--- excursion-size distribution the head is trying to learn (same correction m_cumIsTotal //--- makes, for a target where it matters even more - size and direction are unrelated, so //--- a direction-balanced sample is a biased size sample). if(m_isTrainQueuePrimary[m_isTrainCursor]) ExcursionTrainStep(qi); Net.getResults(TempData); // Must go through ApplyClassificationSoftmax() (3-output case) before reading the // per-class values below - Net.getResults() returns each output neuron's own independent // SIGMOID activation (each already in [0,1] but NOT summing to 1 across the three), not a // true class-conditional probability distribution; ApplyClassificationSoftmax() is what // turns that into one (and is also what pass 1/3's displayNeuron0/1/2 already go through). double qPrevSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; double pt0 = (TempData.Total() > 0) ? TempData[0] : 0.0; double pt1 = (TempData.Total() > 1) ? TempData[1] : 0.0; double pt2 = (TempData.Total() > 2) ? TempData[2] : 0.0; bool qBuy = m_labelCacheHasValue[qi] ? m_labelCacheBuy[qi] : false; bool qSell = m_labelCacheHasValue[qi] ? m_labelCacheSell[qi] : false; ENUM_SIGNAL qTrueSignal = qBuy ? Buy : (qSell ? Sell : Neutral); //--- Per-direction outcomes for this bar - see m_oosBuyPredictedWins. Needed on the IS side //--- too: the operating point is FITTED here and GRADED by the OOS gate, so if the two //--- optimise different quantities the threshold is tuned for the wrong objective. bool qWinLong = (m_labelCacheHasValue[qi] && qi < ArraySize(m_winLongCache)) ? m_winLongCache[qi] : false; bool qWinShort = (m_labelCacheHasValue[qi] && qi < ArraySize(m_winShortCache)) ? m_winShortCache[qi] : false; UpdateTrainingStatusLabel( StringFormat("Training bar %d of %d -> %.2f%% (shuffled)", m_isTrainCursor + 1, m_isTrainQueueCount, (double)(m_isTrainCursor + 1.0) / MathMax(m_isTrainQueueCount, 1) * 100), pt0, pt1, pt2, qPrevSignal); //--- Predicted-signal tally, chart marker, and IS-accuracy stat that pass 1 used to compute //--- from its own (now-removed) redundant feedForward on this same bar - see pass 1's //--- wouldQueue comment. Uses THIS feedForward's result (the only one this bar gets), so //--- these now reflect the model's state as of this bar's own turn in the shuffled replay //--- (post any earlier-shuffled bar's backProp this era), not a separate pre-training //--- snapshot - matching how a standard shuffled-epoch SGD run reports running training //--- accuracy during the epoch rather than in a discarded pre-epoch dry run. switch(DoubleToSignal(qPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } datetime qBarTime = m_Time.GetData(qi); // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(qi < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[qi] = qPrevSignal; } else if(DoubleToSignal(qPrevSignal) == Neutral) DeleteObject(qBarTime); else DrawObject(qBarTime, qPrevSignal, m_High.GetData(qi), m_Low.GetData(qi)); bool qClassified = (DoubleToSignal(qPrevSignal) == Buy || DoubleToSignal(qPrevSignal) == Sell || DoubleToSignal(qPrevSignal) == Neutral); if(qClassified) { bool isHit = (DoubleToSignal(qPrevSignal) == qTrueSignal); if(isHit) dForecast += (100 - dForecast) / Net.recentAverageSmoothingFactor; else dForecast -= dForecast / Net.recentAverageSmoothingFactor; dUndefine -= dUndefine / Net.recentAverageSmoothingFactor; //--- Compounded, persistent DIRECTIONAL win-rate: count only bars the model actually called //--- Buy or Sell (a Neutral "no trade" call is neither a win nor a loss), so this tracks the //--- accuracy of its directional signals rather than the Neutral-inflated all-class rate. //--- ...and count each BAR once, not each oversampled OCCURRENCE (m_isTrainQueuePrimary): //--- the queue duplicates minority bars up to ~21x, so counting every occurrence scored this //--- metric over a ~58%-directional set while its OOS counterpart scored the real ~6% //--- distribution - two numbers that look comparable, aren't, and made a healthy run read as //--- severe overfitting. See m_isTrainQueuePrimary for the worked example. ENUM_SIGNAL qPred = DoubleToSignal(qPrevSignal); //--- Did the implied trade pay? Same distinction as the OOS side - see //--- m_oosBuyPredictedWins - and it has to be made identically on both, because the IS and //--- OOS win rates are read side by side as the overfitting signal. Measuring one in wins //--- and the other in label agreement would put a fixed gap between them that has nothing //--- to do with generalization. bool qTradeWon = (qPred == Buy) ? qWinLong : ((qPred == Sell) ? qWinShort : false); if(m_isTrainQueuePrimary[m_isTrainCursor] && (qPred == Buy || qPred == Sell)) { m_cumIsTotal++; if(qTradeWon) m_cumIsCorrect++; } //--- THE OPERATING-POINT FIT NO LONGER HARVESTS HERE. It used to, on the argument that //--- pass 2's forward pass made the margin free - which was true, and irrelevant: these //--- are the bars the very next line backprops on, so within a handful of eras the //--- histogram describes memorized behaviour and not the model's behaviour on unseen //--- bars. It moved to the held-out calibration walk below; DIR_CONF_CALIB_PCT_OF_IS //--- carries the measured IS-vs-OOS divergence that forced the move. } else if(qBuy && qSell) dUndefine += (100 - dUndefine) / Net.recentAverageSmoothingFactor; TempData.Clear(); if(m_outputNeuronsCount == 1) TempData.Add(qBuy && !qSell ? 1 : !qBuy && qSell ? -1 : 0); else if(m_outputNeuronsCount == 3) { TempData.Add(qBuy ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add(qSell ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); TempData.Add((!qBuy && !qSell) ? LABEL_SMOOTH_HIGH : LABEL_SMOOTH_LOW); } // Per-slot weight from m_isTrainQueueWeightScale[m_isTrainCursor], decided once at queue // time in pass 1. Currently always 1.0: class imbalance is corrected analytically inside // the gradient by the logit-adjusted loss, so there is no per-sample reweighting left to // apply here at all. Kept as a real per-slot value rather than a literal 1.0 inline so a // future supplemental weight can be reintroduced without re-touching the queueing or // shuffle code. //--- FOCAL-LOSS MODULATION REMOVED 2026-07-31. It multiplied this weight by (1-pt)^gamma, //--- a second correction on the same axis as the logit adjustment - the stacking failure //--- Buda et al. 2018 describes and this file already cited in two other places. It was //--- running at an eighth strength (gamma * 0.125), damped by the replay toggle, for a //--- replay path that the adjusted loss had already switched off - so the damping was //--- calibrated against a mechanism that was not running. See the class-imbalance audit in //--- Variables\Inputs.mqh. double qSampleWeight = m_isTrainQueueWeightScale[m_isTrainCursor]; ulong hbBp = GetMicrosecondCount(); Net.backProp(TempData, qSampleWeight); m_passNetUs += GetMicrosecondCount() - hbBp; } //--- YIELD ON TIME **OR** ON A STOP REQUEST. The time budget bounds THROUGHPUT; it does not //--- bound LATENCY to an unload. MetaTrader measures its ~4,500 ms teardown budget from the //--- stop request and OnDeinit cannot start until whatever is in flight returns, so a chunk //--- that keeps training for its full slice after _StopFlag is raised spends that time out of //--- the chart cleanup - which is what strands arrows and panels (see OnDeinit's ordering //--- notes). Pass 1 has checked IsStopped() all along; passes 2, 2.5 and 3 never did, and //--- they are the ones that grow with history. Yielding here is free: the resume state below //--- is written either way, so a stopped chunk simply never gets re-entered. if(m_isTrainCursor + 1 < m_isTrainQueueCount && (IsStopped() || GetTickCount() - chunkStartTick >= TRAIN_TIME_BUDGET_MS)) { //--- yield: save enough to resume PASS 2 mid-queue on the next call - m_isPass2Active //--- and m_isTrainCursor (both members) carry the actual resume position; bars/oosCutoff/ //--- add_loop are stashed the same way pass 1 already does, since era-end logic just //--- below still needs them once pass 2 finishes. m_resumeBars = bars; m_resumeTotalIter = totalIter; m_resumeOosCutoff = oosCutoff; m_resumeAddLoop = add_loop; m_resumeBarIndex = i; m_eraResumePending = true; m_modelEta = eta; return; } } //--- Apply whatever the final (usually short) batch of this era accumulated, and return the net //--- to per-sample updates. MUST happen before pass 3 scores anything: the selection metric has //--- to describe weights with no unapplied gradients sitting behind them, and the checkpoint //--- taken from that score has to be the same model. FlushBatch scales by the REAL sample count, //--- so a short trailing batch still takes a correctly-sized step. Net.FlushBatch(); Net.SetBatchSize(1); m_isPass2Active = false; m_isPass2Done = true; } //--- Pass 2.5: the CALIBRATION walk. Scores the held-out band (see CalibLoIndex for the layout) with //--- the weights pass 2 just finished training, harvests the margin histogram, and fits this era's //--- operating point - all BEFORE pass 3 grades anything. //--- //--- Three properties have to hold at once and only this position gives all three: //--- not trained on - pass 1 kept the band out of the backprop queue, so the histogram measures //--- generalization rather than memorization (the failure that moved it here) //--- not graded - pass 3's OOS window is disjoint from the band, so the numbers the deploy //--- gate ranks are still produced by a threshold that never saw them //--- current weights - after pass 2, so the operating point belongs to the weights it will be //--- applied to; the margin distribution moves with them every era //--- //--- Batch norm is frozen for the walk exactly as pass 3 freezes it, and for the same reason: an //--- unfrozen BN would let the running statistics drift while scoring, so the fitted threshold would //--- describe a slightly different function than the one pass 3 then grades. if(!stop && add_loop && !m_isCalibDone) { int calibLo = CalibLoIndex(oosCutoff); int calibHi = CalibHiIndex(totalIter, oosCutoff); if(!m_isCalibActive) { m_isCalibActive = true; Net.SetBatchNormFrozen(true); ResetDirConfHistogram(); //--- Same upper clamp pass 3 applies: a bar needs m_historyBars of older bars behind it to //--- build a window at all, so the band is trimmed to what is actually scoreable. m_calibStartIndex = (int)MathMin(calibHi - 1, bars - MathMax(m_historyBars, 0) - 2); m_calibIndex = m_calibStartIndex; } for(; m_calibIndex >= calibLo; m_calibIndex--) { int ci = m_calibIndex; //--- Same eligibility test pass 1 gates labelling on (its line reads //--- `i < bars-historyBars-1 && i > 1 && Time[i] > dtStudied`), so this walk can only score bars //--- pass 1 actually produced a label for. Pass 3 applies the identical test on its own window. if(!(ci < (int)(bars - MathMax(m_historyBars, 0) - 1) && ci > 1 && m_Time.GetData(ci) > dtStudied)) continue; TrainHeartbeat("pass 2.5 (calibration), bar", m_calibStartIndex - m_calibIndex + 1, m_calibStartIndex - calibLo + 1, "calibrating"); //--- META TARGET: harvest one histogram sample per CANDIDATE in the band - margin is P(win), //--- outcome is the candidate's own triple-barrier win. The window must be rebuilt per //--- candidate because the appended descriptor differs; the bar features behind it come from //--- the feature cache, so the rebuild is cheap. Bars without candidates contribute nothing - //--- the coverage denominator (m_dirConfPrimaryBars) is CANDIDATES, matching the coverage //--- numerator the threshold admits, and FitDirConfThreshold's coverage x (precision - //--- break-even) objective is exactly the design doc's operating point for the meta head. if(IsMetaTarget()) { for(int cd = MetaCandFirst(ci); cd >= 0; cd = MetaCandNext(cd)) { ulong hbM = GetMicrosecondCount(); bool mWindowOk = BuildFeatureWindow(ci); if(mWindowOk) AppendCandidateFeatures(cd); m_passFeatUs += GetMicrosecondCount() - hbM; hbM = GetMicrosecondCount(); bool mForwardOk = (mWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbM; if(!mForwardOk) break; Net.getResults(TempData); AccumulateDirConfSample(MetaWinProbability(), MetaCandidateWon(cd, ci), true); } } else { ulong hbC = GetMicrosecondCount(); bool cWindowOk = BuildFeatureWindow(ci); m_passFeatUs += GetMicrosecondCount() - hbC; hbC = GetMicrosecondCount(); bool cForwardOk = (cWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbC; if(cForwardOk) { Net.getResults(TempData); //--- RAW argmax softmax, NOT AdjustedSignalFromSoftmax(): feeding the fit its own already- //--- thresholded decisions would make the threshold a fixed point of itself, able only to //--- ratchet upward. ApplyClassificationSoftmax() leaves the probabilities in TempData, //--- which is what DirectionalMargin() reads. double cSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; ENUM_SIGNAL cPred = DoubleToSignal(cSignal); //--- Scored in WINS - did the trade this call implies actually pay - not in agreement with //--- the collapsed 3-class label. Same distinction pass 3 makes (see m_oosBuyPredictedWins); //--- the two must be measured identically or the operating point is chosen for one quantity //--- and graded on another. bool cWinLong = (m_labelCacheHasValue[ci] && ci < ArraySize(m_winLongCache)) ? m_winLongCache[ci] : false; bool cWinShort = (m_labelCacheHasValue[ci] && ci < ArraySize(m_winShortCache)) ? m_winShortCache[ci] : false; bool cTradeWon = (cPred == Buy) ? cWinLong : ((cPred == Sell) ? cWinShort : false); //--- isPrimaryBar is unconditionally true: this walk visits each bar once in chronological //--- order, so there is no oversampled replay to correct for here. AccumulateDirConfSample(DirectionalMargin(), cTradeWon, true); //--- Predicted-class tally and chart marker for the calibration band, which pass 1 used to //--- compute from its own (now-removed) redundant feedForward on this same bar. The panel //--- reads these counts directly against m_trueBuyCount/m_trueSellCount/m_trueNeutralCount, //--- which pass 1 still accumulates over EVERY labelled bar - so the predicted side has to //--- keep spanning the same bars or the two lines stop being comparable. Same value pass 1 //--- used (raw argmax, not the thresholded decision), so only the weights differ: these are //--- post-training now, matching what pass 2 already does for the queued bars. switch(cPred) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } datetime cBarTime = m_Time.GetData(ci); if(ci > 0) { // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(ci < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[ci] = cSignal; } else if(cPred == Neutral) DeleteObject(cBarTime); else DrawObject(cBarTime, cSignal, m_High.GetData(ci), m_Low.GetData(ci)); } } } // end direction (non-meta) calibration body //--- Time OR stop - see pass 2's matching comment. if(m_calibIndex - 1 >= calibLo && (IsStopped() || GetTickCount() - chunkStartTick >= TRAIN_TIME_BUDGET_MS)) { //--- yield: m_isCalibActive + m_calibIndex carry the resume position, same as passes 1-3. m_resumeBars = bars; m_resumeTotalIter = totalIter; m_resumeOosCutoff = oosCutoff; m_resumeAddLoop = add_loop; m_resumeBarIndex = i; m_eraResumePending = true; m_modelEta = eta; return; } } Net.SetBatchNormFrozen(false); //--- An empty band (era too short to carve one - see CalibBandBars) means there is no measurement //--- this era, which is not the same as a measurement that says "trade everything". Leave the //--- operating point exactly where the last successful fit put it rather than refitting on nothing. if(calibHi > calibLo) FitDirConfThreshold(); m_isCalibActive = false; m_isCalibDone = true; } //--- Pass 3: OOS scoring, chronological, AFTER pass 2 has actually trained on this era's IS data - //--- see m_isPass3Active's declaration comment for why this can no longer happen inline during //--- pass 1's scan. if(!stop && add_loop) { if(!m_isPass3Active) { m_isPass3Active = true; //--- Freeze batch-norm running statistics for the whole scoring walk (2026-08-09 audit, F5). //--- Unfrozen, every scored bar advances the EMA mean/variance, so (a) the OOS number partly //--- measures BN drift rather than the trained function, and (b) the same weights score //--- differently depending on what was scored before them - and this pass produces the exact //--- numbers checkpoint selection and the deploy gate rank on, which must be a pure function //--- of the checkpoint. Same reasoning (and same mechanism) as ValidateCpuInference. The //--- freeze persists across mid-pass chunk yields (the flag lives on the layers) and is //--- lifted right after the walk completes; FinalizeTrainRun also unfreezes defensively for //--- the stop-mid-pass path. Live/online adaptation is untouched - only scoring is frozen. Net.SetBatchNormFrozen(true); m_oosScoreStartIndex = (int)MathMin(oosCutoff - 1, bars - MathMax(m_historyBars, 0) - 2); m_oosScoreIndex = m_oosScoreStartIndex; for(int rn = 0; rn < 3; rn++) { m_oosOutMin[rn] = DBL_MAX; m_oosOutMax[rn] = -DBL_MAX; } m_oosOutSpreadSum = 0.0; m_oosOutCount = 0; m_oosNeutralStrict = 0; m_oosNeutralTie = 0; m_oosTieBuySell = 0; m_oosRailBars = 0; } for(; m_oosScoreIndex >= 2; m_oosScoreIndex--) { int oi = m_oosScoreIndex; if(!(oi < (int)(bars - MathMax(m_historyBars, 0) - 1) && m_Time.GetData(oi) > dtStudied)) continue; TrainHeartbeat("pass 3 (OOS scoring), bar", m_oosScoreStartIndex - m_oosScoreIndex + 1, m_oosScoreStartIndex + 1, "scoring"); //--- META TARGET OOS scoring, one row per candidate, feeding the SAME members the era-end //--- selection/deploy block reads - under the win->Buy / loss->Sell mapping (pass 1 comment) //--- every downstream figure keeps a correct meta meaning: //--- dirPrecPct = wins among candidates the operating point trades (the win rate) //--- chancePrec = base win rate of ALL candidates (always-call zero-skill reference, //--- which under cost-charged win-counting IS the break-even coincidence //--- the 2026-08-09 note below derives) //--- coveragePct = fraction of candidates traded //--- buy/sell recall = sensitivity/specificity, so bothSidesLive rejects the //--- always-call and never-call collapses //--- so checkpoint selection, the edge floor's standard error, the plateau ladder and the //--- family-wise deploy gate all run UNCHANGED on the meta head. if(IsMetaTarget()) { for(int cd = MetaCandFirst(oi); cd >= 0; cd = MetaCandNext(cd)) { ulong hbM = GetMicrosecondCount(); bool mWindowOk = BuildFeatureWindow(oi); if(mWindowOk) AppendCandidateFeatures(cd); m_passFeatUs += GetMicrosecondCount() - hbM; hbM = GetMicrosecondCount(); bool mForwardOk = (mWindowOk && TempData.Total() >= NetInputWidth() && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbM; if(!mForwardOk) { if(mWindowOk && !forwardFailureReported) { forwardFailureReported = true; Print(__FUNCTION__ + ": CNet::feedForward FAILED during meta OOS scoring at era " + IntegerToString((int)m_eraCount) + " - affected candidates are excluded."); } break; } Net.getResults(TempData); double oPwin = MetaWinProbability(); bool oWon = MetaCandidateWon(cd, oi); bool oCall = (oPwin >= 0.5); // the 2-class argmax bool oHit = (oCall == oWon); m_oosSamples++; m_oosConfidenceSum += oPwin; if(dOosError < 0) dOosError = 0; //--- mapped confusion counts (recall gate + balanced-accuracy diagnostics) if(oWon) { m_oosBuyTotal++; if(oHit) m_oosBuyHits++; //--- the always-call reference wins exactly when the candidate wins m_oosWinLongTotal++; } else { m_oosSellTotal++; if(oHit) m_oosSellHits++; } //--- predicted-keyed tallies (panel Called/precision diagnostics) if(oCall) { m_oosBuyPredicted++; if(oHit) m_oosBuyPredictedHits++; if(oWon) m_oosBuyPredictedWins++; m_countBuySignals++; //--- persistent OOS precision over called candidates, in WINS (matches the IS side) m_cumOosTotal++; if(oWon) m_cumOosCorrect++; } else { m_oosSellPredicted++; if(oHit) m_oosSellPredictedHits++; m_countSellSignals++; } //--- THE POPULATION THAT TRADES: candidates clearing the fitted operating point - what //--- the deployability gate and selection score actually read (see the era-end block). bool oFired = (oPwin >= m_dirConfThreshold); if(oFired) { m_oosBuyFired++; if(oWon) m_oosBuyFiredHits++; } //--- per-family / per-side decomposition of the same population (see the declaration) int oFam = m_metaCandFamily[cd]; int oSideIdx = (m_metaCandSide[cd] > 0) ? 0 : 1; if(oFam >= 0 && oFam < 4) { m_metaFamCand[oFam]++; if(oWon) m_metaFamWins[oFam]++; if(oFired) { m_metaFamFired[oFam]++; if(oWon) m_metaFamFiredWins[oFam]++; } } m_metaSideCand[oSideIdx]++; if(oWon) m_metaSideWins[oSideIdx]++; if(oFired) { m_metaSideFired[oSideIdx]++; if(oWon) m_metaSideFiredWins[oSideIdx]++; } if(oHit) { dOosForecast += (100 - dOosForecast) / Net.recentAverageSmoothingFactor; dOosError -= dOosError / Net.recentAverageSmoothingFactor; } else { dOosForecast -= dOosForecast / Net.recentAverageSmoothingFactor; dOosError += (100 - dOosError) / Net.recentAverageSmoothingFactor; } UpdateTrainingStatusLabel( StringFormat("Scoring OOS bar %d of %d -> %.2f%% (meta)", m_oosScoreStartIndex - m_oosScoreIndex + 1, m_oosScoreStartIndex + 1, (double)(m_oosScoreStartIndex - m_oosScoreIndex + 1.0) / MathMax(m_oosScoreStartIndex + 1, 1) * 100), (TempData.Total() > 0) ? TempData[0] : 0.0, (TempData.Total() > 1) ? TempData[1] : 0.0, 0.0, oPwin); } } else { ulong hbT = GetMicrosecondCount(); bool oWindowOk = BuildFeatureWindow(oi); m_passFeatUs += GetMicrosecondCount() - hbT; //--- Same guard as pass 2, and it matters more here: OOS accuracy is what checkpoint selection //--- and the plateau ladder's auto-deploy both rank on, so scoring a stale forward pass would //--- not just be wrong, it would be wrong in the one number that decides which model ships. //--- A skipped bar simply isn't counted; it never becomes a hit or a miss. hbT = GetMicrosecondCount(); bool oForwardOk = (oWindowOk && TempData.Total() >= (int)m_historyBars * m_neuronsCount && Net.feedForward(TempData)); m_passNetUs += GetMicrosecondCount() - hbT; if(oWindowOk && !oForwardOk && !forwardFailureReported) { forwardFailureReported = true; Print(__FUNCTION__ + ": CNet::feedForward FAILED during OOS scoring at era " + IntegerToString((int)m_eraCount) + " - affected bars are excluded from the OOS" " accuracy rather than scored against a stale prediction."); } if(oForwardOk) { //--- EXCURSION HEAD scored on the SAME held-out bars the classifier is graded on, and for //--- the same reason: it never trained on them. Before getResults() overwrites TempData. ExcursionScoreStep(oi); Net.getResults(TempData); // Raw output stats MUST be captured here, before ApplyClassificationSoftmax() overwrites // TempData[0..2] in place with the softmax probabilities - see m_oosOutMin's declaration // comment for what these feed. if(m_outputNeuronsCount == 3 && TempData.Total() >= 3) { double rawHi = -DBL_MAX, rawLo = DBL_MAX; for(int rn = 0; rn < 3; rn++) { double rv = TempData.At(rn); if(rv < m_oosOutMin[rn]) m_oosOutMin[rn] = rv; if(rv > m_oosOutMax[rn]) m_oosOutMax[rn] = rv; rawHi = MathMax(rawHi, rv); rawLo = MathMin(rawLo, rv); } m_oosOutSpreadSum += rawHi - rawLo; m_oosOutCount++; //--- WHY Neutral won, split into its two causes - see m_oosNeutralStrict's declaration. //--- Read on the RAW logits, which is legitimate because the softmax below is strictly //--- monotone: it cannot change the ordering, and it cannot break a tie either. Doing it //--- here also means these counters see the same values the min/max/spread stats do, //--- BEFORE ApplyClassificationSoftmax() overwrites TempData[0..2] in place. double rB = TempData.At(0), rS = TempData.At(1), rN = TempData.At(2); bool strictB = (rB > rS && rB > rN); bool strictS = (rS > rB && rS > rN); bool strictN = (rN > rB && rN > rS); if(strictN) m_oosNeutralStrict++; else if(!strictB && !strictS) { //--- No class holds a strict majority, so the top two are EXACTLY equal and //--- ApplyClassificationSoftmax() returns Neutral by the tie rule, not by choice. m_oosNeutralTie++; //--- The expensive subset: Buy and Sell tied AT the top (either a 2-way tie above //--- Neutral, or a 3-way). The net had a directional reading and float equality //--- threw it away. if(rB == rS && rB >= rN) m_oosTieBuySell++; } //--- Sigmoid rails. The head is SIGMOID (Topology.mqh), so 0 and 1 are its asymptotes; //--- a raw value sitting ON one in float32 is the saturation that MAKES exact ties //--- possible. Counted per BAR, not per output, so this is directly comparable to //--- m_oosOutCount. if(rawLo <= 1e-6 || rawHi >= 1.0 - 1e-6) m_oosRailBars++; } double oPrevSignal = (m_outputNeuronsCount == 3) ? ApplyClassificationSoftmax() : TempData[0]; double oDeploySignal = oPrevSignal; if(m_outputNeuronsCount == 3) oDeploySignal = AdjustedSignalFromSoftmax(); double oNeuron0 = (TempData.Total() > 0) ? TempData[0] : 0.0; double oNeuron1 = (TempData.Total() > 1) ? TempData[1] : 0.0; double oNeuron2 = (TempData.Total() > 2) ? TempData[2] : 0.0; bool oBuy = m_labelCacheHasValue[oi] ? m_labelCacheBuy[oi] : false; bool oSell = m_labelCacheHasValue[oi] ? m_labelCacheSell[oi] : false; ENUM_SIGNAL oTrueSignal = oBuy ? Buy : (oSell ? Sell : Neutral); //--- Per-direction OUTCOMES, kept apart from the label - see m_oosBuyPredictedWins. bool oWinLong = (m_labelCacheHasValue[oi] && oi < ArraySize(m_winLongCache)) ? m_winLongCache[oi] : false; bool oWinShort = (m_labelCacheHasValue[oi] && oi < ArraySize(m_winShortCache)) ? m_winShortCache[oi] : false; //--- ONE CURRENCY, and it is the live one. oEnsembleVote is the signed vote this member //--- would have cast on this bar - m_weight x its tier's pattern weight, the same number //--- CExpertSignalCustom::Direction() sums and the same 0-100 win-rate scale //--- Min_Vote_Open/Min_Vote_Close are expressed in. It feeds BOTH consumers below: //--- * the ensemble deploy gate, via EnsembleOosContribute; //--- * the exit simulation, via m_oosDecisionSeries. //--- //--- Both used to be handed the raw signed CONFIDENCE instead (the gate then multiplying it //--- by 100), which is a plausible-looking number in the right RANGE and the wrong //--- CURRENCY: confidence is a head output, the live contribution is a DB-ranked win-rate //--- weight, and nothing ties them together. Certifying on one while trading the other is //--- the 2026-08-09 geometry incident's exact shape, and the exit simulation was modelling //--- a close rule the EA does not run. double oEnsembleVote = LiveVoteContribution(oDeploySignal); //--- The divisor term that goes with it - the member's weight WHENEVER it evaluated the //--- bar, Neutral included, because consensus arithmetic (2026-08-19) has abstention //--- dilute. Was zeroed on abstention under union semantics. double oEnsembleWeight = ModuleWeight(); if(m_ensembleMember && m_labelCacheHasValue[oi]) EnsembleOosContribute(oi, oEnsembleVote, oEnsembleWeight, oWinLong, oWinShort, (oBuy || oSell)); //--- THE DECISION SERIES, for the exit simulation. Recorded for EVERY scanned OOS bar, not //--- only the ones that fire, because a vote-flip exit is read at bars the model did NOT //--- enter on - it is the reversal that closes a position opened earlier. Stored on the //--- bar's own series index so SimulateTradeOutcome can walk it forward against price. if(oi >= 0 && oi < ArraySize(m_oosDecisionSeries)) m_oosDecisionSeries[oi] = oEnsembleVote; UpdateTrainingStatusLabel( StringFormat("Scoring OOS bar %d of %d -> %.2f%% (post-training)", m_oosScoreStartIndex - m_oosScoreIndex + 1, m_oosScoreStartIndex + 1, (double)(m_oosScoreStartIndex - m_oosScoreIndex + 1.0) / MathMax(m_oosScoreStartIndex + 1, 1) * 100), oNeuron0, oNeuron1, oNeuron2, oDeploySignal); // Held-out bar: score the model's freshly-trained-this-era forecast against the actual // outcome without learning from it - keeps the OOS accuracy an honest overfitting signal. bool oClassified = (DoubleToSignal(oPrevSignal) == Buy || DoubleToSignal(oPrevSignal) == Sell || DoubleToSignal(oPrevSignal) == Neutral); if(oClassified) { m_oosSamples++; m_oosConfidenceSum += MathAbs(oPrevSignal); if(dOosError < 0) dOosError = 0; bool hit = (DoubleToSignal(oPrevSignal) == oTrueSignal); ENUM_SIGNAL oPred = DoubleToSignal(oPrevSignal); //--- Did the TRADE this call implies actually pay? Distinct from `hit`, which asks the //--- narrower question of whether the call matched the single label the bar was collapsed //--- to. On a both-won bar the label names one direction and this pays either way. bool oTradeWon = (oPred == Buy) ? oWinLong : ((oPred == Sell) ? oWinShort : false); //--- Zero-skill reference, measured over EVERY scored bar (not just the called ones): //--- what always-long and always-short would have collected. See m_oosWinLongTotal. if(oWinLong) m_oosWinLongTotal++; if(oWinShort) m_oosWinShortTotal++; //--- Compounded, persistent DIRECTIONAL win-rate: count only bars the model actually called //--- Buy or Sell (Neutral "no trade" calls aren't wins or losses). Scored on oTradeWon, so //--- the number the panel shows under "win rate" is one - it used to be label agreement, //--- which is a different quantity and reads low by exactly the both-won share. if(oPred == Buy || oPred == Sell) { m_cumOosTotal++; if(oTradeWon) m_cumOosCorrect++; } // Per-class confusion counts, used for the Buy/Sell recall convergence gate below switch(oTrueSignal) { case Buy: m_oosBuyTotal++; if(hit) m_oosBuyHits++; break; case Sell: m_oosSellTotal++; if(hit) m_oosSellHits++; break; default: m_oosNeutralTotal++; if(hit) m_oosNeutralHits++; break; } //--- DECLUSTERED count: of the calls that would actually become POSITIONS, how many were //--- right. Since 2026-08-09 live NMS gates the trade and not just the arrow (see //--- RefreshLatestSignal), so every other figure on this line describes a strictly larger //--- population than the EA trades - roughly 8x larger at the shipped 6-bar window. This //--- pair is the one that answers "what would I have made". //--- Same rule as PruneDirectionalClusters/NmsLiveAccept, replayed here because pass 3 //--- walks OOS bars oldest-to-newest (m_oosScoreIndex descends, and a HIGH index is an OLD //--- bar), which is exactly the order the live sweep sees them in. //--- Reported alongside, NOT substituted into selectionScore: declustering cuts coverage //--- from ~64% of bars to ~8%, which sits below MIN_COVERAGE_FRACTION_OF_BASE_RATE and //--- would make every checkpoint undeployable overnight. That is the minRR and recall-floor //--- catch-22 twice over, so the floor gets re-derived from these measurements first. if(m_signalClusterWindow > 0) { //--- oDeploySignal, NOT oPrevSignal: live NMS runs downstream of the confidence //--- threshold (RefreshLatestSignal feeds NmsLiveAccept the ADJUSTED decision), so //--- replaying it on the raw argmax declusters a different, strictly larger stream //--- than the EA ever sees - different survivors, not just more of them, because rule 1 //--- collapses runs and rule 3 alternates over whatever sequence it is given. Bars the //--- threshold rejects must not consume a cluster slot or set the alternation state. ENUM_SIGNAL nmsDir = DoubleToSignal(oDeploySignal); if(nmsDir == Buy || nmsDir == Sell) { //--- Confidence for rule 2's cross-direction resolution comes from the same adjusted //--- decision, matching NmsLiveAccept's input exactly. double nmsConf = MathAbs(oDeploySignal); int lastSame = (nmsDir == Buy) ? m_oosNmsLastBuyIdx : m_oosNmsLastSellIdx; //--- 1) same-direction contiguous collapse; last-seen advances either way so a whole //--- run collapses to its first bar. bool cont = (lastSame >= 0 && (lastSame - oi) <= m_signalClusterWindow); if(nmsDir == Buy) m_oosNmsLastBuyIdx = oi; else m_oosNmsLastSellIdx = oi; bool keep = !cont; //--- 2) cross-direction resolution against the last KEPT opposite signal: flicker at //--- one turn zone resolves to the more confident side. if(keep && m_oosNmsKeptIdx >= 0 && m_oosNmsKeptDir != nmsDir && m_oosNmsKeptDir != Neutral && (m_oosNmsKeptIdx - oi) <= m_signalClusterWindow) keep = (nmsConf > m_oosNmsKeptConf); //--- 3) ALTERNATION, identical to NmsLiveAccept's rule 3. MUST match it exactly: //--- this tally is what the deploy gate grades, so any divergence certifies one //--- strategy and trades another - the same class of defect as the geometry the //--- gate certified while OpenParams placed something else (9a7c37f). if(keep && BothDirectionsTradeable() && m_oosNmsKeptIdx >= 0 && m_oosNmsKeptDir == nmsDir) keep = false; if(keep) { m_oosNmsKeptIdx = oi; m_oosNmsKeptDir = nmsDir; m_oosNmsKeptConf = nmsConf; m_oosNmsFired++; //--- oTradeWon, not `hit`: this pair exists specifically to answer "what would I //--- have made", and that is a question about the trade, not about the label. //--- Safe to reuse even though oTradeWon is keyed to oPrevSignal's direction: the //--- threshold only ever turns a direction into Neutral, so reaching here at all //--- means oDeploySignal and oPrevSignal name the SAME side. if(oTradeWon) m_oosNmsHits++; } } } // Same confusion counts keyed by what the model actually PREDICTED this bar, not the // true label - see m_oosBuyPredicted's declaration comment for why recall alone can // hide an over-firing class. switch(DoubleToSignal(oPrevSignal)) { case Buy: m_oosBuyPredicted++; if(hit) m_oosBuyPredictedHits++; if(oWinLong) m_oosBuyPredictedWins++; break; case Sell: m_oosSellPredicted++; if(hit) m_oosSellPredictedHits++; if(oWinShort) m_oosSellPredictedWins++; break; default: m_oosNeutralPredicted++; if(hit) m_oosNeutralPredictedHits++; break; } // Live-decision precision: scores the bars on which the deployed EA would actually cast a // directional vote, using the prior-corrected (logit-adjusted) posterior - see // AdjustedSignalFromSoftmax()/RefreshLatestSignal(). The recall/argmax-precision above stay // on the raw argmax (the model's intrinsic class separation, which the convergence gate // needs); THIS scores what trades live, so the panel's live precision number is the // precision a buyer gets forward. TempData still holds this bar's raw softmax probs // (nothing overwrote them since ApplyClassificationSoftmax above), so the adjustment reads // them directly. Neutral picks aren't counted - they cast no vote. // No confidence-floor term any more: with the floor removed, EVERY non-Neutral adjusted // decision casts a vote (at its tier weight), so any threshold here would score a // different population than the one that actually votes. Whether a given vote goes on to // OPEN a position additionally depends on Min_Vote_Open versus the AVERAGE across all // voting filters, which this per-bar training-time scorer has no visibility of - so this // stays the honest "would have voted, and was it right" measure rather than pretending to // model the aggregate. if(m_outputNeuronsCount == 3) { double adjSig = AdjustedSignalFromSoftmax(); ENUM_SIGNAL adjEnum = DoubleToSignal(adjSig); if(adjEnum != Neutral) { //--- Same substitution as everywhere else in this block: what a buyer gets forward is //--- whether the trade paid, not whether it agreed with a collapsed label. bool fireHit = (adjEnum == Buy) ? oWinLong : oWinShort; //--- Bucket the same fire by confidence tier - see m_oosTierFired. THE ARGUMENT //--- MATTERS: this was ConfidenceTier(), whose comment claimed it "reads the net's //--- CURRENT outputs, which is exactly the bar AdjustedSignalFromSoftmax() just //--- scored". It does not. ConfidenceTier() reads dPrevSignal, and dPrevSignal is //--- assigned in PASS 1 only (the in-sample pass) - never anywhere in this OOS //--- scan. So every scanned bar of the era was bucketed by one stale, unrelated //--- bar's confidence, and the whole era's fires landed in a SINGLE tier. //--- //--- That is the "tier prec T0:72%(828) T1:n/a(0) T2:n/a(0) T3:n/a(0)" symptom //--- recorded on 2026-08-16 and attributed to the calibration clamp. The clamp was //--- a real cause and was fixed then (raw magnitude, see ConfidenceTierFor); this //--- is a SECOND, independent cause that produces the identical single-bucket //--- output and survived that fix untouched. Two causes, one symptom - which is //--- why the log kept reading the same after the first was closed. //--- //--- adjSig is the bar this iteration actually scored, and fireHit below is derived //--- from it, so tier and outcome now describe the same bar by construction. int fireTier = ConfidenceTierFor(adjSig); if(fireTier >= 0 && fireTier < 4) { m_oosTierFired[fireTier]++; if(fireHit) m_oosTierHits[fireTier]++; } if(adjEnum == Buy) { m_oosBuyFired++; if(fireHit) m_oosBuyFiredHits++; } else { m_oosSellFired++; if(fireHit) m_oosSellFiredHits++; } } } if(hit) { dOosForecast += (100 - dOosForecast) / Net.recentAverageSmoothingFactor; dOosError -= dOosError / Net.recentAverageSmoothingFactor; } else { dOosForecast -= dOosForecast / Net.recentAverageSmoothingFactor; dOosError += (100 - dOosError) / Net.recentAverageSmoothingFactor; } } //--- Predicted-class tally for the OOS window, which pass 1 used to compute from its own //--- (now-removed) redundant feedForward on this same bar - see the laterPassForwards //--- comment there. On the RAW argmax, exactly as pass 1 and pass 2 count it: this pair of //--- panel lines reports what the model called versus what was true, so it must not be //--- silently narrowed to the thresholded decision on one third of the bars. switch(DoubleToSignal(oPrevSignal)) { case Buy: m_countBuySignals++; break; case Sell: m_countSellSignals++; break; default: m_countNeutralSignals++; break; } // Chart annotation for this (OOS) bar, using post-training weights - pass 1 no longer // draws these at all (it used to, from a pre-training snapshot that this then overwrote). m_lastBarTime = m_Time.GetData(oi); if(oi > 0) { // NMS on: record only (the era-end sweep renders); off: draw inline. See pass 1's note. if(m_signalClusterWindow > 0) { if(oi < ArraySize(m_arrowSignalCache)) m_arrowSignalCache[oi] = oDeploySignal; } else if(DoubleToSignal(oDeploySignal) == Neutral) DeleteObject(m_lastBarTime); else DrawObject(m_lastBarTime, oDeploySignal, m_High.GetData(oi), m_Low.GetData(oi)); } } } // end direction (non-meta) OOS scoring body //--- Time OR stop - see pass 2's matching comment. if(m_oosScoreIndex - 1 >= 2 && (IsStopped() || GetTickCount() - chunkStartTick >= TRAIN_TIME_BUDGET_MS)) { //--- yield: save enough to resume PASS 3 mid-walk on the next call - m_isPass3Active and //--- m_oosScoreIndex (both members) carry the actual resume position. m_resumeBars = bars; m_resumeTotalIter = totalIter; m_resumeOosCutoff = oosCutoff; m_resumeAddLoop = add_loop; m_resumeBarIndex = i; m_eraResumePending = true; m_modelEta = eta; return; } } m_isPass3Active = false; //--- THE EXIT SIMULATION, and it has to run HERE rather than inline in the scan above. A vote-flip //--- exit for a trade entered at bar r is decided by what the model says at bars r-1, r-2, ... - //--- which are NEWER bars, and pass 3 walks oldest-to-newest (m_oosScoreIndex descends, a high //--- index is an old bar). So at the moment bar r is graded its own exit has not been decided yet. //--- Only now is m_oosDecisionSeries complete over the whole OOS window. SimulateExitPolicyOutcomes(); ReportExitPolicyDivergence(); //--- Excursion head's verdict for this era, printed while its accumulators are complete and //--- before the next era's fresh-era block clears them. ExcursionReport(); //--- Scoring finished - resume the normal always-adapting statistics (see the freeze at pass-3 //--- start) before anything else runs a forward pass. Net.SetBatchNormFrozen(false); //--- Pass 3 done => every scored bar's prediction is now in m_arrowSignalCache. Collapse each //--- same-direction cluster to its earliest bar so the chart shows one arrow per real turn. //--- ONCE PER ERA, AT ITS END, and this is the only place the historical arrows are rendered: //--- with NMS on, passes 1/2.5/3 only RECORD into the cache (see the "NMS on: record only" //--- comment in pass 1), and nothing repaints or erases at the start of an era. Verified //--- 2026-08-16 after a report of arrows vanishing when the next era begins - the vanishing is //--- this sweep deleting arrows on bars the era genuinely scored Neutral, not a timing fault. PruneDirectionalClusters(bars); //--- ...and now that this era's per-tier outcomes are complete, turn them into the vote weights //--- the NEXT era (and live trading) will use. See RankTiersFromOos(). RankTiersFromOos(); } //--- Diagnostic recall snapshot for the periodic progress log further below - populated inside //--- the m_oosSamples>0 recall-gate block when this era actually computes it; stays -1 ("n/a" //--- in the log) on eras that don't (era 0, or a stopped/cap-hit era). int logBuyRecallPct = -1, logSellRecallPct = -1, logNeutralRecallPct = -1; //--- Balanced accuracy (macro-recall) this era, surfaced in the log so the metric the checkpoint //--- is now selected on is visible - see m_bestBalancedOos. -1 ("n/a") on eras that don't score. int logBalancedAccPct = -1; int logCoveragePct = -1; int logDirPrecPct = -1; //--- Zero-skill precision for this era's label mix - see chancePrecPct. Logged beside the selection //--- score because the raw precision number is meaningless without it: 44% is excellent against a //--- 3% chance level and worthless against a 43% one, and the whole 2026-08-01 confusion was //--- reading the first as if it were the second. int logChancePrecPct = -1; //--- Predicted-rate (of all OOS bars this era, how often the model called this class at all) and //--- precision (of the calls it made, how many were right) for Buy/Sell - m_oosBuyPredicted/ //--- m_oosSellPredicted (see that member's declaration comment) were already being tracked for //--- exactly this but never surfaced anywhere. A recall-only view can't tell "the model never once //--- calls Sell" (predicted rate stuck at 0%) apart from "the model calls Sell plenty but always on //--- the wrong bars" (predicted rate healthy, precision near 0%) - both show up identically as 0% //--- Sell recall, but point at completely different problems (a suppressed/dead output vs. a //--- miscalibrated decision boundary), so this splits them out. int logBuyPredPct = -1, logSellPredPct = -1, logBuyPrecPct = -1, logSellPrecPct = -1; //--- Live-fired precision (%) per direction this era - the precision on just the bars that cleared //--- the confidence floor under the live/prior-corrected decision, i.e. what would actually trade. int logBuyFiredPrecPct = -1, logSellFiredPrecPct = -1; bool shouldLogProgress = false; //--- era complete (ran out of bars) or a stop was requested mid-era if(add_loop) { m_eraCount++; m_erasSinceCooldown++; //--- EMA shadow-weight deployment: blend the shadow a small step (SHADOW_WEIGHT_TAU) toward //--- Net's just-updated weights, every era - see m_shadowNet's declaration comment. Must run //--- here, inside the era loop, not just once at Train()-end: the whole point is damping the //--- WITHIN-run oscillation (era-to-era whipsaw), which a single end-of-run blend would miss //--- entirely. EnsureShadowNet(); if(CheckPointer(m_shadowNet) != POINTER_INVALID) m_shadowNet.BlendWeightsFrom(Net, SHADOW_WEIGHT_TAU); //--- Status-label progress is invisible with no chart (headless/optimization runs), and even in //--- visual mode a long training run can otherwise look "stuck" for a long time with no //--- Journal output at all - log progress at most every ~5s (real wall-clock, not simulated //--- time) so an operator can tell it's actively working, not hung. The actual Print() is //--- deferred past the recall-gate block below (see logBuyRecallPct etc.) so this line can //--- show per-class OOS recall - once OOS accuracy alone clears the target, recall is the //--- most common thing still silently blocking convergence, and previously had no visibility //--- outside of a regression event. static uint lastProgressLogTick = 0; uint nowTick = GetTickCount(); shouldLogProgress = (nowTick - lastProgressLogTick >= 5000); if(shouldLogProgress) lastProgressLogTick = nowTick; //--- Era cap. There used to be a second, much smaller cap here for throwaway auto-tune //--- candidates; the filter tuner does not train candidates at all, so only the real one remains. int effectiveEraCap = m_maxErasPerRun; //--- PLATEAU LADDER, terminal stage: training stopped improving and both escape attempts (two //--- learning-rate warm restarts) failed to find anything better - see the ladder in the era-end block //--- below, which is what raised m_plateauStage this far and already logged why. This is the //--- normal, expected way a run finishes now that there is no absolute accuracy target to hit: //--- it trains until it genuinely stops getting better, then deploys its best checkpoint. //--- Same mechanism as the operator's "No" answer at the era cap (see that branch's comments for //--- why m_trainingComplete is set here and why m_trainingStopRequested deliberately is NOT): //--- stop ends this era loop, FinalizeTrainRun() then restores and deploys the best checkpoint. //--- ENSEMBLE: the ladder and the gate are the ensemble's, so the trigger is its verdict //--- (g_ensDeployApproved), not this member's own stage. Every member sees the same flag on //--- its next Train() call and finalises within one era of the others, each restoring its own //--- half of the JOINT checkpoint - see the ENSEMBLE DEPLOY GATE block. bool deployNow = m_ensembleMember ? (g_ensDeployApproved && m_haveOosCheckpoint) : ((m_plateauStage >= PLATEAU_STAGE_DEPLOY || m_isErrorPlateaued) && m_bestPassedRecall && m_haveOosCheckpoint //--- AND the market was measured to hold directional information in the //--- first place. See m_dirEvidence: the MI suite has always printed this //--- verdict and then deployed regardless of what it said. && m_dirEvidence); if(deployNow) { stop = true; m_trainingComplete = true; } else if(effectiveEraCap > 0 && m_erasSinceCooldown >= effectiveEraCap) { //--- Era cap reached without converging: ask the operator whether to keep training or //--- deploy the best checkpoint and stop (see PromptContinuePastEraCap / m_maxErasPerRun). if(PromptContinuePastEraCap(dOosForecast)) { m_erasSinceCooldown = 0; // keep training - reset the cap window Print(ID + ": hit the " + IntegerToString(m_maxErasPerRun) + "-era cap (best dir-precision " + DoubleToString(m_bestBalancedOos, 1) + "%, blended OOS " + DoubleToString(dOosForecast, 1) + "%) - CONTINUING training by operator choice."); } else { //--- stop: end THIS era loop now; FinalizeTrainRun (reached via the stop path below, //--- because stop==true) deploys the best checkpoint. Deliberately do NOT set //--- m_trainingStopRequested here: m_trainingComplete alone already routes every later //--- tick to RefreshConvergedSignal (see ScheduleTrainingIfNeeded's if-branch precedence), //--- so training never re-arms - and leaving m_trainingStopRequested false lets the deployed //--- model run live inference AND online continual learning IN-SESSION, exactly like a //--- normal-convergence deploy (which never sets it either). A panel Stop (StopTraining()) //--- still sets it and halts everything, including online learning - that distinction is //--- preserved. See OnlineLearnStep()'s gate. stop = true; //--- Operator DELIBERATELY chose to deploy this best checkpoint as the final model. That's a //--- terminal decision and must be PERSISTED as such: mark it complete so a later reload //--- (chart restart OR strategy tester) runs inference instead of silently resuming a full //--- training run. This is the terminal-deploy path; a mid-training Stop click //--- (StopTraining()) leaves m_trainingComplete false on purpose so that genuinely- //--- interrupted run does resume. Note the m_trainingComplete=(m_objectiveMet&&m_oosStable) //--- line below is inside if(!stop), so it can't clobber this back to false on this path. m_trainingComplete = true; Print(ID + ": hit the " + IntegerToString(m_maxErasPerRun) + "-era cap before the plateau ladder finished (best dir-precision " + DoubleToString(m_bestBalancedOos, 1) + "%, blended OOS " + DoubleToString(dOosForecast, 1) + "%) - operator chose to DEPLOY the best checkpoint as final (marked complete; reloads will run inference, not retrain). Reaching this cap now means the run was still finding new bests, or never cleared the per-class recall floor (need >=" + IntegerToString(m_minDirectionalRecallPct) + "% each) - raise the era cap for the former, relax MinRecall/SwingConfirmationBars for the latter."); //--- NOT blocked - this branch is an explicit operator decision and stays one. But the //--- automatic ladder would refuse this model, so say so plainly rather than letting the //--- deploy read as a clean pass. See DEPLOY_FAMILY_WISE_ALPHA. ReportSelectionGateVerdict("era-cap deploy"); } } } if(!stop) { dError = Net.getRecentAverageError(); if(add_loop) { if(m_oosSamples > 0) { // Confidence calibration (classification head only - see m_confidenceCalScale's // declaration comment): compare this era's actual OOS accuracy against the average // confidence magnitude the model claimed, EMA-blend the resulting scale into // m_confidenceCalScale so SignedAIConfidence() reports something closer to a real // probability instead of the raw, uncalibrated softmax value. if(m_outputNeuronsCount == 3 && m_oosConfidenceSum > 0.0) { double empiricalAccuracy = (double)(m_oosBuyHits + m_oosSellHits + m_oosNeutralHits) / m_oosSamples; double avgClaimedConfidence = m_oosConfidenceSum / m_oosSamples; double eraScale = MathMax(0.3, MathMin(1.5, empiricalAccuracy / avgClaimedConfidence)); m_confidenceCalScale += (eraScale - m_confidenceCalScale) / Net.recentAverageSmoothingFactor; } // Per-class recall gate, symmetric across all three classes: a model that "wins" on // blended dOosForecast purely by calling everything Neutral (or, just as biased, by // over-calling Buy/Sell at Neutral's expense) would still pass a plain accuracy check - // require Buy, Sell, AND Neutral OOS recall to each individually clear // m_minDirectionalRecallPct so the network can't converge while biased toward any one // output. A class with FEWER than MIN_OOS_CLASS_SAMPLES_FOR_GATE true OOS samples this // era doesn't block (recallPct == -1 => treated as passing) so a thin OOS window doesn't // deadlock convergence early in a run. Computed BEFORE the checkpoint/eta-decay block // below (not just the final m_objectiveMet gate) so "best" ranking is recall-aware too - // see isBetterEra's comment for why that matters. // // The threshold matters: a bare ">0" here (the original behavior) let a run converge at // era 44-46 with the OOS window containing exactly ZERO true Buy/Sell bars that era // (logged as "OOS recall Buy:n/a Sell:n/a Neutral:100%") - a full Neutral-only collapse // that the gate waved through because there was nothing to measure recall against, not // because the model was actually unbiased. Requiring a real minimum sample count means // an unlucky/thin OOS slice blocks convergence instead of silently passing it. int buyRecallPct = (m_oosBuyTotal >= MIN_OOS_CLASS_SAMPLES_FOR_GATE) ? (int)MathRound(100.0 * m_oosBuyHits / m_oosBuyTotal) : -1; int sellRecallPct = (m_oosSellTotal >= MIN_OOS_CLASS_SAMPLES_FOR_GATE) ? (int)MathRound(100.0 * m_oosSellHits / m_oosSellTotal) : -1; int neutralRecallPct = (m_oosNeutralTotal >= MIN_OOS_CLASS_SAMPLES_FOR_GATE) ? (int)MathRound(100.0 * m_oosNeutralHits / m_oosNeutralTotal) : -1; logBuyRecallPct = buyRecallPct; logSellRecallPct = sellRecallPct; logNeutralRecallPct = neutralRecallPct; m_lastBuyRecallPct = buyRecallPct; m_lastSellRecallPct = sellRecallPct; // Predicted-rate (share of ALL OOS bars this era the model called this class, regardless // of whether that call was right) and precision (of just those calls, how many were // right) - see logBuyPredPct's declaration comment above for why this is worth logging // alongside recall. Denominator is the per-era OOS bar count (sum of the per-class true // totals, all tallied in the same pass-3 block and reset together each era) - NOT // m_oosSamples, which only resets on a full model reset and so accumulates across every // era of the run: dividing this era's calls by that all-run total diluted the logged // rate by roughly the era number (observed: era-15 "Buy:2%" that was really ~30%), // making a genuinely directional model read as a nearly-dead output. int oosEraBars = m_oosBuyTotal + m_oosSellTotal + m_oosNeutralTotal; logBuyPredPct = (oosEraBars > 0) ? (int)MathRound(100.0 * m_oosBuyPredicted / oosEraBars) : -1; logSellPredPct = (oosEraBars > 0) ? (int)MathRound(100.0 * m_oosSellPredicted / oosEraBars) : -1; logBuyPrecPct = (m_oosBuyPredicted > 0) ? (int)MathRound(100.0 * m_oosBuyPredictedHits / m_oosBuyPredicted) : -1; logSellPrecPct = (m_oosSellPredicted > 0) ? (int)MathRound(100.0 * m_oosSellPredictedHits / m_oosSellPredicted) : -1; //--- Live-fired precision (what actually trades - see m_oosBuyFired): of the directional //--- calls that cleared the confidence floor under the live/prior-corrected rule this era, //--- how many were right. Cached for the panel/log; -1 = the model fired none this era. logBuyFiredPrecPct = (m_oosBuyFired > 0) ? (int)MathRound(100.0 * m_oosBuyFiredHits / m_oosBuyFired) : -1; logSellFiredPrecPct = (m_oosSellFired > 0) ? (int)MathRound(100.0 * m_oosSellFiredHits / m_oosSellFired) : -1; m_lastBuyFiredPrecPct = logBuyFiredPrecPct; m_lastSellFiredPrecPct = logSellFiredPrecPct; m_lastBuyFired = m_oosBuyFired; m_lastSellFired = m_oosSellFired; //--- SELECTION METRIC. Ranking moved off balanced accuracy (macro-recall) 2026-07-30 //--- because that metric is maximized by exactly the model this system must never deploy. //--- Measured frontier at a fixed signal strength, base rate 6.1%: //--- tau 0.00 -> calls 0.2% of bars at 27.3% precision, balanced 34.0% //--- tau 0.35 -> calls 2.0% of bars at 15.5% precision, balanced 36.3% //--- tau 1.00 -> calls 49.6% of bars at 6.4% precision, balanced 53.5% //--- Balanced accuracy rises monotonically as the model calls MORE and is right LESS, //--- because two of its three terms are directional recalls that a call-everything model //--- drives to ~95% while the Neutral term it sacrifices counts for only a third. The //--- 2026-07-29 run landed exactly there: balanced 58-64% while calling a direction on //--- ~100% of bars at a 5-7% win rate against a ~6% base rate - no information at all. //--- Only the per-class recall floor stopped those from deploying, i.e. a guard was doing //--- the job the objective should have been doing, and the same guard also rejected the //--- genuinely useful sparse-but-precise checkpoints (directional recall 4-6%). //--- Ranking is now DIRECTIONAL PRECISION - of the bars this model called Buy or Sell, //--- how many were right - which is what a trading edge actually is. Two anti-degenerate //--- floors bracket it, because precision alone is trivially maximized by calling almost //--- nothing: coverage must reach a fraction of the true base rate, and precision must at //--- least beat that base rate (a model no better than the coin is not an edge). //--- THE POPULATION THAT ACTUALLY TRADES (m_oosBuyFired - the calls surviving the //--- confidence threshold), not the raw argmax (m_oosBuyPredicted). Those two were the //--- same set for as long as the threshold sat near zero, so the distinction cost nothing //--- and the gate read the argmax. The held-out calibration slice ended that: thresholds //--- moved from ~0.02 to 0.14-0.40, and on 2026-08-10 PAI era 256 the argmax population //--- was 100% of bars while the traded population was 21% - so the gate was certifying a //--- trade-every-bar strategy that the EA does not run. AdjustedSignalFromSoftmax() gates //--- the live order, the arrow and the panel; it has to gate the deployment decision too. //--- This is the 9a7c37f defect class (gate certifies one thing, execution does another), //--- and the NMS block below carried a comment warning about it while committing it. //--- //--- The threshold can only turn a direction into Neutral, never flip Buy to Sell, so the //--- fired set is a strict subset of the argmax set and every per-bar outcome (oWinLong/ //--- oWinShort) is the one already computed for that bar. //--- //--- Deliberately NOT changed alongside: the recall figures and logBuyPrecPct, which stay //--- on the raw argmax. Those measure the model's intrinsic class separation - a //--- diagnostic of whether it is learning at all - and thresholding them would conflate //--- "cannot separate the classes" with "declines to act on the separation it found". int oosDirCalls = m_oosBuyFired + m_oosSellFired; //--- WINS, not label agreement - see m_oosBuyPredictedWins for the full argument. The //--- label-agreement figure is still computed and still logged (logBuyPrecPct/ //--- logSellPrecPct), because it is the right diagnostic for class separation; it is just //--- not the right thing to gate a DEPLOYMENT on, which is a question about money. int oosDirHits = m_oosBuyFiredHits + m_oosSellFiredHits; int oosDirTrue = m_oosBuyTotal + m_oosSellTotal; bool coverageMeasurable = (oosEraBars > 0 && oosDirTrue > 0); double coveragePct = coverageMeasurable ? 100.0 * oosDirCalls / oosEraBars : -1.0; double baseRatePct = coverageMeasurable ? 100.0 * oosDirTrue / oosEraBars : -1.0; double dirPrecPct = (oosDirCalls > 0) ? 100.0 * oosDirHits / oosDirCalls : -1.0; double minCoveragePct = coverageMeasurable ? baseRatePct * MIN_COVERAGE_FRACTION_OF_BASE_RATE : -1.0; //--- ZERO-SKILL PRECISION: what a model with no information scores on this metric, by //--- always calling whichever direction is more common. Its precision is that class's //--- share of ALL bars, because the bars it calls are uncorrelated with the labels. //--- This REPLACED `dirPrecPct >= baseRatePct` on 2026-08-01, which was wrong the moment //--- the labels stopped being rare. baseRatePct is Buy+Sell as a share of all bars: at the //--- old exact-pivot target that was ~6%, so "beat the base rate" read as "beat chance" //--- and the test looked sound. Triple-barrier labels put it at ~83%, and the gate then //--- demanded 83% directional precision - unreachable by construction, so NOTHING could //--- ever deploy. Observed live: all four topologies cycling "PLATEAU stage 3 ... nothing //--- safe to deploy" at a genuinely healthy 43-45% precision. //--- max(Buy,Sell) is the right benchmark at ANY base rate: it is exactly the score of the //--- degenerate always-call-one-direction model this floor exists to reject, and it //--- degrades correctly to ~3% on the old rare-pivot labels. //--- //--- COUNTED IN WINS since 2026-08-09, matching dirPrecPct above. The always-Buy model is //--- scored the way the real model now is: how often its trade PAID, which is //--- m_oosWinLongTotal / all scored bars - not how often the collapsed label happened to //--- read Buy. Those diverged the moment the measured geometry put the target nearer than //--- the stop: label-Buy was 37.5% of bars while a long actually won on ~67% of them, so //--- the gate was benchmarking a win rate against a label frequency and clearing models //--- 30pp short of break-even. Now chance and break-even coincide again by construction - //--- an always-long model wins m/(m+k), which IS the break-even rate for a k:m trade - so //--- clearing this reference by EDGE_MIN_SIGMAS means positive expectancy and nothing else. double chancePrecPct = coverageMeasurable ? 100.0 * MathMax(m_oosWinLongTotal, m_oosWinShortTotal) / oosEraBars : -1.0; logCoveragePct = (int)MathRound(coveragePct); logDirPrecPct = (int)MathRound(dirPrecPct); logChancePrecPct = (chancePrecPct >= 0.0) ? (int)MathRound(chancePrecPct) : -1; //--- Deployability. Replaces the per-class recall floor as the gate the checkpoint //--- selection and the plateau ladder's "is there anything safe to deploy" test read. //--- MinRecall still drives the diagnostic recall line below, but no longer decides what //--- ships - it is the input that produced the catch-22 where nothing ever qualified. //--- The margin is not arbitrary and not a knob: beating chance by any amount at all is a //--- coin-flip result once the estimate's own sampling error is accounted for. With //--- oosDirCalls directional calls at a chance rate p, the standard error of the measured //--- precision is sqrt(p(1-p)/n) - about 0.4pp at the ~11,000 calls these runs produce - so //--- `dirPrecPct > chancePrecPct` was passing models whose entire "edge" was under one //--- sigma. Observed 2026-08-01: the perceptron deployed at edge +0pp. //--- Requiring EDGE_MIN_SIGMAS standard errors instead scales the bar with the evidence: //--- a sparse model needs a bigger measured edge to qualify than a dense one, which is //--- exactly right, and no constant has to be re-tuned when coverage changes. //--- ON THE EFFECTIVE SAMPLE (2026-08-17). The comment above quotes "about 0.4pp at the //--- ~11,000 calls these runs produce" - that figure assumed 11,000 INDEPENDENT calls. //--- They are triple-barrier outcomes on consecutive bars, overlapping by the label's mean //--- lifespan, so the real error is larger by ~sqrt(L) - at the 384-bar horizon this run //--- shipped, an order of magnitude larger. Every "clears by N sigma" verdict in the //--- project's history was computed against the optimistic figure. See //--- EffectiveSampleSize() for the correction and the evidence that forced it. double chanceP = (chancePrecPct >= 0.0) ? chancePrecPct / 100.0 : 0.0; double precSE = (oosDirCalls > 0 && chanceP > 0.0 && chanceP < 1.0) ? 100.0 * MathSqrt(chanceP * (1.0 - chanceP) / EffectiveSampleSize((double)oosDirCalls)) : 0.0; double edgeFloorPct = chancePrecPct + EDGE_MIN_SIGMAS * precSE; //--- PUBLISHED so the era line can state the bar instead of leaving it implicit. A gate //--- that is arithmetically unreachable must SAY so: with a 4,738-bar OOS window and a //--- 75.6-bar mean label lifespan there are only ~63 independent observations in it, and //--- at 18% coverage that is ~11 - which puts the required win rate near 66% against a 37% //--- chance rate. Nothing will ever clear that, and until this line printed it the //--- symptom was indistinguishable from "the models are close but not quite". m_lastEdgeFloorPct = edgeFloorPct; m_lastPrecSE = precSE; m_lastEffN = EffectiveSampleSize((double)oosDirCalls); //--- CONTRIBUTE THIS ERA'S EVIDENCE TO THE CROSS-INSTRUMENT POOL, then read the pool //--- back. Publishing unconditionally - not only when the local gate passes - because a //--- symbol that is short of its own bar is still evidence about whether the STRATEGY //--- has an edge, and a pool that only hears from winners is a selection effect, not a //--- meta-analysis. See PooledGate.mqh. if(coverageMeasurable && dirPrecPct >= 0.0 && chancePrecPct > 0.0) { PublishPoolRecord(chancePrecPct, dirPrecPct, m_lastEffN); m_lastPoolPasses = PooledGatePasses(m_lastPoolReport); } //--- BOTH sides must still be alive - see DEPLOY_MIN_SIDE_RECALL_PCT. A negative recall //--- means "not measurable this era" (no true bars of that class in the OOS window), and //--- that must not be read as a dead side, so it passes. bool bothSidesLive = (buyRecallPct < 0 || buyRecallPct >= DEPLOY_MIN_SIDE_RECALL_PCT) && (sellRecallPct < 0 || sellRecallPct >= DEPLOY_MIN_SIDE_RECALL_PCT); //--- Folded into tradeableOK rather than checked only at deploy time, deliberately: this //--- flag is also the lexicographic ranking key (isBetterEra) and the eta-decay trigger, //--- so a one-sided era must not be allowed to become the best-so-far in the first place. //--- Checking it only at the deploy gate would let the ladder spend its whole patience //--- budget ranking one-sided eras against each other and then refuse to ship the winner. bool tradeableOK = coverageMeasurable && dirPrecPct >= 0.0 && bothSidesLive && coveragePct >= minCoveragePct && dirPrecPct > edgeFloorPct; //--- Ranking key: precision, DISCOUNTED by how far short of the coverage floor the era //--- fell. Raw precision was wrong here and the 2026-07-30 run caught it within 8 eras - //--- HYBRID made exactly ONE directional call, got it right, scored 100%, and locked that //--- in as best-ever. Nothing can beat 100%, so the checkpoint was frozen on a single //--- sample and the run could only burn to the era cap. The coverage floor was already //--- computed and already blocked that era from being DEPLOYABLE, but the ranking ignored //--- it whenever no era had qualified yet - which is exactly the phase this matters in. //--- Discounting rather than thresholding keeps the ordering continuous: an era at half //--- the floor scores half its precision, so more coverage and better precision both //--- improve rank and neither can be traded away entirely. Above the floor the credit //--- saturates at 1.0, so ranking among genuinely deployable eras stays pure precision. double coverageCredit = 1.0; if(minCoveragePct > 0.0 && coveragePct >= 0.0) coverageCredit = MathMin(1.0, coveragePct / minCoveragePct); double selectionScore = (dirPrecPct >= 0.0) ? dirPrecPct * coverageCredit : 0.0; //--- THE S2 REPORT (Meta_Labeling_Design.md): the meta head's era verdict in the //--- design's own terms - coverage x (win rate - break-even) against the no-skill null. //--- Two references on purpose: chancePrecPct (the base win rate + its SE) answers //--- "does the head KNOW anything", the geometric break-even answers "would trading its //--- calls MAKE anything" - a head can clear the first and still sit under the second //--- when the candidate stream itself is unprofitable (the honest-floor case). if(IsMetaTarget() && coverageMeasurable && oosEraBars > 0) { double mSl, mTp; BarrierMultiples(mSl, mTp); double mBePct = (mSl + mTp > 0.0) ? 100.0 * mSl / (mSl + mTp) : 50.0; double mScore = (dirPrecPct >= 0.0 && coveragePct >= 0.0) ? coveragePct * (dirPrecPct - mBePct) / 100.0 : 0.0; PrintFormat("%s: META era %d - %d candidates OOS, base win %.1f%% | trades %d (%.1f%%" " coverage) at %.1f%% win vs %.1f%% break-even -> cov x (p-BE) = %+.2f |" " skill vs base %+.1fpp (needs > %+.1fpp at %d sigma) %s", ID, (int)m_eraCount, oosEraBars, chancePrecPct, oosDirCalls, coveragePct, dirPrecPct, mBePct, mScore, dirPrecPct - chancePrecPct, EDGE_MIN_SIGMAS * precSE, (int)EDGE_MIN_SIGMAS, tradeableOK ? "| DEPLOYABLE this era" : ""); //--- The decomposition the aggregate can hide (see the member declaration): each //--- cell reads "traded/candidates base->traded win rate". A cell whose traded win //--- clears mBePct on real volume is a deployable SUBSET even when the blend is not; //--- judge it against the family-wise rule before believing it (32 cells is a //--- best-of-N search by construction). string famLine = ""; for(int mf = 0; mf < 4; mf++) { double fb = (m_metaFamCand[mf] > 0) ? 100.0 * m_metaFamWins[mf] / m_metaFamCand[mf] : 0.0; double fw = (m_metaFamFired[mf] > 0) ? 100.0 * m_metaFamFiredWins[mf] / m_metaFamFired[mf] : 0.0; famLine += StringFormat("%s %d/%d %.1f->%.1f%% ", MetaFamilyName(mf), m_metaFamFired[mf], m_metaFamCand[mf], fb, fw); } double lb = (m_metaSideCand[0] > 0) ? 100.0 * m_metaSideWins[0] / m_metaSideCand[0] : 0.0; double lw = (m_metaSideFired[0] > 0) ? 100.0 * m_metaSideFiredWins[0] / m_metaSideFired[0] : 0.0; double sb = (m_metaSideCand[1] > 0) ? 100.0 * m_metaSideWins[1] / m_metaSideCand[1] : 0.0; double sw = (m_metaSideFired[1] > 0) ? 100.0 * m_metaSideFiredWins[1] / m_metaSideFired[1] : 0.0; PrintFormat("%s: META breakdown (traded/cands base->traded win, BE %.1f%%): %s|" " LONG %d/%d %.1f->%.1f%% SHORT %d/%d %.1f->%.1f%%", ID, mBePct, famLine, m_metaSideFired[0], m_metaSideCand[0], lb, lw, m_metaSideFired[1], m_metaSideCand[1], sb, sw); } //--- NEUTRAL CANNOT BLOCK WHEN IT IS TOO RARE TO LEARN. The floor exists to stop a //--- one-class collapse, and for that only the DIRECTIONAL floors are load-bearing: a //--- model that called everything Neutral would show Buy and Sell recall at 0% and be //--- blocked by them. Neutral's own floor was protecting against the mirror bias //--- (over-calling Buy/Sell at Neutral's expense) - which was a real risk when Neutral //--- was the ~94% majority under exact-pivot labels, and stopped being one when //--- first-touch resolution reduced it to a 0.65% same-bar-tie residue. At that //--- prevalence, almost never calling Neutral is CORRECT rather than biased, so the //--- floor was demanding the model be wrong in a specific way before it could converge. //--- Prevalence-guarded rather than hardcoded off, so it comes back by itself if a //--- future label rule makes Neutral substantial again. //--- Deliberately NOT extended to Buy/Sell: exempting a thin directional class would //--- reopen the era-44-46 hole (converging on a window with no directional bars to //--- disprove the model), which directionalRecallMeasured below only half-covers - it //--- checks those classes were MEASURED, not that they passed. int neutralGatePct = neutralRecallPct; if(oosEraBars > 0 && (100.0 * m_oosNeutralTotal / oosEraBars) < MIN_GATE_CLASS_SHARE_PCT) neutralGatePct = -1; //--- DERIVED, per class, from that class's own effective sample - see //--- CollapseRecallFloorPct() for why it sits BELOW chance rather than above it, and for //--- the two occasions a fixed constant here made convergence structurally impossible. double buyFloor = CollapseRecallFloorPct(m_oosBuyTotal); double sellFloor = CollapseRecallFloorPct(m_oosSellTotal); double neutralFloor = CollapseRecallFloorPct(m_oosNeutralTotal); m_lastRecallFloorPct = (buyFloor + sellFloor + neutralFloor) / 3.0; bool directionalRecallOK = (buyRecallPct < 0 || buyRecallPct >= buyFloor) && (sellRecallPct < 0 || sellRecallPct >= sellFloor) && (neutralGatePct < 0 || neutralGatePct >= neutralFloor); // Balanced accuracy (macro-recall): the mean of the three per-class recalls - the metric // the checkpoint SELECTION ranks on (see m_bestBalancedOos). Unlike blended accuracy it // weights Buy, Sell and Neutral equally, so it can't be inflated by the ~96%-Neutral base // rate. Computed from the same raw per-era recalls the floor uses (not smoothed - the // whole recall-driven side of this block is per-era-raw by design). When a directional // class is thin/unmeasured this era (recall -1), balanced accuracy isn't meaningful, so // fall back to the blended dOosForecast for ranking that era (prior behavior) rather than // averaging a partial set - the recall floor + directionalRecallMeasured still guard the // actual convergence decision separately. double balancedOosEra = (buyRecallPct >= 0 && sellRecallPct >= 0 && neutralRecallPct >= 0) ? (buyRecallPct + sellRecallPct + neutralRecallPct) / 3.0 : dOosForecast; logBalancedAccPct = (buyRecallPct >= 0 && sellRecallPct >= 0 && neutralRecallPct >= 0) ? (int)MathRound(balancedOosEra) : -1; // A real (non-thin-sample, i.e. not the -1 "n/a" sentinel) 0% recall on any class means // the model never once got that class right this era - a majority-class collapse // (predict-everything-Neutral, or symmetrically a Buy/Sell-only collapse), not progress // toward separating classes. Before any era has ever passed the recall floor, // isBetterEra's fallback below is a pure blended-accuracy tiebreak, and blended accuracy // is trivially maximized by collapsing to the majority class. Observed in practice // (2026-07-19, SP500 H4): once a run landed on a 0%/0%/100% Buy/Sell/Neutral era, its // accuracy kept creeping upward for 124 STRAIGHT eras purely from sharpening the // Neutral-vs-everything boundary - each tick registered as a "new best", re-anchoring the // checkpoint AND bumping eta back toward its ceiling (the recovery bump below), actively // rewarding the collapse instead of remaining neutral to it. Excluding these eras from // isBetterEra denies them that anchor/reward without touching the restore/decay branch // below, which stays exactly as gated on m_bestPassedRecall as before - see that block's // own comment for why loosening THAT part pre-pass caused a worse failure historically. //--- Under precision ranking the degenerate era is the one that called NOTHING //--- directional (precision undefined, nothing to trade), not one whose per-class //--- recall touched zero - a sparse high-precision model legitimately has low recall. bool isFullyCollapsedEra = (oosDirCalls <= 0); //--- N for the family-wise deployment gate. Counted here, next to the exclusion it mirrors: //--- an era that called nothing directional can never become the best (isBetterEra excludes //--- it), so counting it would inflate N and make the gate stricter than the search that //--- actually happened. Every era that COULD have won is counted, whether it did or not - //--- that is precisely the set the maximum was taken over. if(coverageMeasurable && !isFullyCollapsedEra) m_deployCandidateEras++; // Lexicographic "better than the best-so-far" ordering: passing the directional recall // floor always outranks not passing it, regardless of blended dOosForecast; only WITHIN // the same pass/fail category does blended accuracy break the tie. Without this, an era // that traded a few "safe" Neutral calls for genuinely useful (recall-improving) Buy/Sell // calls would look like a regression in blended-accuracy-only terms and get its // checkpoint skipped / learning rate cut - fighting directly against the network learning // to call Buy/Sell at all, since Neutral is the large majority class (~80%+ of labels) and // a model that just calls everything Neutral already scores well on blended accuracy // alone. isWorseEra mirrors the same ordering for the eta-decay-on-regression trigger. // (directionalRecallOK implies !isFullyCollapsedEra already, since the floor is always // >0%, so the first clause below needs no extra guard - only the pre-pass accuracy-only // tiebreak in the second clause does.) // Within the same recall-pass category the tie now breaks on BALANCED accuracy, not the // Neutral-dominated blended dOosForecast - see m_bestBalancedOos. This is what deploys the // most class-balanced era instead of the most Neutral-leaning one, and it also strengthens // the pre-pass phase: a Neutral-only era scores (0+0+N)/3 in balanced terms (low) rather // than the ~80% it scores in blended terms, so it can no longer re-anchor the checkpoint. //--- tradeableOK / selectionScore, not directionalRecallOK / balancedOosEra - see the //--- SELECTION METRIC note above. The lexicographic shape is unchanged: qualifying //--- always outranks not qualifying, and the score only breaks ties within a category. //--- //--- THREE tiers since 2026-08-09, with bothSidesLive in the middle: (deployable) > //--- (two-sided) > (score). Forced by a measured failure, not symmetry: HYBRID's era 29 //--- collapsed to always-Buy and scored 67.1% - EXACTLY chance, because under win-based //--- scoring the degenerate always-call-the-drift-side model IS the chance reference - //--- while every honest two-sided era scored 63-66% (shorts win less often against //--- SP500's drift). Raw score ranking crowned it, every regression restored it, and NMS //--- collapsed its constant signal to ~25 trades/era. One-sidedness already blocked //--- DEPLOYMENT (tradeableOK), but among not-yet-deployable eras score alone decided. //--- A one-sided era now cannot displace a two-sided best NO MATTER its score, and a //--- two-sided era displaces a one-sided best no matter how much lower it scores - by //--- construction the one-sided score is a property of the DATA's drift, not the model. bool isBetterEra = (tradeableOK && !m_bestPassedRecall) || (tradeableOK == m_bestPassedRecall && bothSidesLive && !m_bestBothSidesLive) || (tradeableOK == m_bestPassedRecall && bothSidesLive == m_bestBothSidesLive && !isFullyCollapsedEra && selectionScore > m_bestBalancedOos); // The recall-pass-loss clause used to fire on ANY drop out of a full 3-way recall pass, // even a near-miss on one class at unchanged accuracy (e.g. observed: Buy:56% Sell:41% // Neutral:34% - Neutral alone missing the 40% floor by a few points) - treating that // identically to a total collapse back to Neutral-only. With three classes all needing // to simultaneously clear the floor, that made isWorseEra fire on most eras once a pass // was ever achieved, ratcheting eta toward ETA_MIN within a handful of eras and then // (before the recovery bump below existed) leaving it stuck there permanently - visible // in practice as ~25 back-to-back identical "regressed from best 70.6% to 70.6%" eras. // Now only counts as worse if accuracy ALSO dropped meaningfully (same threshold // regardless of whether the recall-pass flag changed too) - losing the recall-pass flag // at flat/improved accuracy is borderline variance, not a regression worth // restoring+decaying over. bool isWorseEra = selectionScore < m_bestBalancedOos - ETA_DECAY_REGRESSION_PCT; //--- ENSEMBLE: ranking and checkpointing belong to the ensemble as a unit (see //--- EnsembleCommitJointCheckpoint). A member's own best era is NOT the deployable one - //--- committing it here would overwrite the joint checkpoint with a quartet no combined //--- measurement ever covered, which is the exact failure the ensemble gate exists to //--- prevent. The eta recovery bump still applies: that is this net's own learning-rate //--- dynamics, not a deployment decision. if(isBetterEra && m_ensembleMember) eta = MathMin(m_etaCeiling, eta / ETA_DECAY_FACTOR); if(isBetterEra && !m_ensembleMember) { //--- Snapshot BOTH scores at the checkpoint: m_bestBalancedOos is what ranking compares //--- against next era; m_bestOosForecast keeps the blended value FinalizeTrainRun() and //--- the restore branch reset dOosForecast to (see m_bestBalancedOos' declaration). m_bestOosForecast = dOosForecast; m_bestBalancedOos = selectionScore; m_bestPassedRecall = tradeableOK; m_bestBothSidesLive = bothSidesLive; //--- Raw significance inputs for the family-wise gate, taken at the same instant as the //--- weight snapshot below so the test always describes the weights that would ship. //--- selectionScore cannot substitute: it is precision x coverage credit, and the test //--- needs the unweighted precision plus the n that sets its standard error. m_bestDirPrecPct = dirPrecPct; m_bestChancePrecPct = chancePrecPct; m_bestDirCalls = oosDirCalls; //--- The operating point is part of the model, not of the run: these OOS numbers were //--- produced by these weights UNDER this threshold, and restoring one without the //--- other would deploy a model whose coverage and precision are not the ones the gate //--- cleared. Captured at the same instant as the weight snapshot below. m_bestDirConfThreshold = m_dirConfThreshold; //--- eval candidates are throwaway - track the score (above) but never write a checkpoint //--- file; m_haveOosCheckpoint=false then also skips the worse-era RestoreWeights() restore. //--- In-MEMORY weight snapshot (not a file): the file-based checkpoint re-created every //--- neuron on restore, which the CPU-DLL backend can't do for a second live set - see //--- CNet::CaptureWeights/RestoreWeights. eval candidates snapshot nothing. m_haveOosCheckpoint = Net.CaptureWeights(); // Recovery bump: ETA_DECAY_FACTOR-only ever shrinks eta, and previously nothing ever // grew it back - a losing streak early in a run (even a since-corrected one) would // permanently cap how fast every later era could learn for the rest of the run, all // the way down to ETA_MIN with no way back. A genuinely better era (new best, not // just a tie) means the current eta is working, so nudge it back up a bit - capped at // this model's own configured starting rate (m_etaCeiling - AdamLearningRate for // ADAM, SgdLearningRate for SGD, see that member's declaration comment) so this // can't runaway past the rate training was actually tuned to start at. eta = MathMin(m_etaCeiling, eta / ETA_DECAY_FACTOR); } else if(isWorseEra && m_bestOosForecast > 0) { // Decaying eta alone only softens FUTURE steps - it does nothing to undo the // regression this era already baked into the weights, so a run could (and in // practice did) spend 15+ eras compounding forward from one bad era's damage, // each new era fighting the last one's overshoot instead of building on the best // state found so far. Restore the last checkpointed-good weights before continuing // (mirrors what FinalizeTrainRun() does at the END of a run, just applied live so // the oscillation can't compound within a single run) - this is what actually turns // "reduce LR on regression" into "step back, then retry slower", not just "drift // slower". // // BOTH the restore AND the eta decay below are gated on m_bestPassedRecall: before // ANY era has ever cleared the per-class recall floor, isBetterEra's own // lexicographic ordering degrades to a pure blended-accuracy tiebreak // (directionalRecallOK==false on both sides of the comparison), so "best checkpoint" // during that phase just means "called Neutral most confidently so far" - restoring // it would actively defend the majority-class collapse against any era that trades // some accuracy for real Buy/Sell recall, which is exactly the bias this whole // recall-gate mechanism exists to prevent (see isBetterEra's own comment above). // Observed in practice: era 1-3 all "improved" on accuracy alone // (24.9%->41.4%->52.3%) while Buy/Sell recall stayed at a flat 0% the entire time - // restoring pre-pass would have locked training into that trajectory instead of // letting it explore past it. Decaying eta has the same bias one step removed: // every regression relative to a Neutral-collapse "best" shrinks eta a little more, // steadily strangling the exploration needed to escape that collapse until eta // bottoms out at ETA_MIN with no real solution ever found and no checkpoint to fall // back on either - observed in practice as a run whose best-ever blended accuracy // kept landing on 0%/0%/100% Buy/Sell/Neutral recall eras, each one triggering // another decay on the very next era, until eta floored out around era 20 and the // remaining eras just oscillated between collapse states with no way to make a // large-enough move to escape and no way to reset. Once m_bestPassedRecall is true, // there IS a genuinely good state worth protecting, and both restoring the // checkpoint and decaying eta on regression are safe/correct again. // 2026-07-29: the m_bestPassedRecall gate above has an escape now, because its // stated premise expired. It was written when the pre-pass tiebreak really was // blended-accuracy-only; the balanced-selection change (m_bestBalancedOos) replaced // that with `balancedOosEra > m_bestBalancedOos` AND an isFullyCollapsedEra // exclusion, so a Neutral-only era now scores ~33% (the FLOOR of the balanced // metric) and cannot anchor the checkpoint at all. "Best checkpoint" pre-pass // therefore no longer means "called Neutral most confidently" - it means "most // class-balanced state found so far", which is worth defending, and isWorseEra is // itself a balanced-accuracy regression, so it cannot fire merely for trading // Neutral calls for Buy/Sell. // // Leaving the gate absolute had a failure mode of its own, and it is not // hypothetical: if NO checkpoint ever clears the recall floor, m_bestPassedRecall // stays false forever, so there is never any restore and never any eta decay. // Observed on SP500 H1 2026-07-29 across three topologies - CONV ran 228 eras with // eta pinned at its 0.000300 start while balanced accuracy slid 40% -> 35% and Buy // recall 11% -> 2%. The run had no regression control whatsoever, and the plateau // ladder could not end it either (stage 3 refuses to deploy without a recall pass), // so it was a 1000-era one-way trip into a Neutral collapse. // // The original concern still applies while the best-so-far IS near-collapse: // decaying eta against such a "best" strangles the exploration needed to escape it. // So the escape is margin-guarded - defend the checkpoint only once it sits clearly // above the one-class floor, which is exactly when there is something real to lose. bool bestWorthDefending = (m_bestBalancedOos > BALANCED_COLLAPSE_PCT + BALANCED_WORTH_DEFENDING_MARGIN_PCT); //--- PATIENCE (see ETA_DECAY_PATIENCE_ERAS). Restoring the checkpoint AND cutting //--- eta on the FIRST regressing era makes the next era start from an identical //--- state with a smaller step - so it regresses again, and the response to that is //--- another restore and another cut. The loop is self-sustaining and cannot //--- discover anything, because rolling the weights back is precisely what removes //--- the exploration that would end it. Wait for several consecutive regressions //--- before concluding the step is too big; a single bad era is noise. m_consecutiveRegressions++; if((m_bestPassedRecall || bestWorthDefending) && m_consecutiveRegressions >= ETA_DECAY_PATIENCE_ERAS) { m_consecutiveRegressions = 0; if(m_haveOosCheckpoint && Net.RestoreWeights()) { dOosForecast = m_bestOosForecast; //--- The operating point goes back with the weights it was fitted for. //--- Leaving the current one in place would pair restored weights with a //--- threshold chosen for the rejected ones - see m_bestDirConfThreshold. m_dirConfThreshold = m_bestDirConfThreshold; //--- 2026-08-09 audit, F3: the snapshot restores WEIGHTS only, so without //--- this the Adam moments still encode the just-rejected trajectory and //--- the first updates after the restore push straight back toward the //--- state that was rolled back - the restore -> regress-again -> restore //--- oscillation. A restore is a new starting point; it gets a fresh //--- optimizer. Net.ResetOptimizerState(); } if(eta > ETA_MIN) eta = MathMax(ETA_MIN, eta * ETA_DECAY_FACTOR); Print(ID + ": OOS selection score (coverage-weighted dir-precision) regressed from best " + DoubleToString(m_bestBalancedOos, 1) + "% to " + DoubleToString(selectionScore, 1) + "% (blended " + DoubleToString(m_bestOosForecast, 1) + "%->" + DoubleToString(dOosForecast, 1) + "%) - restoring best checkpoint and decaying learning rate to " + DoubleToString(eta, 6)); } else Print(ID + ": OOS selection score (coverage-weighted dir-precision) regressed from best " + DoubleToString(m_bestBalancedOos, 1) + "% to " + DoubleToString(selectionScore, 1) + "% (blended " + DoubleToString(m_bestOosForecast, 1) + "%->" + DoubleToString(dOosForecast, 1) + "%) - best so far is still within " + DoubleToString(BALANCED_WORTH_DEFENDING_MARGIN_PCT, 1) + "pp of the " + DoubleToString(BALANCED_COLLAPSE_PCT, 1) + "% one-class floor, so there is nothing worth" + " restoring yet - continuing to explore without decaying eta (still " + DoubleToString(eta, 6) + ")"); } //=== IN-SAMPLE ERROR PLATEAU: THE HONEST EARLY STOP ==================================== //--- The ladder below stops on the OOS SELECTION score. That is a peek: the run has by //--- then evaluated every one of those eras out of sample, so all of them are in the //--- family the deploy gate must correct over (g_ensCandidateEras, Sidak) - stopping late //--- does not just cost compute, it RAISES the bar the winner has to clear. //--- This stop reads the TRAINING error instead, which the gate never looks at. When the //--- optimiser has stopped making progress on the data it can see, more eras are not //--- going to find a better model - they only enlarge the OOS family. So ending here //--- shrinks the correction rather than inflating it, and the shrinkage is legitimate //--- precisely BECAUSE the stopping rule never consulted an out-of-sample number. //--- The distinction matters and it is the one this project has got wrong four times: //--- stop on IS -> the family really is smaller; stop on OOS -> those eras were searched //--- and still count. Both stops exist; only this one buys a lower bar. if(dError >= 0.0 && MathIsValidNumber(dError)) { //--- Relative improvement, so this does not depend on the loss's absolute scale. if(m_bestIsError < 0.0 || dError < m_bestIsError * (1.0 - IS_ERROR_IMPROVE_FRAC)) { m_bestIsError = dError; m_erasSinceBestIsError = 0; } else { m_erasSinceBestIsError++; //--- Deliberately more patient than the OOS ladder: training error is noisy per era //--- (mini-batch order alone moves it), and ending a run that is still learning is //--- far more expensive than a few wasted eras. Only acts when there is something //--- to deploy - with no checkpoint this would end the run with nothing to show. if(m_erasSinceBestIsError >= TrainPlateauPatienceEras() * IS_ERROR_PATIENCE_MULT && m_haveOosCheckpoint && !m_isErrorPlateaued) { Print(ID + ": IN-SAMPLE ERROR PLATEAU - training error has not improved by " + DoubleToString(100.0 * IS_ERROR_IMPROVE_FRAC, 1) + "% in " + IntegerToString(m_erasSinceBestIsError) + " eras (best " + DoubleToString(m_bestIsError, 4) + ", now " + DoubleToString(dError, 4) + "). The optimiser has stopped learning from the data it CAN see, so further" " eras cannot find a better model - they would only add candidates to the" " family the deploy gate corrects over, raising the bar the winner has to" " clear. Ending the search and deploying the best checkpoint. This stop" " never read an out-of-sample number, which is what makes the smaller" " family legitimate rather than a peek."); //--- LATCH FIRST, and let the LATCH - not m_plateauStage - be what the deploy //--- conditions read. m_plateauStage is mirrored from the shared ensemble ladder //--- on every era (EnsembleEraVerdict), so writing the decision there meant it //--- survived until the next verdict and no longer. See m_isErrorPlateaued. m_isErrorPlateaued = true; m_plateauStage = PLATEAU_STAGE_DEPLOY; } } } //=== PLATEAU LADDER ==================================================================== //--- Neither branch above fires in the dead zone between "new best" and "regressed by more //--- than ETA_DECAY_REGRESSION_PCT". This is the response to sitting in it: count eras since //--- the last new best and escalate. See the PLATEAU_* constants for the full rationale and //--- ENSEMBLE: one shared ladder, run by the verdict below, so the members escalate and //--- finish together instead of drifting into different stages of different searches. //--- Stash this era's figures first - the last member to arrive needs every member's, and //--- commits them if the era's VOTE wins. See the ENSEMBLE DEPLOY GATE block. if(m_ensembleMember) { EnsembleStashEraStats(dirPrecPct, chancePrecPct, oosDirCalls, tradeableOK, bothSidesLive, selectionScore, dOosForecast); //--- m_eraCount was already incremented at the top of this block, so the era that just //--- finished - the one the vote buffer is stamped with - is m_eraCount - 1. EnsembleOosPassComplete(m_eraCount - 1, eta); } else if(isBetterEra) { //--- Moving again: retire the ladder AND the restart boost. The recovery bump in the //--- checkpoint block above has already clamped eta back to at most m_etaCeiling this //--- era, and that is deliberate now that restarts overshoot the ceiling //--- (PLATEAU_RESTART_BOOST): the boost exists to kick the run OUT of a basin, and a //--- new best is the signal it worked - continuing to train at several times the //--- tuned rate FROM a state worth keeping risks destroying it. The checkpoint just //--- snapshotted this era regardless. The normal per-era eta schedule takes over. if(m_plateauStage > 0) Print(ID + ": new best selection score (coverage-weighted dir-precision) " + DoubleToString(m_bestBalancedOos, 1) + "% - plateau escape worked, clearing plateau stage " + IntegerToString(m_plateauStage)); m_erasSinceBestBalanced = 0; m_plateauStage = 0; m_restartBoostErasLeft = 0; //--- Patience is about CONSECUTIVE regressions - an era that improves clears it, so a //--- run that alternates improve/regress never accumulates its way into a decay. m_consecutiveRegressions = 0; } else { m_erasSinceBestBalanced++; int dueStage = m_erasSinceBestBalanced / TrainPlateauPatienceEras(); if(dueStage > m_plateauStage) { m_plateauStage = dueStage; string stageNote = IntegerToString(m_erasSinceBestBalanced) + " eras with no new best selection score (best " + DoubleToString(m_bestBalancedOos, 1) + "%)"; if(m_plateauStage == PLATEAU_STAGE_RESTART || m_plateauStage == PLATEAU_STAGE_ANNEAL) { //--- BOOSTED WARM RESTART: a plateau needs a bigger step to climb out of its //--- basin, not a smaller one - and "back to the ceiling" was a NO-OP whenever //--- the run plateaued without ever tripping the regression decay, because eta //--- was still AT the ceiling (2026-08-09 audit, F2). Overshoot it instead; the //--- era-end anneal below walks the rate back to the ceiling over //--- PLATEAU_PATIENCE_ERAS eras, so this is a bounded kick, not a new permanent //--- rate. See PLATEAU_RESTART_BOOST for the amplitude rationale. double etaBefore = eta; eta = m_etaCeiling * PLATEAU_RESTART_BOOST; m_restartBoostErasLeft = TrainPlateauPatienceEras(); //--- A restart is a new schedule: replaying the plateau's own accumulated Adam //--- momentum at 5x the rate would retrace the same basin, harder. Fresh moments //--- make the kick explore instead (audit F3, same mechanism as the //--- regression-restore reset). Net.ResetOptimizerState(); //--- The focal-gamma anneal that used to accompany this went with focal loss //--- on 2026-07-31. It was only ever a monotone step toward zero on a second //--- imbalance correction; the warm restart above is and always was the //--- actual escape, so both ladder stages keep their distinct patience //--- thresholds and simply retry the restart. Print(ID + ": PLATEAU stage " + IntegerToString(m_plateauStage) + " - " + stageNote + ". Boosted warm restart: learning rate " + DoubleToString(etaBefore, 6) + "->" + DoubleToString(eta, 6) + " (annealing back to " + DoubleToString(m_etaCeiling, 6) + " over " + IntegerToString(TrainPlateauPatienceEras()) + " eras), optimizer momentum reset. Best checkpoint is safe - this only changes how the NEXT eras train."); } else if(m_plateauStage >= PLATEAU_STAGE_DEPLOY) { //--- Exhausted: both escapes were tried and neither found a better model, so //--- this IS the best this configuration reaches. The deploy itself happens in //--- the era-cap/plateau branch at the TOP of the next era, which reuses the //--- proven "stop + mark complete -> FinalizeTrainRun restores and deploys the //--- best checkpoint" path rather than duplicating it here. //--- Safety: only ever auto-deploys a checkpoint that CLEARED the per-class //--- recall floor (m_bestPassedRecall). If nothing ever did, there is no model //--- worth deploying - so the ladder resets and keeps trying instead, leaving //--- the era cap as the ultimate backstop. That is what stops "train to the //--- best possible result" from degenerating into "deploy a Neutral collapse". //--- SECOND gate, and the one that matters on a long run: the checkpoint must //--- survive having been CHOSEN out of every era this run ranked. See //--- DEPLOY_FAMILY_WISE_ALPHA - the per-era floor alone opens on noise with //--- probability 1-(1-0.0228)^N, which is 93% by era 112. double zBest = 0.0, pFam = 1.0; int nTried = 0; bool survivesSelection = BestCheckpointSurvivesSelection(zBest, pFam, nTried); string selectionNote = " | best-of-" + IntegerToString(nTried) + " test: edge " + DoubleToString(m_bestDirPrecPct - m_bestChancePrecPct, 1) + "pp on " + IntegerToString(m_bestDirCalls) + " calls = " + DoubleToString(zBest, 2) + " sigma, family-wise p=" + DoubleToString(pFam, 4) + " (need <=" + DoubleToString(DEPLOY_FAMILY_WISE_ALPHA, 2) + ")"; //--- THE SCREEN HAS A VETO. Everything else in this block asks whether the //--- MODEL is good enough; this asks whether there was anything to find. If //--- neither the feature/label MI nor the normalised excursion asymmetry //--- cleared its null, no amount of fitting created directional information - //--- and every closed direction verdict in this project was reached after //--- exactly that was attempted anyway. Reported separately from the //--- statistical gate because the remedy is completely different: a failed //--- selection test says train differently, this says look somewhere else. if(!m_dirEvidence) Print(ID + ": DEPLOY REFUSED BY THE MEASUREMENT SCREEN - " + m_dirEvidenceWhy + ". Neither the feature/label mutual information nor the normalised" " excursion asymmetry cleared its permutation null on this" " configuration, so there is no measured directional information here" " for a model to have learned. The checkpoint is kept and training" " state is untouched - this is a refusal to go LIVE, not a failure." " The productive move is a different target or a different market," " not more eras: excursion SIZE keeps clearing where direction does" " not, and that is a risk-control head rather than an entry signal."); if(m_bestPassedRecall && m_haveOosCheckpoint && survivesSelection && m_dirEvidence) Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + " across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " warm restarts. Training has converged on what this" + " configuration can reach - deploying the best checkpoint (dir-precision " + DoubleToString(m_bestBalancedOos, 1) + "%, blended " + DoubleToString(m_bestOosForecast, 1) + "%)." + selectionNote + " - CLEARS."); else if(m_bestPassedRecall && m_haveOosCheckpoint) { //--- Passed the per-era floor but not the selection correction: this is a //--- maximum that a pure-noise search of this length produces routinely. //--- Same verdict the geometry scan and the indicator tuner reach on this //--- data, and for the same reason - so say so in the same language //--- instead of implying the model was merely mediocre. Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + ". The best checkpoint clears the per-era deployability floor but DOES NOT clear the" + " null of the MAXIMUM over the eras it was chosen from" + selectionNote + ". A best-of-N this large happens routinely when every era is a noise draw, so the" + " ranking carries no evidence of an edge and this model is not safe to trade." + " Restarting the plateau ladder and continuing to train; the " + IntegerToString(m_maxErasPerRun) + "-era cap remains the backstop."); m_erasSinceBestBalanced = 0; m_plateauStage = 0; } else { Print(ID + ": PLATEAU stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " - " + stageNote + ", but no checkpoint has ever cleared the deployability floor (directional calls on" + " at least a quarter as many bars as actually swing, at a precision above that base rate, with BOTH Buy and Sell" + " recall at or above " + DoubleToString(DEPLOY_MIN_SIDE_RECALL_PCT, 0) + "%), so there is nothing safe to" + " deploy. Restarting the plateau ladder and continuing to train rather than deploying a" + " one-class model; the " + IntegerToString(m_maxErasPerRun) + "-era cap remains the backstop."); m_erasSinceBestBalanced = 0; m_plateauStage = 0; } } } } //--- Restart-boost anneal (see PLATEAU_RESTART_BOOST): walk eta geometrically from //--- boost x ceiling back down to the ceiling over PLATEAU_PATIENCE_ERAS eras, one step //--- per completed era - the SGDR-style decaying half of the cycle, which is what makes //--- the boost a bounded kick instead of a new permanent rate. MathMax guards the case //--- where the regression decay already pulled eta at or below the ceiling mid-window: //--- the anneal then just expires without fighting it. A new best cleared the counter //--- above, so a successful escape never reaches here still boosted. if(m_restartBoostErasLeft > 0) { eta = MathMax(m_etaCeiling, eta * MathPow(PLATEAU_RESTART_BOOST, -1.0 / TrainPlateauPatienceEras())); m_restartBoostErasLeft--; } m_oosWindow.Add(dOosForecast); while(m_oosWindow.Total() > STABILITY_WINDOW) m_oosWindow.Delete(0); m_oosStable = false; if(m_oosWindow.Total() >= STABILITY_WINDOW) { double oosMin = m_oosWindow.At(0), oosMax = m_oosWindow.At(0); for(int w = 1; w < m_oosWindow.Total(); w++) { oosMin = MathMin(oosMin, m_oosWindow.At(w)); oosMax = MathMax(oosMax, m_oosWindow.At(w)); } m_oosStable = (oosMax - oosMin) <= STABILITY_TOLERANCE; } // The dError<0.1 RMS-error floor is meaningful for the single-neuron regression head // (m_outputNeuronsCount==1), where it's the only convergence signal available. For the // 3-neuron one-hot classification head it's redundant with, and far stricter than, // dOosForecast/directionalRecallOK: reaching RMS error 0.1 across 3 one-hot targets // needs every output neuron within ~0.17 of its target on average, i.e. near-perfect // confident calibration on EVERY bar, not just correct argmax calls - unreachable in // practice under normal market label noise, so classification runs would oscillate // forever (era after era hitting good OOS accuracy and passing recall, but never // satisfying this) without this carve-out. bool errorGateOK = (m_outputNeuronsCount == 3) ? true : (dError < 0.1); // Convergence (unlike isBetterEra's ranking) FINALIZES the model, so both directional // classes must have actually been MEASURED this era. An n/a (-1, thin-sample) Buy or // Sell recall passing directionalRecallOK is deliberate for ranking (early thin // windows shouldn't deadlock "best" tracking), but letting it pass HERE converges on // a window that contained no directional bars to disprove the model. Observed // 2026-07-19: a mid-run label-cache wipe relabeled the whole window Neutral, // "accuracy" hit 84.9% with Buy/Sell recall both n/a - without this gate a Neutral-only // model finalizes as a certified success. bool directionalRecallMeasured = (m_outputNeuronsCount != 3) || (buyRecallPct >= 0 && sellRecallPct >= 0); //--- VALIDITY of this era's model, no longer "did it hit a target accuracy". The absolute //--- OOS-accuracy target (the old MinWR input) is gone: an accuracy number typed in ahead of //--- time is either unreachable for the symbol/timeframe - in which case the run never //--- converges and burns to the era cap - or set low enough to stop a run that was still //--- improving. Neither is what "train to the best result" means. Quality is now enforced by //--- WHICH era gets deployed (balanced-accuracy checkpoint ranking + this per-class recall //--- floor) and WHEN a run ends (the plateau ladder), not by an accuracy threshold. Note the //--- recall floor deliberately stays: it is not a performance target but the anti-collapse //--- gate that makes auto-deploy safe. m_objectiveMet = errorGateOK && directionalRecallOK && directionalRecallMeasured; } // Only mark the persisted model "complete" once it actually converged this era - // an interruption (stop) or an ordinary in-progress era must stay flagged incomplete // so a restart resumes training instead of quietly treating a partial run as done. // Convergence = "the plateau ladder is exhausted AND there is a recall-passing checkpoint // to deploy" - the exact same condition the deploy branch beside the era-cap check uses, so // the flag written into the .nnw here can never disagree with the decision to stop. While a // run is still improving (or still has an escape stage left to try) this stays false and the // per-era save correctly records an in-progress run. Previously this was // (m_objectiveMet && m_oosStable), which needed the removed absolute accuracy target to mean // anything: with that target gone m_oosStable alone - just 3 eras inside a 2pp band, which // is true constantly - would have converged the run at the first flat spot. //--- ...and the family-wise selection gate, for the same reason the deploy branch applies it: //--- these two conditions MUST stay identical or the flag persisted into the .nnw disagrees //--- with the decision to stop, and a reload would run inference on a model the ladder had //--- refused to deploy. Cheap enough to re-evaluate per era (one normal-tail evaluation). double zConv = 0.0, pConv = 1.0; int nConv = 0; //--- ENSEMBLE: the verdict is the ensemble's, so the flag persisted into this member's .nnw //--- has to be the ensemble's too - otherwise a reload would run one member live against //--- three still training, which is not the model that was measured. Same //--- must-stay-identical rule as the solo pair, one level up. m_trainingComplete = m_ensembleMember ? (g_ensDeployApproved && m_haveOosCheckpoint && m_checkpointEra == g_ensBestEra) : ((m_plateauStage >= PLATEAU_STAGE_DEPLOY || m_isErrorPlateaued) && m_bestPassedRecall && m_haveOosCheckpoint && m_dirEvidence && BestCheckpointSurvivesSelection(zConv, pConv, nConv)); double currentIndicatorParams[]; m_indicatorTuner.Flatten(currentIndicatorParams); if(!Net.Save(m_activeFileName + ".nnw", dError, dUndefine, dForecast, dtStudied, m_activeFileCommon, m_eraCount, m_trainingComplete, currentIndicatorParams)) Print(__FUNCTION__ + ": ERROR - era-end Net.Save failed for " + m_activeFileName + ".nnw (era " + IntegerToString(m_eraCount) + "). Training continues but this era's checkpoint was NOT persisted - a crash/restart now would resume from an older era."); if(!SaveModelStats(m_activeFileName, m_activeFileCommon)) // keep calibration state paired with the just-saved weights Print(__FUNCTION__ + ": ERROR - SaveModelStats failed for " + m_activeFileName + " (era " + IntegerToString(m_eraCount) + "). Calibration/online-learning state not persisted this era."); SaveShadowNet(currentIndicatorParams); } } if(shouldLogProgress) { string recallInfo = (logBuyRecallPct < 0 && logSellRecallPct < 0 && logNeutralRecallPct < 0) ? "" : (" | OOS recall Buy:" + (logBuyRecallPct < 0 ? "n/a" : IntegerToString(logBuyRecallPct) + "%") + " Sell:" + (logSellRecallPct < 0 ? "n/a" : IntegerToString(logSellRecallPct) + "%") + " Neutral:" + (logNeutralRecallPct < 0 ? "n/a" : IntegerToString(logNeutralRecallPct) + "%") + //--- The floor is DERIVED per class now, so it has to be printed rather than assumed - a //--- reader comparing these recalls against a remembered "40" would be reading the wrong //--- bar. It is a COLLAPSE floor sitting below the 33.3% chance recall, not a quality bar; //--- the quality bar is the DEPLOY BAR further along this same line. StringFormat(" (collapse floor >=%.1f%% each - DERIVED from each class's effective sample," " and it sits BELOW the 33.3%% zero-skill recall on purpose: it refuses a" " COLLAPSED model, it does not certify a good one)", m_lastRecallFloorPct)); //--- Balanced accuracy = the checkpoint-selection metric (see m_bestBalancedOos). Shown so the //--- number the deployed model is actually chosen on is visible next to the recalls it averages. //--- Balanced accuracy is retained as a DIAGNOSTIC only - selection ranks on directional //--- precision now (see the SELECTION METRIC note). Both are shown so a run where they //--- disagree - the signature of an over-calling model - is visible at a glance. string balancedInfo = (logBalancedAccPct < 0) ? "" : (" | OOS balanced acc " + IntegerToString(logBalancedAccPct) + "% (diagnostic)"); //--- "win-rate", not "dir-precision": since 2026-08-09 this counts calls whose TRADE reached //--- target before stop, and the chance figure beside it is what always-long/always-short //--- collected on the same bars. The rename is not cosmetic - the old name described label //--- agreement, and reading the new number as the old one would understate the model by the //--- both-won share while overstating its edge against a benchmark that had moved. //--- "post-threshold" in the label because the population moved: this counts only the calls that //--- survive the fitted operating point, which is what the EA trades and what the deploy gate //--- now ranks. Reading it as the old whole-argmax figure would understate coverage as a //--- regression when it is the threshold doing its job. string selectionInfo = (logDirPrecPct < 0) ? " | SELECT: no directional calls survived the threshold" : (" | SELECT win-rate " + IntegerToString(logDirPrecPct) + "% on " + IntegerToString(logCoveragePct) + "% of bars (post-threshold)" + (logChancePrecPct >= 0 ? " (chance=break-even " + IntegerToString(logChancePrecPct) + "%, edge " + (logDirPrecPct - logChancePrecPct >= 0 ? "+" : "") + IntegerToString(logDirPrecPct - logChancePrecPct) + "pp)" : "")); //--- The operating point that produced the coverage figure just above it, so the two are read //--- together: coverage falling is only good news if it is this that caused it. selectionInfo += " @margin>=" + DoubleToString(m_dirConfThreshold, 2); //--- TRADED precision: the same calls after declustering, which since 2026-08-09 is exactly the //--- set that becomes positions (live NMS gates the trade, not just the arrow). Printed next to //--- the figure it corrects rather than replacing it, because the two answer different questions //--- - "how good is the model's directional call" vs "how good are the trades it would take" - //--- and the gap between them is itself the diagnostic. Compare against the SAME chance rate: //--- declustering changes which bars are called, not what a no-skill model would score on them. if(m_signalClusterWindow > 0 && m_oosNmsFired > 0) { int nmsPrec = (int)MathRound(100.0 * m_oosNmsHits / m_oosNmsFired); selectionInfo += " | TRADED (declustered) " + IntegerToString(nmsPrec) + "% on " + IntegerToString(m_oosNmsFired) + " calls" + (logChancePrecPct >= 0 ? " (edge " + (nmsPrec - logChancePrecPct >= 0 ? "+" : "") + IntegerToString(nmsPrec - logChancePrecPct) + "pp)" : ""); } // See logBuyPredPct's declaration comment for why this is worth logging alongside recall - // it's what tells apart a suppressed/dead output (predicted rate stuck at 0%) from a // miscalibrated boundary (predicted rate healthy, precision poor), which look identical from // recall alone. string predictedInfo = (logBuyPredPct < 0 && logSellPredPct < 0) ? "" : (" | OOS calls Buy:" + (logBuyPredPct < 0 ? "n/a" : IntegerToString(logBuyPredPct) + "%") + " (win rate " + (logBuyPrecPct < 0 ? "n/a" : IntegerToString(logBuyPrecPct) + "%") + ")" + " Sell:" + (logSellPredPct < 0 ? "n/a" : IntegerToString(logSellPredPct) + "%") + " (win rate " + (logSellPrecPct < 0 ? "n/a" : IntegerToString(logSellPrecPct) + "%") + ")"); //--- Live-fired precision: the number that actually predicts forward-trading performance - only //--- the directional calls that cleared the confidence floor under the live/prior-corrected rule //--- (see AdjustedSignalFromSoftmax). Count in parentheses = how many bars the model would have //--- traded this era. "0" fires = the calibration is (this era) suppressing all directional trades. string liveInfo = (m_lastBuyFired <= 0 && m_lastSellFired <= 0) ? " | live fires 0 this era" : (" | live win rate Buy:" + (logBuyFiredPrecPct < 0 ? "n/a" : IntegerToString(logBuyFiredPrecPct) + "%") + " (" + IntegerToString(m_lastBuyFired) + ")" + " Sell:" + (logSellFiredPrecPct < 0 ? "n/a" : IntegerToString(logSellFiredPrecPct) + "%") + " (" + IntegerToString(m_lastSellFired) + ")"); //--- Precision BY CONFIDENCE TIER, and cumulatively from each tier upward - the two numbers a //--- decision about Min_Vote_Open actually needs. The per-tier figure says whether confidence is //--- calibrated to correctness at all (it should rise T0->T3; if it does not, raising the floor //--- buys nothing and the finding is that the head's confidence is uninformative). The ">=Tn" //--- figure is what you would ACTUALLY get, because a floor keeps every tier at or above it, and //--- it comes with the fire count so the coverage cost of raising the floor is visible in the //--- same line. Tier weights are 25/50/75/100, so for an AI-only config the input maps straight //--- across: Min_Vote_Open 50 = ">=T1", 75 = ">=T2", 100 = ">=T3". string tierInfo = ""; int tierFiredTotal = 0; for(int ti = 0; ti < 4; ti++) tierFiredTotal += m_oosTierFired[ti]; if(tierFiredTotal > 0) { tierInfo = " | tier prec"; for(int ti = 0; ti < 4; ti++) { int cumFired = 0, cumHits = 0; for(int tj = ti; tj < 4; tj++) { cumFired += m_oosTierFired[tj]; cumHits += m_oosTierHits[tj]; } tierInfo += " T" + IntegerToString(ti) + ":" + (m_oosTierFired[ti] > 0 ? IntegerToString((int)MathRound(100.0 * m_oosTierHits[ti] / m_oosTierFired[ti])) + "%" : "n/a") + "(" + IntegerToString(m_oosTierFired[ti]) + ")" + (cumFired > 0 ? "[>=" + IntegerToString((int)MathRound(100.0 * cumHits / cumFired)) + "%/" + IntegerToString(cumFired) + "]" : ""); } } // Per-layer weight movement. Pairs with rawOutInfo below: a collapsed constant-classifier state // has two very different causes, and only this tells them apart. If every layer moves and the // output still collapses, the architecture or the objective is at fault; if one stage sits at // ~0.000% era after era while the others move, that stage is receiving no gradient and no amount // of retraining or hyperparameter work will help. See CNet::LayerLearningReport. string layerInfo = (CheckPointer(Net) == POINTER_INVALID) ? "" : (" | dW/W" + Net.LayerLearningReport()); // Raw-output saturation diagnostic - see m_oosOutMin's declaration comment. Spread ~0 with // all six min/max values pinned together = the collapsed constant-classifier state. string rawOutInfo = (m_oosOutCount <= 0) ? "" : StringFormat(" | OOS raw out B:%.3f..%.3f S:%.3f..%.3f N:%.3f..%.3f spread avg %.4f", m_oosOutMin[0], m_oosOutMax[0], m_oosOutMin[1], m_oosOutMax[1], m_oosOutMin[2], m_oosOutMax[2], m_oosOutSpreadSum / m_oosOutCount); //--- Splits a reported "Neutral" into the two events that share that label. CHOSE = the net //--- ranks Neutral highest (a class-prior problem); TIED = the top two are exactly equal and //--- the tie-break reported Neutral (a saturation problem). These need OPPOSITE fixes, and //--- until now nothing in the logs could tell them apart. 'of which B=S' is the costly subset - //--- a directional reading thrown away by float equality - and 'rail' is the saturation that //--- makes exact ties possible at all. See m_oosNeutralStrict's declaration comment. //--- THE ZERO-SKILL REFERENCE, measured on THIS era's own OOS bars. Both accumulators have //--- existed for a while and neither was ever printed, which is why a 2026-08-17 run of three //--- separate topologies all sitting at 62% read as a mysterious coincidence instead of as the //--- obvious base rate. It is not a coincidence: with the measured geometry putting the target //--- (1.70 ATR) nearer than the stop (3.07 ATR), BOTH sides win on ~24.5% of bars, so //--- winLong + winShort covers ~124% of them and a caller with no edge collects ~62% whichever //--- way it calls. Every win rate on this line has to be read against this number, not against //--- 50% and not against the label frequency - see m_oosWinLongTotal for the 30pp-short models //--- the deploy gate cleared back when it benchmarked one against the other. //--- The gate itself ranks on MAX(long, short) (see chancePrecPct); both are shown here because //--- the SPREAD between them is the directional drift, and a model that merely reproduces it //--- has found the drift, not an edge. double beFrictionless = 50.0; { double slBe = 0.0, tpBe = 0.0; BarrierMultiples(slBe, tpBe); if(slBe + tpBe > 0.0) beFrictionless = 100.0 * slBe / (slBe + tpBe); } //--- DENOMINATOR IS THE PER-ERA BAR COUNT, not m_oosSamples (fixed 2026-08-17). This is the exact //--- bug already caught and fixed for logBuyPredPct thirty lines up - "era-15 Buy:2% that was //--- really ~30%" - and it was left sitting in the one line whose entire job is to be the //--- reference every other number on this line is read against. m_oosWinLongTotal is reset every //--- era; m_oosSamples only resets on a full model reset, so it accumulates across the whole run //--- and this ratio decayed as ~1/era. It was therefore correct at era 1 and wrong everywhere //--- after: an SP500 H4 run whose true always-long rate is 37% printed 1.2% at era 33 and 0.0% //--- at era 2219, which is what made the always-short figure look like a broken field rather //--- than a diluted one. Any historical reading of this line is invalid unless it came from //--- era 1 - including the "62% zero-skill" figure quoted in the 2026-08-16 notes, which was //--- derived by hand and only ever agreed with this line at the very start of a run. int zsBars = m_oosBuyTotal + m_oosSellTotal + m_oosNeutralTotal; string zeroSkillInfo = (zsBars <= 0) ? "" : StringFormat(" | zero-skill on these bars: always-long %.1f%%, always-short %.1f%%," " coin-flip %.1f%% (the gate ranks on the LARGER of the first two; the gap" " between them IS the directional drift, and a model that only reproduces it" " has found the drift, not an edge) | break-even %.1f%% frictionless, %.1f%%" " AFTER SPREAD (%.3f*ATR)", 100.0 * (double)m_oosWinLongTotal / zsBars, 100.0 * (double)m_oosWinShortTotal / zsBars, 50.0 * ((double)m_oosWinLongTotal + (double)m_oosWinShortTotal) / zsBars, beFrictionless, CostAdjustedBreakEvenPct(), m_spreadAtr); //--- THE DEPLOY BAR, stated. Reading "edge -1pp" era after era tells you the model is short; it //--- does not tell you whether it is short by a hair or by an amount no strategy could ever //--- cover. That distinction is the difference between "keep training" and "the measurement //--- design is wrong", and it is the single most expensive thing this log could not say. string gateInfo = (m_lastEdgeFloorPct < 0.0 || m_lastEffN <= 0.0) ? "" : StringFormat(" | DEPLOY BAR %.1f%% (chance + %.0f x SE %.1fpp on %.0f INDEPENDENT calls -" " %d raw calls deflated by the %.1f-bar mean label lifespan)%s", m_lastEdgeFloorPct, EDGE_MIN_SIGMAS, m_lastPrecSE, m_lastEffN, (int)(m_oosBuyFired + m_oosSellFired), MeanLabelLifespan(), //--- A bar above 100% is not "hard", it is unreachable, and no amount of //--- training addresses it - only a bigger independent sample does. (m_lastEdgeFloorPct >= 100.0 ? " <-- UNREACHABLE: no win rate can clear this. The OOS window does not hold" " enough independent observations to certify ANY edge; widen the sample" " (more instruments / lower timeframe) or narrow the barrier." : "")); string neutralWhy = (m_oosOutCount <= 0) ? "" : StringFormat(" | Neutral CHOSE %.1f%% / TIED %.1f%% (of which B=S %d) | rail %.1f%%", 100.0 * (double)m_oosNeutralStrict / m_oosOutCount, 100.0 * (double)m_oosNeutralTie / m_oosOutCount, (int)m_oosTieBuySell, 100.0 * (double)m_oosRailBars / m_oosOutCount); //--- No "(target X%)" any more - there is no absolute accuracy target. What replaces it as the //--- progress indicator is the plateau counter: how many eras since the last new best, and how //--- close that is to ending the run (see the PLATEAU_* ladder). string plateauInfo = (m_bestBalancedOos < 0) ? "" : (" | best bal " + DoubleToString(m_bestBalancedOos, 1) + "%, " + IntegerToString(m_erasSinceBestBalanced) + " eras since (stage " + IntegerToString(m_plateauStage) + "/" + IntegerToString(PLATEAU_STAGE_DEPLOY) + ")"); //--- Lifetime IS/OOS directional accuracy. The panel now shows the out-of-sample half alone (see //--- ComputeCompoundedAccuracyLine - the in-sample figure grades the model on bars it trained on, //--- so it always reads higher than anything forward trading will deliver and does not belong on //--- a product's face). The GAP between the two is still the over-fitting read, so it survives //--- here, once per era, behind the compile-time DebuggingMode constant. string lifetimeInfo = (!DebuggingMode || (m_cumIsTotal <= 0 && m_cumOosTotal <= 0)) ? "" : (" | lifetime dir acc IS " + (m_cumIsTotal > 0 ? IntegerToString((int)MathRound(m_cumIsCorrect * 100.0 / m_cumIsTotal)) + "%" : "n/a") + " OOS " + (m_cumOosTotal > 0 ? IntegerToString((int)MathRound(m_cumOosCorrect * 100.0 / m_cumOosTotal)) + "%" : "n/a") + " over " + IntegerToString(m_cumIsTotal + m_cumOosTotal) + " calls"); //--- Wall-clock split for the era that just finished, but only when it was SLOW - a healthy //--- era stays exactly one line. Same purpose as TrainHeartbeat, for the completed case: an //--- era that took 30 minutes must say where the minutes went, or it is undiagnosable from //--- the outside (2026-08-10). //--- An era finished: the stall clock restarts from here (see m_lastEraCompleteTick). m_lastEraCompleteTick = GetTickCount(); string eraTimeInfo = ""; { double eraS = (GetTickCount() - m_eraStartTick) / 1000.0; if(eraS > 120.0) //--- The excursion head gets its OWN column. It used to fall into "other", where a 3.6x //--- era-time regression showed up as an unexplained jump in the one bucket nobody //--- attributes - a cost invisible in the timing line cannot be traded off against //--- anything. "other" is now genuinely everything else. eraTimeInfo = StringFormat(" | ERA TOOK %.0fs (feature windows %.0fs, net fwd/back %.0fs," " excursion head %.0fs, other %.0fs)", eraS, m_passFeatUs / 1000000.0, m_passNetUs / 1000000.0, m_excUs / 1000000.0, MathMax(eraS - m_passFeatUs / 1000000.0 - m_passNetUs / 1000000.0 - m_excUs / 1000000.0, 0.0)); } Print(ID + ": training in progress - era " + IntegerToString(m_eraCount) + ", OOS accuracy " + DoubleToString(dOosForecast, 1) + "%, IS error " + DoubleToString(dError, 2) + recallInfo + balancedInfo + selectionInfo + predictedInfo + liveInfo + tierInfo + plateauInfo + lifetimeInfo + zeroSkillInfo + gateInfo + m_lastPoolReport + rawOutInfo + neutralWhy + layerInfo + eraTimeInfo); // Forced (unthrottled) panel refresh, right here alongside the console line above, using this // era's own just-finalized m_eraCount/dOosForecast - see UpdateTrainingStatusLabel's // declaration comment for why this can't just rely on the next throttled bar-scan call to // catch up (it would, but a full era later than the console already reported it). UpdateTrainingStatusLabel("Era complete", m_lastDisplayNeuron0, m_lastDisplayNeuron1, m_lastDisplayNeuron2, m_lastDisplaySignal, true); } //--- Genuine convergence THIS era (not a stale m_trainingComplete carried over from a previous //--- run) - (re)start the evaluation-only continual-learning OOS walk. Always rebuilt fresh from //--- the just-converged weights; never resumes a stale walk from a superseded model. //--- m_trainingComplete is the plateau ladder's verdict now (see where it is assigned): "stopped //--- improving after both escape attempts, and there is a recall-passing checkpoint to deploy". //--- It replaces the old (m_objectiveMet && m_oosStable) test, which depended on the removed //--- absolute accuracy target to mean anything - without it, m_oosStable alone (3 eras inside a 2pp //--- band) would have declared convergence at the first flat spot in every run. if(!stop && m_trainingComplete) { Print(ID + ": training CONVERGED at era " + IntegerToString(m_eraCount) + " - this is the best this configuration reached: dir-precision " + DoubleToString(m_bestBalancedOos, 1) + "%, blended OOS " + DoubleToString(dOosForecast, 1) + "%, IS error " + DoubleToString(dError, 2) + ". No new best for " + IntegerToString(m_erasSinceBestBalanced) + " eras across " + IntegerToString(PLATEAU_STAGE_DEPLOY - 1) + " learning-rate warm restarts." + " Weights saved, switching to live inference."); StartOosContinualSimulation(bars, oosCutoff); } if(stop || m_trainingComplete) FinalizeTrainRun(); //--- Deliberately AFTER FinalizeTrainRun(): that call restores the DEPLOYED checkpoint's weights //--- (which may differ from the last era's, if the plateau ladder's best era wasn't the last one //--- run), and this backfill must score with exactly what is about to trade live. if(!stop && m_trainingComplete) StartPatternDatabaseBackfill(bars, totalIter, oosCutoff); //--- else: this era is done but the run continues - the next Train() call (re-triggered via //--- ScheduleTrainingIfNeeded()'s custom event, same mechanism as always) starts the next era //--- fresh, since m_eraResumePending is false while m_trainRunActive stays true //--- Save this model's own learning-rate trajectory back out of the shared global before //--- returning - see m_modelEta's declaration comment. Covers every path that reaches here //--- (natural era completion, whether or not the run itself just finalized). m_modelEta = eta; } //+------------------------------------------------------------------+ //| Ends the current Train() run: restores the best-scoring era's | //| checkpointed weights (if any beat the era the loop happened to | //| end on), persists final state, and clears the resumable-run | //| flags. Called both from Train() itself (natural stop/converge) | //| and from StopTraining() (a mid-chunk Stop click won't get | //| another "New Bar" event to resume into, since | //| ScheduleTrainingIfNeeded() refuses to schedule while | //| m_trainingStopRequested is set, so it must finalize synchronously | //| there instead of being left dangling). | //+------------------------------------------------------------------+ //| Era-cap decision: keep training (true) or deploy best + stop | //| (false). Live chart -> operator dialog; headless -> stop. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::PromptContinuePastEraCap(double bestOos) { //--- No GUI in the Strategy Tester/optimizer - MessageBox() is unavailable there and would just //--- stall a headless run, so deploy the best checkpoint found so far and stop (the safe default). if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD)) return false; //--- Reaching this cap is now the UNUSUAL outcome: a run normally ends itself when the plateau ladder //--- runs out of escapes (see the PLATEAU_* constants), which is a statement about the run having //--- stopped improving rather than about any accuracy number. So the interesting question here is why //--- the ladder had not finished yet - either the run was still finding new bests (just needs more //--- eras), or nothing has ever cleared the per-class recall floor, which blocks auto-deploy on //--- purpose so a one-class model can never ship. Spell out which. bool recallMet = (m_lastBuyRecallPct < 0 || m_lastBuyRecallPct >= m_minDirectionalRecallPct) && (m_lastSellRecallPct < 0 || m_lastSellRecallPct >= m_minDirectionalRecallPct); string neutralNote = (m_priorNeutral > 0.0) ? ("inflated by the ~" + IntegerToString((int)MathRound(m_priorNeutral * 100.0)) + "% Neutral base rate") : "inflated by the dominant Neutral class"; string reasons = ""; if(!m_bestPassedRecall) reasons += " - No era has ever cleared the per-class recall floor, so there is no model safe to\n" + " auto-deploy yet (a model that ignores Buy or Sell must never ship)\n"; else reasons += " - Still improving: " + IntegerToString(m_erasSinceBestBalanced) + " eras since the last new best, plateau stage " + IntegerToString(m_plateauStage) + " of " + IntegerToString(PLATEAU_STAGE_DEPLOY) + " (the run ends itself at stage " + IntegerToString(PLATEAU_STAGE_DEPLOY) + ")\n"; if(!recallMet) reasons += " - Latest era's per-class recall below the floor: Buy " + (m_lastBuyRecallPct < 0 ? "n/a" : IntegerToString(m_lastBuyRecallPct) + "%") + " / Sell " + (m_lastSellRecallPct < 0 ? "n/a" : IntegerToString(m_lastSellRecallPct) + "%") + " (need >=" + IntegerToString(m_minDirectionalRecallPct) + "% each)\n"; if(!m_objectiveMet) reasons += " - The latest era did not produce a valid model (recall floor not met/not measured)\n"; string balancedStr = (m_bestBalancedOos > 0.0) ? ("\nBest directional precision, coverage-weighted (the metric the deployed\ncheckpoint is chosen on): " + DoubleToString(m_bestBalancedOos, 1) + "%\nBest blended OOS accuracy: " + DoubleToString(bestOos, 1) + "% (" + neutralNote + ")\n") : ""; string msg = ID + ": training reached the " + IntegerToString(m_maxErasPerRun) + "-era cap before it finished on its own.\n\n" + "Training now runs until it stops improving, then deploys its best model. Status:\n" + reasons + balancedStr + "\nContinue training?\n\n" + "Yes = keep training for another " + IntegerToString(m_maxErasPerRun) + " eras\n" + "No = deploy the best checkpoint so far and stop training"; int res = MessageBox(msg, "Warrior EA - training", MB_YESNO | MB_ICONQUESTION); return (res == IDYES); } //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| See the declaration comment - the single deploy-persistence path. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::PersistDeployedModel(void) { if(CheckPointer(Net) == POINTER_INVALID) return; double currentIndicatorParams[]; m_indicatorTuner.Flatten(currentIndicatorParams); if(!Net.Save(m_activeFileName + ".nnw", dError, dUndefine, dForecast, dtStudied, m_activeFileCommon, m_eraCount, m_trainingComplete, currentIndicatorParams)) Print(__FUNCTION__ + ": ERROR - Net.Save failed for " + m_activeFileName + ".nnw. The deployed model was NOT persisted to disk."); //--- Deploy-time gate: does this model's pure-MQL5 forward pass match the backend? If so, an //--- inference-only backtest can run DLL-free (see ValidateCpuInference / CNet::SetCpuInference). //--- Persisted into the .stats written next. Chart-only; safe-false everywhere else. m_mqlInferenceValidated = ValidateCpuInference(); if(!SaveModelStats(m_activeFileName, m_activeFileCommon)) // keep calibration state paired with the just-saved weights Print(__FUNCTION__ + ": ERROR - SaveModelStats failed for " + m_activeFileName + ". Calibration state not persisted."); SaveShadowNet(currentIndicatorParams); } //+------------------------------------------------------------------+ void CExpertSignalAIBase::FinalizeTrainRun(void) { //--- A run stopped mid-pass-2.5 or mid-pass-3 never reached that pass's own unfreeze, so lift the //--- scoring freeze here before anything else touches the net - the deployed model must adapt live //--- (see the freeze at pass-3 start, and the identical one the calibration walk takes for the same //--- reason). Harmless no-op when already unfrozen or on a net with no normalization. //--- Same treatment for a run stopped mid-pass-2, which never reached that pass's flush: apply the //--- partial batch and drop back to per-sample updates, so the net this function is about to //--- checkpoint, persist and hand to live inference has nothing accumulated behind it. if(CheckPointer(Net) != POINTER_INVALID) { Net.SetBatchNormFrozen(false); Net.FlushBatch(); Net.SetBatchSize(1); } //--- deploy the most stable/best-scoring era's weights rather than whatever the run happened to //--- end on (which may reflect drift after the objective was first hit, or an aborted run). Restore //--- is now the in-MEMORY snapshot (CNet::RestoreWeights) - see CaptureWeights' note for why the old //--- file-based restore couldn't work on the CPU-DLL backend. if(m_haveOosCheckpoint) { if(Net.RestoreWeights()) { dOosForecast = m_bestOosForecast; //--- Deploy the checkpoint's operating point alongside its weights - the OOS coverage and //--- precision this run is about to report were measured with this pair together. m_dirConfThreshold = m_bestDirConfThreshold; //--- Same F3 reset as the mid-run restore: the deployed weights are the checkpoint's, so the //--- optimizer state that continues from here (online continual learning backprops on this //--- same net - see OnlineLearnStep) must not be the dead run's momentum. Net.ResetOptimizerState(); RefreshLatestSignal(); //--- NOT during shutdown. RestoreWeights() above is an in-MEMORY swap, so the best checkpoint is //--- already the live net by this line - and OnDeinit's PersistWeightsOnShutdown() is about to //--- write exactly those weights anyway. Persisting here too means TWO full ~1MB model writes per //--- signal on the shutdown path, ahead of the chart cleanup, which is what put OnDeinit over //--- MetaTrader's budget: measured 4.46 s to "Abnormal termination" on 2026-08-01, with the chart //--- cleanup completing 0.2 s AFTER the kill. Nothing is lost by skipping it; the same bytes reach //--- the same file one call later. if(!m_shutdownInProgress) PersistDeployedModel(); } } //--- Clean up any legacy on-disk checkpoint from an older (file-based) build so it can't linger. int checkpointFlags = m_activeFileCommon ? FILE_COMMON : 0; if(FileIsExist(m_activeFileName + "_ckpt.tmp", checkpointFlags)) FileDelete(m_activeFileName + "_ckpt.tmp", checkpointFlags); //--- (dtStudied used to be held back while scoring a throwaway candidate - that marker belongs //--- to the DEPLOYED model's "studied up to" state; a candidate eval must leave it untouched. The //--- checkpoint block above is already inert in eval mode (m_haveOosCheckpoint stays false). if(m_eraCount > 0) dtStudied = m_lastBarTime; m_trainRunActive = false; m_eraResumePending = false; m_haveOosCheckpoint = false; m_checkpointEra = -1; // the joint-checkpoint era stamp goes with the snapshot it describes //--- Persist the arrows now drawn on the chart so a deploy/stop survives a later re-add/recompile //--- without a retrain (durable even if the terminal never gets a clean OnDeinit). //--- The prune is suppressed when a STOP is in flight, because that means StopTraining() called us and //--- ShutdownChartCleanup() is about to bulk-purge every arrow anyway. Without this, removing a chart //--- MID-ERA takes the expensive path twice: once here and once in the cleanup that follows, both //--- before anything has been cleared. Same defect as the shutdown prune, one call site earlier - see //--- the prune block in SaveChartSignals() for the measurement. SaveChartSignals(!m_trainingStopRequested); } #endif // WARRIOR_AIBASE_TRAINING_MQH