//+------------------------------------------------------------------+ //| Warrior_EA | //| AnimateDread | //| | //| Model .stats / .cfg sidecars, CPU-inference validation, share-aw| //| | //| PARTIAL IMPLEMENTATION FILE - not standalone. | //| This holds CExpertSignalAIBase method BODIES only. The class | //| declaration lives in Expert\ExpertSignalAIBase.mqh, which | //| #includes this file at the bottom, after the declaration. Do not | //| include it anywhere else and do not compile it on its own. | //| | //| Split out purely to make the 8216-line original navigable; the | //| code inside was moved verbatim, not rewritten. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AIBASE_PERSISTENCE_MQH #define WARRIOR_AIBASE_PERSISTENCE_MQH //+------------------------------------------------------------------+ //| Re-assert the parts of a just-loaded net that the CODE owns but | //| the FILE also stores. See OutputLayerActivation()'s declaration | //| comment and CNet::EnforceOutputActivation() (AI\Network.mqh). | //| | //| Loud on purpose. A repair here means the model on disk was built | //| by a superseded version of this EA and has been silently training | //| against the wrong architecture ever since - the user needs to see | //| that, because the weights learned under the old head are not | //| necessarily worth keeping even once the head itself is corrected. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::EnforceTopologyContract(void) { if(CheckPointer(Net) == POINTER_INVALID) return; ENUM_ACTIVATION intended = OutputLayerActivation(); ENUM_ACTIVATION stale = intended; if(!Net.EnforceOutputActivation(intended, stale)) return; Print(ID + ": REPAIRED loaded model - output layer activation was " + ActivationName(stale) + " on disk, topology specifies " + ActivationName(intended) + ". The saved file was produced by an older build; it has been corrected in memory and the next" + " save will persist the correction. If training looks wrong from here, reset weights and retrain -" + " these weights were learned against the stale head."); } //+------------------------------------------------------------------+ //| Persist/restore the calibration state that must survive a restart | //| for live trading to behave like training: the true class priors | //| and m_confidenceCalScale. Same FILE_COMMON/tester write-guard as | //| CNet::Save so a backtest never overwrites the shared production | //| stats. Versioned/magic-prefixed; a mismatch is treated as absent. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::SaveModelStats(string fileName, bool common) { //--- mirror CNet::Save's guard: shared production stats are never written from inside a backtest if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD))) return true; //--- Staged through a temp file + atomic rename (System\AtomicFile.mqh). Writing straight to //--- .stats truncated it on open, so an interrupted write left a corrupt sidecar AND - because a //--- writer with no FILE_SHARE_* blocks every concurrent open - defeated the share flags that //--- LoadModelStats() carries specifically so a tester agent can read this while a live chart runs. int statsCommonFlag = (common ? FILE_COMMON : 0); string statsTmpName = ""; int handle = AtomicWriteBegin(fileName + ".stats", statsCommonFlag, statsTmpName); if(handle == INVALID_HANDLE) { Print(__FUNCTION__ + ": FileOpen failed for " + statsTmpName + ", error " + IntegerToString(GetLastError()) + " - calibration/online-learning state not persisted."); return false; } //--- WST6 has the SAME field layout as WST5 - the bump exists to invalidate stale IS counters, whose //--- MEANING changed: they used to count every oversampled occurrence, now they count each bar once //--- (see m_isTrainQueuePrimary). Mixing the two would leave the panel averaging an inflated old //--- number into a corrected new one for a very long time, so the reader below drops the IS pair from //--- WST5-and-older files. The OOS pair was always per-bar and carries over untouched. //--- Every write below is validated (<=0 means the call failed, e.g. a disk-full mid-write) - unlike //--- the unchecked version, a truncated file is now detected and rejected instead of silently read //--- back later as all-zero calibration state (see LoadModelStats). bool ok = true; if(FileWriteInteger(handle, 0x57535436) <= 0) // 'WST6' magic/version (WST5 = +compounded IS/OOS counts, WST4 = +live reliability, WST3 = +online state, WST2 = +CPU-marker, WST1 = base) ok = false; if(ok && FileWriteDouble(handle, m_priorBuy) <= 0) ok = false; if(ok && FileWriteDouble(handle, m_priorSell) <= 0) ok = false; if(ok && FileWriteDouble(handle, m_priorNeutral) <= 0) ok = false; if(ok && FileWriteDouble(handle, m_confidenceCalScale) <= 0) ok = false; //--- CPU-inference-safe marker (see ValidateCpuInference): gates whether an inference-only backtest //--- of this model may run DLL-free. Appended after the v1 fields so a v1 reader stops cleanly before it. if(ok && FileWriteInteger(handle, m_mqlInferenceValidated ? 1 : 0) <= 0) ok = false; //--- Online continual-learning state (WST3, see OnlineLearnStep) - the bar-time watermark of the //--- newest bar already learned from, the rolling guardrail accuracy, and the cumulative update count. //--- Appended after the WST2 fields so a WST2 reader stops cleanly before them. if(ok && FileWriteLong(handle, (long)m_onlineLearnedUpToTime) <= 0) ok = false; if(ok && FileWriteDouble(handle, m_onlineRollingAcc) <= 0) ok = false; if(ok && FileWriteLong(handle, m_onlineSamples) <= 0) ok = false; //--- Deployed model's last-measured OOS reliability (WST4) - the live status panel shows this as the //--- "signal hit-rate" so a freshly-reloaded, inference-only model still reports what to expect live //--- (these members are computed only during training, so without persistence they read n/a on reload). //--- Appended after the WST3 fields so a WST3 reader stops cleanly before them. if(ok && FileWriteInteger(handle, m_lastBuyFiredPrecPct) <= 0) ok = false; if(ok && FileWriteInteger(handle, m_lastSellFiredPrecPct) <= 0) ok = false; if(ok && FileWriteInteger(handle, m_lastBuyRecallPct) <= 0) ok = false; if(ok && FileWriteInteger(handle, m_lastSellRecallPct) <= 0) ok = false; if(ok && FileWriteInteger(handle, m_lastBuyFired) <= 0) ok = false; if(ok && FileWriteInteger(handle, m_lastSellFired) <= 0) ok = false; //--- Compounded, persistent IS/OOS accuracy counts (WST5) - see m_cumIsCorrect. Carried across //--- restarts so the panel's accuracy keeps compounding instead of restarting each session. //--- Appended after the WST4 fields so a WST4 reader stops cleanly before them. if(ok && FileWriteLong(handle, m_cumIsCorrect) <= 0) ok = false; if(ok && FileWriteLong(handle, m_cumIsTotal) <= 0) ok = false; if(ok && FileWriteLong(handle, m_cumOosCorrect) <= 0) ok = false; if(ok && FileWriteLong(handle, m_cumOosTotal) <= 0) ok = false; //--- A partial write (disk full mid-write) now discards the temp and leaves the previous good //--- .stats in place, instead of publishing a truncated one that reads back as all-zero //--- calibration state. return AtomicWriteEnd(handle, fileName + ".stats", statsTmpName, statsCommonFlag, ok, __FUNCTION__); } //+------------------------------------------------------------------+ bool CExpertSignalAIBase::LoadModelStats(string fileName, bool common) { if(!FileIsExist(fileName + ".stats", common ? FILE_COMMON : 0)) return false; //--- share flags: read-only, must not fail just because another process holds the file - see CopySharedFile(). int handle = FileOpen(fileName + ".stats", (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ | FILE_SHARE_READ | FILE_SHARE_WRITE); if(handle == INVALID_HANDLE) return false; int magic = FileReadInteger(handle); if(magic != 0x57535431 && magic != 0x57535432 && magic != 0x57535433 && magic != 0x57535434 && magic != 0x57535435 && magic != 0x57535436) { FileClose(handle); return false; } m_priorBuy = FileReadDouble(handle); m_priorSell = FileReadDouble(handle); m_priorNeutral = FileReadDouble(handle); m_confidenceCalScale = FileReadDouble(handle); //--- v2+ appended the CPU-inference-safe marker; v1 files predate it (treated as not-yet-validated so //--- the model stays on the DLL path until re-deployed by a build that runs ValidateCpuInference). m_mqlInferenceValidated = (magic == 0x57535432 || magic == 0x57535433 || magic == 0x57535434 || magic == 0x57535435 || magic == 0x57535436) ? (FileReadInteger(handle) != 0) : false; //--- v3 appended the online continual-learning state. Older files predate it: leave the watermark at 0 //--- (OnlineLearnStep anchors it to the current frontier on first run - no retroactive backprop) and //--- the rolling accuracy at -1 (re-seeded from the deploy baseline on the first update). if(magic == 0x57535433 || magic == 0x57535434 || magic == 0x57535435 || magic == 0x57535436) { m_onlineLearnedUpToTime = (datetime)FileReadLong(handle); m_onlineRollingAcc = FileReadDouble(handle); m_onlineSamples = FileReadLong(handle); } //--- v4 appended the deployed model's last-measured OOS reliability (for the live status panel). Older //--- files predate it: the members keep their -1 / 0 ctor defaults, so the panel shows no hit-rate line //--- until the model is re-trained (or re-deployed) by a WST4+ build - exactly the pre-persistence behaviour. if(magic == 0x57535434 || magic == 0x57535435 || magic == 0x57535436) { m_lastBuyFiredPrecPct = FileReadInteger(handle); m_lastSellFiredPrecPct = FileReadInteger(handle); m_lastBuyRecallPct = FileReadInteger(handle); m_lastSellRecallPct = FileReadInteger(handle); m_lastBuyFired = FileReadInteger(handle); m_lastSellFired = FileReadInteger(handle); } //--- v5 appended the compounded/persistent IS/OOS accuracy counts (see m_cumIsCorrect). Older files //--- predate it: the counts keep their 0 ctor defaults, so the panel shows "measuring" until the next //--- era scores signals - then it resumes compounding from there. if(magic == 0x57535435 || magic == 0x57535436) { m_cumIsCorrect = FileReadLong(handle); m_cumIsTotal = FileReadLong(handle); m_cumOosCorrect = FileReadLong(handle); m_cumOosTotal = FileReadLong(handle); //--- A WST5 file's IS pair counted every OVERSAMPLED OCCURRENCE, so it was measured against a //--- ~58%-directional queue instead of the real ~6% distribution - not comparable with the OOS pair //--- beside it, and the source of the "IS 77% / OOS 12%, looks like overfitting" reading. The fix //--- (m_isTrainQueuePrimary) changed what the counter MEANS, not the file layout, so old totals must //--- be discarded rather than compounded into the corrected ones. Dropped, not migrated: there is no //--- per-bar count recoverable from an occurrence-weighted total. The panel shows "measuring" for one //--- era, then compounds honestly. The OOS pair always counted bars once - keep it. if(magic == 0x57535435) { m_cumIsCorrect = 0; m_cumIsTotal = 0; } } FileClose(handle); return true; } //+------------------------------------------------------------------+ //| Deploy-time self-check (chart only, where a compute backend | //| exists): run the just-saved deployed model through both the | //| backend Net and a throwaway pure-MQL5 clone (CNet::SetCpuInference| //| loaded from the same .nnw) on one real input window, and return | //| true only if their outputs match within CPU_INFERENCE_MAX_DIFF. | //| This is what lets an inference-only backtest run DLL-free; any | //| error, size mismatch, or a not-yet-ported architecture (conv/LSTM | //| CPU load fails) returns false -> the model stays on the DLL path. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::ValidateCpuInference(void) { //--- Chart-only: needs a real backend to compare against, and only the shared production model (not a //--- per-agent optimization cache) is ever seeded into a buyer's inference-only backtest. if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD)) return false; if(CheckPointer(Net) == POINTER_INVALID || CheckPointer(TempData) == POINTER_INVALID) return false; //--- Build the same latest-bar input window RefreshLatestSignal() feeds the deployed model. TempData.Clear(); TempData.Reserve((int)m_historyBars * m_neuronsCount); for(int b = 0; b < (int)m_historyBars; b++) if(!BufferTempData(b)) return false; if(TempData.Total() < (int)m_historyBars * m_neuronsCount) return false; //--- Reference: the compute backend Net actually trained on. Batch-norm statistics are frozen for //--- the duration: unfrozen, a forward pass ALSO advances them, so this reference pass would leave //--- the live model one EMA step ahead of the .nnw the candidate below loads - and the check would //--- then be measuring its own side effect rather than the two backends. See //--- CNet::SetBatchNormFrozen. A no-op on topologies without normalization. Net.SetBatchNormFrozen(true); bool refOk = Net.feedForward(TempData); Net.SetBatchNormFrozen(false); if(!refOk) return false; CArrayDouble *refOut = new CArrayDouble(); if(CheckPointer(refOut) == POINTER_INVALID) return false; Net.getResults(refOut); //--- Candidate: a throwaway pure-MQL5 clone loaded from the just-saved deployed weights. CNet *cpu = new CNet(NULL); if(CheckPointer(cpu) == POINTER_INVALID) { delete refOut; return false; } cpu.SetCpuInference(true); double e, u, f; datetime tm; long era; bool complete; double ip[]; bool loaded = cpu.Load(m_activeFileName + ".nnw", e, u, f, tm, m_activeFileCommon, era, complete, ip); //--- Frozen for the same reason as the reference pass above - both sides must evaluate the SAME //--- statistics, which are the ones sitting in the .nnw. if(loaded) cpu.SetBatchNormFrozen(true); bool pass = false; double maxDiff = DBL_MAX; if(loaded && cpu.feedForward(TempData)) { CArrayDouble *cpuOut = new CArrayDouble(); if(CheckPointer(cpuOut) != POINTER_INVALID) { cpu.getResults(cpuOut); if(cpuOut.Total() == refOut.Total() && refOut.Total() > 0) { maxDiff = 0.0; for(int i = 0; i < refOut.Total(); i++) maxDiff = MathMax(maxDiff, MathAbs(refOut.At(i) - cpuOut.At(i))); pass = (maxDiff <= CPU_INFERENCE_MAX_DIFF); } delete cpuOut; } } delete cpu; delete refOut; PrintVerbose(ID + ": CPU-inference validation " + (pass ? "PASSED - backtests may run DLL-free" : "FAILED - backtests keep using the DLL") + " (max |delta| = " + (maxDiff == DBL_MAX ? "n/a" : DoubleToString(maxDiff, 8)) + ", tol " + DoubleToString(CPU_INFERENCE_MAX_DIFF, 8) + ")"); return pass; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::SaveTopologyConfiguration(string fileName, int initialNeuronsCount, int hiddenLayersCount, double neuronsReduction, int minNeuronsCount, int optimizationAlgo, int historyBars, int outputNeuronsCount, int neuronsCount, int studyPeriod, int minTrainYear, bool isInitialized, int stopTrainWR, int fractalPeriods, bool common) { string configFileName = fileName + ".cfg"; //--- Staged through a temp file + atomic rename (System\AtomicFile.mqh). This one matters most of //--- the four sidecars: FileOpen(FILE_WRITE) truncates on open, so an interrupted write left a SHORT //--- .cfg, and LoadAndCompareTopologyConfiguration() reads a short read as a mismatch - which //--- discards the trained model and restarts from era 0. It also stops an exclusive writer from //--- blocking that reader's FILE_SHARE_READ|FILE_SHARE_WRITE open on another instance. int cfgCommonFlag = (common ? FILE_COMMON : 0); string cfgTmpName = ""; int handle = AtomicWriteBegin(configFileName, cfgCommonFlag, cfgTmpName); if(handle == INVALID_HANDLE) { Print("Error: Unable to open file ", cfgTmpName, " : Error code: ", GetLastError()); ResetLastError(); return false; } //--- ON-DISK LAYOUT - DO NOT REORDER OR RETYPE. LoadAndCompareTopologyConfiguration() reads these //--- back positionally, and every existing .cfg on every deployed install has this exact sequence; //--- changing it silently invalidates them all (-> "configuration mismatch" -> retrain from era 0). //--- Appending a NEW field at the end is the only backward-safe change. //--- This used to be 13 copy-pasted 6-line blocks that each returned WITHOUT FileClose(handle), //--- leaking the handle on every write failure; the shared ok-chain closes exactly once below. bool ok = (FileWriteInteger(handle, initialNeuronsCount) >= sizeof(int)); if(ok && FileWriteInteger(handle, hiddenLayersCount) < sizeof(int)) ok = false; if(ok && FileWriteDouble(handle, neuronsReduction) < sizeof(double)) ok = false; if(ok && FileWriteInteger(handle, minNeuronsCount) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, optimizationAlgo) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, historyBars) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, outputNeuronsCount) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, neuronsCount) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, studyPeriod) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, minTrainYear) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, isInitialized) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, stopTrainWR) < sizeof(int)) ok = false; if(ok && FileWriteInteger(handle, fractalPeriods) < sizeof(int)) ok = false; if(!ok) { Print("Error writing ", configFileName, " : Error code: ", GetLastError()); ResetLastError(); } return AtomicWriteEnd(handle, configFileName, cfgTmpName, cfgCommonFlag, ok, __FUNCTION__); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::LoadAndCompareTopologyConfiguration(string fileName, int initialNeuronsCount, int hiddenLayersCount, double neuronsReduction, int minNeuronsCount, int optimizationAlgo, int historyBars, int outputNeuronsCount, int neuronsCount, int studyPeriod, int minTrainYear, bool isInitialized, int stopTrainWR, int fractalPeriods, bool common) { string configFileName = fileName + ".cfg"; if(!FileIsExist(configFileName, common ? FILE_COMMON : 0)) { //--- Not an error: no cached config for this parameter set yet - normal on the first run of a config //--- (and every first tester/optimizer pass on a fresh agent). The caller treats a false return as //--- "start fresh", so log it as informational rather than "Error" (which read as a real failure). PrintVerbose(__FUNCTION__ + ": no cached topology config at " + configFileName + " yet - treating as a fresh start for this configuration"); return false; } //--- share flags: read-only, see CopySharedFile(). int handle = FileOpen(configFileName, FILE_READ | FILE_BIN | FILE_SHARE_READ | FILE_SHARE_WRITE | (common ? FILE_COMMON : 0)); if(handle == INVALID_HANDLE) { Print("Error: Unable to open file ", configFileName); return false; } int savedInitialNeurons = FileReadInteger(handle); int savedHiddenLayers = FileReadInteger(handle); double savedReductionFactor = FileReadDouble(handle); int savedMinNeurons = FileReadInteger(handle); int savedOptimizationAlgo = FileReadInteger(handle); int savedHistoryBars = FileReadInteger(handle); int savedOutputNeuronsCount = FileReadInteger(handle); int savedNeuronsCount = FileReadInteger(handle); int savedStudyPeriod = FileReadInteger(handle); int savedMinTrainYear = FileReadInteger(handle); bool savedIsInitialized = FileReadInteger(handle); // read for layout only - NOT compared (see below) int savedStopTrainWR = FileReadInteger(handle); // retired MinWR slot - layout only, NOT compared (see below) int savedFractalPeriods = FileReadInteger(handle); FileClose(handle); //--- isInitialized is DELIBERATELY excluded from the comparison: it is a runtime lifecycle flag, not a //--- topology/input parameter, and it is always false at the point the compare and the fresh-start save //--- run (set true only at the end of InitNeuralNetwork). Including it used to make a reset-written .cfg //--- (saved post-init, flag true) spuriously mismatch on the next attach and discard the just-saved //--- weights (see ResetWeights); ignoring it here also neutralises any such stale .cfg left on disk. //--- savedStopTrainWR is excluded for a related reason: the MinWR input it came from no longer exists //--- (convergence is decided by the plateau ladder - see LEGACY_CONVERGE_WR_SLOT), so whatever value an //--- older .cfg happens to hold says nothing about whether the saved WEIGHTS are compatible. Comparing //--- it would discard a perfectly good model from anyone whose MinWR was not the shipped default. //--- Build an explicit per-field diff so a genuine mismatch says WHICH parameter changed, instead of a //--- bare "Configuration mismatch" that leaves the user (or us) guessing why a restart retrained from 0. string diff = ""; if(savedInitialNeurons != initialNeuronsCount) diff += " initialNeurons " + IntegerToString(savedInitialNeurons) + "->" + IntegerToString(initialNeuronsCount) + ";"; if(savedHiddenLayers != hiddenLayersCount) diff += " hiddenLayers " + IntegerToString(savedHiddenLayers) + "->" + IntegerToString(hiddenLayersCount) + ";"; if(savedReductionFactor != neuronsReduction) diff += " reduction " + DoubleToString(savedReductionFactor, 4) + "->" + DoubleToString(neuronsReduction, 4) + ";"; if(savedMinNeurons != minNeuronsCount) diff += " minNeurons " + IntegerToString(savedMinNeurons) + "->" + IntegerToString(minNeuronsCount) + ";"; if(savedOptimizationAlgo != optimizationAlgo) diff += " optimizer " + IntegerToString(savedOptimizationAlgo) + "->" + IntegerToString(optimizationAlgo) + ";"; if(savedHistoryBars != historyBars) diff += " historyBars " + IntegerToString(savedHistoryBars) + "->" + IntegerToString(historyBars) + ";"; if(savedOutputNeuronsCount != outputNeuronsCount) diff += " outputs " + IntegerToString(savedOutputNeuronsCount) + "->" + IntegerToString(outputNeuronsCount) + ";"; if(savedNeuronsCount != neuronsCount) diff += " inputWidth " + IntegerToString(savedNeuronsCount) + "->" + IntegerToString(neuronsCount) + " (a feature toggle changed);"; if(savedStudyPeriod != studyPeriod) diff += " studyPeriod " + IntegerToString(savedStudyPeriod) + "->" + IntegerToString(studyPeriod) + ";"; if(savedMinTrainYear != minTrainYear) diff += " minTrainYear " + IntegerToString(savedMinTrainYear) + "->" + IntegerToString(minTrainYear) + ";"; //--- (savedStopTrainWR deliberately NOT compared - retired input, see the note above) if(savedFractalPeriods != fractalPeriods) diff += " fractalPeriods " + IntegerToString(savedFractalPeriods) + "->" + IntegerToString(fractalPeriods) + ";"; if(diff != "") { Print("Configuration mismatch for ", configFileName, " -> retraining from era 0. Changed:", diff, " (a saved model only reloads when these parameters match exactly; revert the changed input to resume the existing model.)"); FileDelete(configFileName, common ? FILE_COMMON : 0); return false; } return true; } //+------------------------------------------------------------------+ //| See the declaration comment - retries a FileCopy FROM FILE_COMMON | //| that raced a concurrent writer (typically a live chart's own | //| atomic Save(), mid write-then-rename on the SAME source file). | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::CopyFileWithRetry(string srcFileName, string dstFileName) { //--- Retries are only a backstop for a genuinely transient hiccup. The failure actually observed here //--- (2026-07-26: error 5004 on ALL 8 attempts across ~9s, surviving a full terminal restart, while //--- FileIsExist on the very same file returned true) is NOT transient and no retry budget can fix it: //--- FileCopy opens its SOURCE without share flags, i.e. it demands exclusive access, so it fails //--- outright whenever any other process holds the file open - which is precisely the normal state of //--- the deployed .nnw while a live chart runs this EA. CopySharedFile() is therefore the real fix //--- (it opens the source with FILE_SHARE_READ|FILE_SHARE_WRITE); this loop just covers the remaining //--- ordinary races (e.g. reading mid atomic-rename). const int RETRY_ATTEMPTS = 5; const int RETRY_DELAY_CAP_MS = 1000; int delayMs = 150; bool ok = false; for(int attempt = 0; attempt < RETRY_ATTEMPTS && !ok; attempt++) { if(attempt > 0) { Sleep(delayMs); delayMs = (int)MathMin(delayMs * 2, RETRY_DELAY_CAP_MS); } //--- quiet on every attempt but the last, so an ordinary race that resolves on retry 2 doesn't //--- spam the journal with scary per-attempt failures. ok = CopySharedFile(srcFileName, dstFileName, attempt < RETRY_ATTEMPTS - 1); } if(!ok) Print(__FUNCTION__ + ": WARNING - failed to copy " + srcFileName + " (shared folder) -> " + dstFileName + " after " + IntegerToString(RETRY_ATTEMPTS) + " attempts (see the per-stage reason above)." + " This run will train from scratch instead of reusing the deployed model."); return ok; } //+------------------------------------------------------------------+ //| Share-aware streamed file copy FROM the shared (FILE_COMMON) | //| folder INTO this program's own sandbox (the tester agent's local | //| MQL5\Files when running under the Strategy Tester). | //| | //| Exists because FileCopy() cannot do this: it opens the source | //| WITHOUT FILE_SHARE_* flags, so it returns 5004 ("cannot open | //| file") whenever another process already has the file open - and a | //| live chart running this EA holds the deployed model open as its | //| normal state, which made a backtest silently fall back to an | //| untrained network. Opening the source with FILE_SHARE_READ | | //| FILE_SHARE_WRITE explicitly permits reading a file someone else | //| is using, which is exactly the semantics wanted here: the source | //| is only ever READ, never written, so sharing it is always safe. | //| | //| Writes via a .copytmp + atomic rename (same doctrine as | //| CNet::Save) so an interrupted copy can never leave a truncated | //| .nnw that the next run would load as a corrupt model. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::CopySharedFile(string srcFileName, string dstFileName, bool quiet) { ResetLastError(); int src = FileOpen(srcFileName, FILE_COMMON | FILE_BIN | FILE_READ | FILE_SHARE_READ | FILE_SHARE_WRITE); if(src == INVALID_HANDLE) { //--- Distinguish the two failure stages explicitly - a source failure here (with the share flags //--- already set) would mean the shared folder itself is unreachable from this sandbox, which is a //--- completely different problem from a destination/sandbox write failure below. if(!quiet) Print(__FUNCTION__ + ": cannot open SOURCE " + srcFileName + " in the shared folder, error " + IntegerToString(GetLastError()) + " (share flags were set, so this is not a lock)."); return false; } ulong size = FileSize(src); uchar buf[]; //--- 0-byte source would produce a 0-byte model file that CNet::Load rejects later as a stub - refuse //--- it here instead, so the caller falls back to a fresh topology with an accurate reason logged. if(size == 0 || ArrayResize(buf, (int)size) != (int)size) { FileClose(src); if(!quiet) Print(__FUNCTION__ + ": refusing to copy " + srcFileName + " - source is " + IntegerToString((int)size) + " bytes (empty, or too large to buffer)."); return false; } uint read = FileReadArray(src, buf, 0, (int)size); FileClose(src); if(read != (uint)size) { if(!quiet) Print(__FUNCTION__ + ": short read on " + srcFileName + " (" + IntegerToString((int)read) + " of " + IntegerToString((int)size) + " bytes) - not copying a partial model."); return false; } //--- destination is this program's OWN sandbox (no FILE_COMMON) - never the shared production folder. string tmpName = dstFileName + ".copytmp"; ResetLastError(); int dst = FileOpen(tmpName, FILE_BIN | FILE_WRITE); if(dst == INVALID_HANDLE) { if(!quiet) Print(__FUNCTION__ + ": cannot open DESTINATION " + tmpName + " in this agent's sandbox, error " + IntegerToString(GetLastError()) + "."); return false; } uint written = FileWriteArray(dst, buf, 0, ArraySize(buf)); FileFlush(dst); FileClose(dst); if(written != (uint)size) { FileDelete(tmpName); if(!quiet) Print(__FUNCTION__ + ": short write to " + tmpName + " (" + IntegerToString((int)written) + " of " + IntegerToString((int)size) + " bytes) - discarded the partial copy."); return false; } //--- atomic swap into place, so a reader never sees a half-written file if(!FileMove(tmpName, 0, dstFileName, FILE_REWRITE)) { if(!quiet) Print(__FUNCTION__ + ": atomic rename " + tmpName + " -> " + dstFileName + " failed, error " + IntegerToString(GetLastError()) + "."); ResetLastError(); return false; } return true; } //+------------------------------------------------------------------+ //| See the declaration comment - retries Net.Load() against the | //| active file. Covers the same transient-lock class as | //| CopyFileWithRetry (e.g. antivirus briefly holding the file just | //| written into the tester's local sandbox) rather than assuming any | //| single failed read means "no model"/"corrupt file". | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::LoadNetWithRetry(double &indicatorParams[]) { //--- same exponential-backoff reasoning as CopyFileWithRetry - see its declaration comment. const int RETRY_ATTEMPTS = 5; const int RETRY_DELAY_CAP_MS = 2000; int delayMs = 200; bool loaded = false; for(int attempt = 0; attempt < RETRY_ATTEMPTS && !loaded; attempt++) { if(attempt > 0) { Sleep(delayMs); delayMs = (int)MathMin(delayMs * 2, RETRY_DELAY_CAP_MS); } loaded = Net.Load(m_activeFileName + ".nnw", dError, dUndefine, dForecast, dtStudied, m_activeFileCommon, m_eraCount, m_trainingComplete, indicatorParams); } return loaded; } #endif // WARRIOR_AIBASE_PERSISTENCE_MQH