//+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include #include #include #include #include "..\System\AtomicFile.mqh" //--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available, //--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Sizes //--- WarriorCPU.dll's worker thread pool - no effect at all when a GPU tier (OpenCL or DirectML) is //--- active, since neither one calls into WarriorCPU.dll. //--- //--- A FIXED SMALL THREAD COUNT PER NETWORK, not a share of the machine. This replaced a TargetCPULoad //--- input divided by the live chart count, which was wrong twice over: //--- - The count is a SNAPSHOT taken when each net's pool is built, and charts are attached one at a //--- time. Measured 2026-07-29 with five charts: they took 10/6/5/4/4% of the same budget, because //--- the first chart only ever saw itself and the last saw all five. So the earliest chart got //--- several times the threads of the latest - which silently skews any cross-topology comparison //--- run on those charts, the exact thing the setting existed to make fair. //--- - Nothing rebalances when a chart is added or removed, and rebalancing would mean tearing down a //--- DLL context underneath a running trainer. //--- Neither problem exists once the answer stops depending on how many charts are running. //--- //--- Why 2 threads is not a compromise: since the topology became data-derived the widest dense layer //--- is 64 units (ExpertSignalAIBase.mqh's ComputeFirstLayerWidth), so each ParallelFor has almost //--- nothing to split and per-dispatch overhead dominates. The measurements agree - an MLP era cost //--- ~66s at a wildly oversubscribed 12 threads and ~80s at 1 thread, a 20% spread across a 12x //--- difference in thread count. Two per net also lands six concurrent charts exactly on a 12-core box. //--- //--- Removing the input costs nothing on the product side: a Market build has no DLL tier at all (MQL5 //--- Market rule IV strips the #import blocks - see AI\NeuronDirectML.mqh), so it was already compiled //--- out to a constant there and no buyer could ever reach it. #define CPU_THREADS_PER_NETWORK 2 //+------------------------------------------------------------------+ //| The percentage CDirectMLMy needs in order to land on | //| CPU_THREADS_PER_NETWORK workers, given the detected core count. | //| (WarriorCPU.dll takes a percentage of cores, not a thread count.) | //+------------------------------------------------------------------+ int EffectiveCpuLoadPercent() { int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES); //--- Unknown core count: assume a small machine rather than a large one. Guessing high here would //--- reintroduce exactly the oversubscription this function exists to prevent. if(cores <= 0) cores = 4; //--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the //--- DLL, so this is the one value that needs no arithmetic. if(cores <= CPU_THREADS_PER_NETWORK) return 100; //--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread //--- short of the target is a worse error than landing one over. int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores); return (int)MathMax(1, MathMin(100, pct)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference //--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's //--- abstract "0.001" mention, which the book's own worked examples don't actually use either. //--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005) //--- this project had separately tuned lr to before this input existed, so no behavior conflict. //--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak //--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to //--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this //--- reverts to the literature/book default. input double AdamLearningRate = 0.0003; // Adam learning rate input double AdamBeta1 = 0.9; // Adam beta1 input double AdamBeta2 = 0.999; // Adam beta2 //--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method //--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses //--- Adam's book-default rate as its starting point too. The book also states no numeric default for //--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses //--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for. input double SgdLearningRate = 0.0003; // SGD learning rate input double SgdMomentum = 0.9; // SGD momentum #define lr AdamLearningRate #define b1 AdamBeta1 #define b2 AdamBeta2 #define momentum SgdMomentum double eta = lr; #define defConnect 0x7781 #define defArrayConnects 0x7782 #define defNeuronBase 0x7783 #define defNeuron 0x7784 #define defNeuronConv 0x7785 #define defNeuronPool 0x7786 #define defLayer 0x7787 #define defArrayLayer 0x7788 #define defNet 0x7789 #define defNeuronLSTM 0x7791 //--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like //--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps //--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh. #define defNeuronBatchNorm 0x7792 //--- #define defBufferDouble 0x7882 #define defNeuronBaseOCL 0x7883 #define defNeuronLSTMOCL 0x7884 #define defNeuronConvOCL 0x7885 #define defNeuronPoolOCL 0x7886 #define defNeuronBatchNormOCL 0x7887 //--- #define def_k_FeedForward 0 #define def_k_ff_matrix_w 0 #define def_k_ff_matrix_i 1 #define def_k_ff_matrix_o 2 #define def_k_ff_inputs 3 #define def_k_ff_activation 4 //--- #define def_k_CaclOutputGradient 1 #define def_k_cog_matrix_t 0 #define def_k_cog_matrix_o 1 #define def_k_cog_matrix_ig 2 #define def_k_cog_activation 3 //--- #define def_k_CaclHiddenGradient 2 #define def_k_chg_matrix_w 0 #define def_k_chg_matrix_g 1 #define def_k_chg_matrix_o 2 #define def_k_chg_matrix_ig 3 #define def_k_chg_outputs 4 #define def_k_chg_activation 5 //--- #define def_k_UpdateWeightsMomentum 3 #define def_k_uwm_matrix_w 0 #define def_k_uwm_matrix_g 1 #define def_k_uwm_matrix_i 2 #define def_k_uwm_matrix_dw 3 #define def_k_uwm_inputs 4 #define def_k_uwm_learning_rates 5 #define def_k_uwm_momentum 6 #define def_k_uwm_optimizer 7 //--- #define def_k_UpdateWeightsAdam 4 #define def_k_uwa_matrix_w 0 #define def_k_uwa_matrix_g 1 #define def_k_uwa_matrix_i 2 #define def_k_uwa_matrix_m 3 #define def_k_uwa_matrix_v 4 #define def_k_uwa_inputs 5 #define def_k_uwa_l 6 #define def_k_uwa_b1 7 #define def_k_uwa_b2 8 //--- #define def_k_FeedForwardProof 15 #define def_k_ffp_matrix_i 0 #define def_k_ffp_matrix_o 1 #define def_k_ffp_inputs 2 #define def_k_ffp_window 3 #define def_k_ffp_step 4 //--- #define def_k_CalcInputGradientProof 16 #define def_k_cigp_matrix_i 0 #define def_k_cigp_matrix_g 1 #define def_k_cigp_matrix_o 2 #define def_k_cigp_matrix_ig 3 #define def_k_cigp_outputs 4 #define def_k_cigp_window 5 #define def_k_cigp_step 6 //--- #define def_k_FeedForwardConv 5 #define def_k_ffc_matrix_w 0 #define def_k_ffc_matrix_i 1 #define def_k_ffc_matrix_o 2 #define def_k_ffc_inputs 3 #define def_k_ffc_step 4 #define def_k_ffc_window_in 5 #define def_k_ffc_window_out 6 #define def_k_ffc_activation 7 //--- #define def_k_CalcHiddenGradientConv 6 #define def_k_chgc_matrix_w 0 #define def_k_chgc_matrix_g 1 #define def_k_chgc_matrix_o 2 #define def_k_chgc_matrix_ig 3 #define def_k_chgc_outputs 4 #define def_k_chgc_step 5 #define def_k_chgc_window_in 6 #define def_k_chgc_window_out 7 #define def_k_chgc_activation 8 //--- #define def_k_UpdateWeightsConvMomentum 7 #define def_k_uwcm_matrix_w 0 #define def_k_uwcm_matrix_g 1 #define def_k_uwcm_matrix_i 2 #define def_k_uwcm_matrix_dw 3 #define def_k_uwcm_inputs 4 #define def_k_uwcm_learning_rates 5 #define def_k_uwcm_momentum 6 #define def_k_uwcm_window_in 7 #define def_k_uwcm_window_out 8 #define def_k_uwcm_step 9 #define def_k_uwcm_optimizer 10 //--- #define def_k_UpdateWeightsConvAdam 8 #define def_k_uwca_matrix_w 0 #define def_k_uwca_matrix_g 1 #define def_k_uwca_matrix_i 2 #define def_k_uwca_matrix_m 3 #define def_k_uwca_matrix_v 4 #define def_k_uwca_inputs 5 #define def_k_uwca_l 6 #define def_k_uwca_b1 7 #define def_k_uwca_b2 8 #define def_k_uwca_window_in 9 #define def_k_uwca_window_out 10 #define def_k_uwca_step 11 //--- // LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient // flows back into h_prev/c_prev from a prior step). Supports both Adam and // SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) - // see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch. // Derived from scratch from the standard LSTM equations - NOT ported from // the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites // the live weights buffer instead of writing to weights_gradient. #define def_k_LSTM_Gates 9 #define def_k_lstmg_matrix_w 0 #define def_k_lstmg_hidden_prev 1 #define def_k_lstmg_inputs 2 #define def_k_lstmg_concatenated 3 #define def_k_lstmg_hidden_size 4 #define def_k_lstmg_input_size 5 //--- #define def_k_LSTM_State 10 #define def_k_lstms_concatenated 0 #define def_k_lstms_memory 1 #define def_k_lstms_hidden_prev 2 #define def_k_lstms_hidden_cache 3 #define def_k_lstms_output 4 #define def_k_lstms_hidden_size 5 //--- #define def_k_LSTM_GateGradient 11 #define def_k_lstmgg_gradient 0 #define def_k_lstmgg_memory 1 #define def_k_lstmgg_concatenated 2 #define def_k_lstmgg_concatenated_gradient 3 #define def_k_lstmgg_hidden_size 4 //--- #define def_k_LSTM_WeightsGradient 12 #define def_k_lstmwg_concatenated_gradient 0 #define def_k_lstmwg_hidden_cache 1 #define def_k_lstmwg_inputs 2 #define def_k_lstmwg_weights_gradient 3 #define def_k_lstmwg_hidden_size 4 #define def_k_lstmwg_input_size 5 //--- #define def_k_LSTM_InputsGradient 13 #define def_k_lstmig_concatenated_gradient 0 #define def_k_lstmig_matrix_w 1 #define def_k_lstmig_inputs_gradient 2 #define def_k_lstmig_hidden_size 3 #define def_k_lstmig_input_size 4 //--- #define def_k_LSTM_UpdateWeightsAdam 14 #define def_k_lstmuwa_matrix_w 0 #define def_k_lstmuwa_weights_gradient 1 #define def_k_lstmuwa_matrix_m 2 #define def_k_lstmuwa_matrix_v 3 #define def_k_lstmuwa_l 4 #define def_k_lstmuwa_b1 5 #define def_k_lstmuwa_b2 6 //--- // SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see // AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body. #define def_k_LSTM_UpdateWeightsMomentum 17 #define def_k_lstmuwm_matrix_w 0 #define def_k_lstmuwm_weights_gradient 1 #define def_k_lstmuwm_matrix_dw 2 #define def_k_lstmuwm_learning_rates 3 #define def_k_lstmuwm_momentum 4 #define def_k_lstmuwm_optimizer 5 //--- // Sequence LSTM. One launch PER TIMESTEP - the recurrence is sequential and OpenCL // barriers only span a work-group, so the host loop is what provides the global // ordering. See the block comment above LSTM_SeqStepForward in AI\Network.cl. #define def_k_LSTM_SeqStepForward 18 #define def_k_lsf_matrix_w 0 #define def_k_lsf_inputs 1 #define def_k_lsf_cache_gates 2 #define def_k_lsf_cache_cell 3 #define def_k_lsf_cache_hidden 4 #define def_k_lsf_output 5 #define def_k_lsf_hidden_size 6 #define def_k_lsf_step_inputs 7 #define def_k_lsf_steps 8 #define def_k_lsf_t 9 //--- #define def_k_LSTM_SeqStepGateGrad 19 #define def_k_lsgg_out_gradient 0 #define def_k_lsgg_dh_buf 1 #define def_k_lsgg_dc_buf 2 #define def_k_lsgg_cache_gates 3 #define def_k_lsgg_cache_cell 4 #define def_k_lsgg_gate_grad 5 #define def_k_lsgg_hidden_size 6 #define def_k_lsgg_steps 7 #define def_k_lsgg_t 8 //--- #define def_k_LSTM_SeqStepWeightGrad 20 #define def_k_lswg_gate_grad 0 #define def_k_lswg_cache_hidden 1 #define def_k_lswg_inputs 2 #define def_k_lswg_weights_gradient 3 #define def_k_lswg_hidden_size 4 #define def_k_lswg_step_inputs 5 #define def_k_lswg_t 6 //--- #define def_k_LSTM_SeqStepInputGrad 21 #define def_k_lsig_gate_grad 0 #define def_k_lsig_matrix_w 1 #define def_k_lsig_inputs_gradient 2 #define def_k_lsig_dh_buf 3 #define def_k_lsig_hidden_size 4 #define def_k_lsig_step_inputs 5 #define def_k_lsig_t 6 //--- // Mini-batch gradient accumulation - see the block comment above AccumulateWeightGrad in // AI\Network.cl. The APPLY is a kernel too on OpenCL since 2026-08-09 (def_k_ApplyAccumAdam below); // the host-side MQL5 step in NeuronOCLBase.mqh remains the implementation for the DLL and // pure-MQL5 tiers, and the reference the kernels must stay line-for-line identical to. #define def_k_AccumulateWeightGrad 22 #define def_k_awg_matrix_acc 0 #define def_k_awg_matrix_g 1 #define def_k_awg_matrix_i 2 #define def_k_awg_inputs 3 //--- #define def_k_AccumulateWeightGradConv 23 #define def_k_awgc_matrix_acc 0 #define def_k_awgc_matrix_g 1 #define def_k_awgc_matrix_i 2 #define def_k_awgc_inputs 3 #define def_k_awgc_window_in 4 #define def_k_awgc_window_out 5 #define def_k_awgc_step 6 //--- #define def_k_AccumulateBufferInto 24 #define def_k_abi_dst 0 #define def_k_abi_src 1 //--- // Mini-batch APPLY. Replaces eight full-weight-matrix transfers per batch per block with one // dispatch - see the block comment above ApplyAccumAdam in AI\Network.cl for why that dominated // OpenCL training time, and why it matters more than it looks (Market builds forbid DLL imports, // so OpenCL is the tier paying clients run). #define def_k_ApplyAccumAdam 25 #define def_k_aaa_matrix_w 0 #define def_k_aaa_matrix_acc 1 #define def_k_aaa_matrix_m 2 #define def_k_aaa_matrix_v 3 #define def_k_aaa_scale 4 #define def_k_aaa_l 5 #define def_k_aaa_b1 6 #define def_k_aaa_b2 7 //--- #define def_k_ApplyAccumMomentum 26 #define def_k_aam_matrix_w 0 #define def_k_aam_matrix_acc 1 #define def_k_aam_matrix_dw 2 #define def_k_aam_scale 3 #define def_k_aam_lr 4 #define def_k_aam_momentum 5 //--- One latch for the whole process, not per net: the apply kernels either built on this machine's //--- OpenCL device or they did not, and that answer cannot change mid-run. Set false by the first //--- dispatch failure so the fallback costs one warning instead of a failed Execute every batch. //--- Deliberately NOT reset between nets - four charts share the device. bool g_applyAccumKernelUsable = true; //--- // Batch-norm kernels (2026-08-09) - see the BATCH NORM block in AI\Network.cl. Same speed-only // degradation contract as the apply kernels: without them the layer computes host-side exactly as // before. g_bnKernelUsable is the same style of process-wide latch, cleared at init when the kernels // fail to build, or at runtime by a failed dispatch OR a failed SELF-CHECK - each kernel is compared // against its host twin on first use (NeuronBatchNorm.mqh), because this layer's persisted running // statistics turn a wrong kernel into a permanently poisoned .nnw, and a check that runs once per // process is cheap insurance against exactly that. #define def_k_BatchNormForward 27 #define def_k_bnf_matrix_i 0 #define def_k_bnf_matrix_o 1 #define def_k_bnf_options 2 #define def_k_bnf_w 3 #define def_k_bnf_frozen 4 //--- #define def_k_BatchNormHiddenGrad 28 #define def_k_bnh_matrix_g 0 #define def_k_bnh_prev_o 1 #define def_k_bnh_prev_g 2 #define def_k_bnh_options 3 #define def_k_bnh_activation 4 //--- #define def_k_BatchNormAccumGammaBeta 29 #define def_k_bna_matrix_g 0 #define def_k_bna_options 1 #define def_k_bna_acc 2 //--- #define def_k_BatchNormApplyGammaBeta 30 #define def_k_bnp_options 0 #define def_k_bnp_acc 1 #define def_k_bnp_scale 2 #define def_k_bnp_lt 3 #define def_k_bnp_b1 4 #define def_k_bnp_b2 5 #define def_k_bnp_lr 6 #define def_k_bnp_momentum 7 #define def_k_bnp_optimizer 8 bool g_bnKernelUsable = true; //--- // b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see // AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era // same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root // cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE // gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever // could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled // kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam // signatures), so they're safe to expose as ordinary inputs. // Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had // already gone unstable (real collapses were happening at weight magnitudes several orders of // magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh // engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually // reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below. #define MAX_WEIGHT 100.0 // Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in // DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see // WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright // +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of // training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi- // hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. // 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates, // ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's // sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a // weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the // discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the // calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the // spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt // dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay // strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins // argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth. #define WEIGHT_DECAY 0.001 // Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full // rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden // Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras), // most likely from 5x back-to-back oversampling replay building artificially correlated momentum. // Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the // post-update weight value and is far too loose (1e6) to prevent this. #define MAX_WEIGHT_DELTA 0.1 // Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see // SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its // activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that // neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated // unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in // Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine. // 2026-07-27: Increased from 1.0e-4 to 1.0e-3 — must stay in sync with AI\Network.cl's matching // constant. See that file's comment for the full rationale (fp32 OpenCL saturation floor fix). #define MIN_ACTIVATION_DERIVATIVE 1.0e-3 // Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND // read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a // different distribution than it was trained on). The classification outputs are SIGMOID-bounded to // [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 - // the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training // can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three // outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the // ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal // modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays // (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the // loss, keeping gradient magnitudes (and thus eta tuning) unchanged. #define CLASS_LOGIT_SCALE 6.0 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "Network.cl" as string cl_program //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { NONE, TANH, SIGMOID, PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction }; //+------------------------------------------------------------------+ //| Translates ENUM_ACTIVATION to the "activation" int code every | //| native compute backend actually understands (Network.cl's | //| kernels, and the mirrored Activation()/inline switches in | //| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and | //| deliberately anything else (incl. NONE) falls through every one | //| of those switches unmatched, which is exactly linear passthrough -| //| there's no case 3 anywhere on the native side, so PRELU relies on | //| the FeedForwardConv-family kernels specifically, and NONE never | //| needs a case at all. This is NOT the same numbering as | //| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a | //| raw (int)activation cast at a kernel call site silently sends the | //| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native | //| SIGMOID). Only use this at actual kernel-dispatch call sites - | //| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw | //| ENUM_ACTIVATION value instead, and must keep using (int)activation| //| directly so saved topology files round-trip through Load() as-is. | //+------------------------------------------------------------------+ int NativeActivationCode(ENUM_ACTIVATION value) { switch(value) { case TANH: return 0; case SIGMOID: return 1; case PRELU: return 2; default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches } } //+------------------------------------------------------------------+ //| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the | //| load-time architecture repair (CNet::EnforceOutputActivation) so | //| the log names the stale value it found instead of printing a bare | //| integer nobody can decode months later. | //+------------------------------------------------------------------+ string ActivationName(ENUM_ACTIVATION value) { switch(value) { case NONE: return "NONE"; case TANH: return "TANH"; case SIGMOID: return "SIGMOID"; case PRELU: return "PRELU"; } return "UNKNOWN(" + IntegerToString((int)value) + ")"; } //--- //--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\ //--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead of //--- this AI header - without a duplicate-definition error. Whichever file is parsed first defines it; //--- the other's copy is skipped. Keep the two definitions in sync. #ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED #define WARRIOR_ENUM_OPTIMIZATION_DEFINED //--- 2026-07-28: a third DFA entry was removed. It was never Direct Feedback Alignment: its feedback //--- signal multiplied dL/dw by a DETERMINISTIC sign pattern (connectionIndex % 2), which makes half of //--- every weight tensor perform gradient ASCENT permanently - it diverges by construction, with no //--- hyperparameter able to rescue it. Real DFA (Nokland 2016) works because a FIXED RANDOM matrix gives //--- a consistent feedback direction the forward weights can align to; an index-parity sign flip has no //--- such alignment property. Its backward pass was also structurally incompatible with the OpenCL/ //--- DirectML neuron model this project actually runs on (one CNeuronBaseOCL object holds a whole layer //--- in a device buffer, so the per-neuron host-scalar loops it used saw layer.Total()==1 and updated //--- nothing). SGD/ADAM keep their ordinal values 0/1 - m_optimizationAlgo feeds the weights-filename //--- fingerprint, so these must never be renumbered. enum ENUM_OPTIMIZATION { SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras) ADAM // Adam (adaptive step, faster convergence, can overfit) }; #endif //--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronPrimitives.mqh" class CLayer; //--- class CNeuronBase : public CObject { protected: double outputVal; double prevVal; uint m_myIndex; double gradient; CArrayCon *Connections; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; int t; //--- virtual bool feedForward(CLayer *prevLayer) { return false; } virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; } virtual double activationFunction(double x); virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); } virtual double TanhFunction(double x) { return tanh(x); } virtual CLayer *getOutputLayer(void) { return NULL; } public: CNeuronBase(void); ~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back //--- what a Load() restored from disk without caring which neuron model the net was built from. virtual ENUM_ACTIVATION Activation(void) { return activation; } //--- static double alpha; //--- virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; } virtual double getOutputVal() { return outputVal; } virtual double getPrevVal() { return prevVal; } virtual void setGradient(double val) { gradient = val; } virtual double getGradient() { return gradient; } virtual CArrayCon *getConnections() { return Connections;} virtual double activationFunctionDerivative(double x); virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); } virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); } //--- virtual bool feedForward(CObject *&SourceObject); virtual bool calcHiddenGradients(CObject *&TargetObject); virtual bool updateInputWeights(CLayer *prevLayer) { return false; } virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle) { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- Forget the optimizer's trajectory memory (Adam m/v, SGD momentum, step counter) while leaving //--- the weights untouched - see CNet::ResetOptimizerState for when and why. Scope matches //--- CNet::CaptureWeights on this legacy hierarchy: the neuron's OWN connections (gate/inner-layer //--- weights of the scalar pool/LSTM subclasses are outside the checkpoint too, so resetting more //--- than the checkpoint restores would desynchronize the pair). virtual bool ResetOptimizerState(void) { t = 1; if(CheckPointer(Connections) == POINTER_INVALID) return true; for(int i = 0; i < Connections.Total(); i++) { CConnection *con = Connections.At(i); if(CheckPointer(con) == POINTER_INVALID) continue; con.deltaWeight = 0.0; con.mt = 0.0; con.vt = 0.0; } return true; } //--- virtual int Type(void) const { return defNeuronBase; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronCPU.mqh" class COpenCLMy : public COpenCL { public: COpenCLMy(void) {}; ~COpenCLMy(void) {}; template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); }; #include "NeuronDirectML.mqh" class CLayer: public CArrayObj { private: uint iOutputs; int iFileHandle; COpenCLMy *OpenCL; CDirectMLMy *DirectML; public: CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL); ~CLayer(void) {}; //--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below. bool CreateElementScaled(int const index, double weighScale); //--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool //--- CreateElement(const int index)` - because it is the real virtual override that CArrayObj::Load() //--- dispatches through, and CArrayObj::Load() is how EVERY saved layer is read back (CLayer::Load -> //--- CNet::Load). In MQL5 an override must match the base parameter list //--- exactly; adding even a DEFAULTED parameter makes it a separate method that merely hides the base //--- one, silently leaving the base's `return(false)` stub in the vtable slot. That is exactly what a //--- `double weighScale = -1.0` parameter added here did: from then on every single model load failed //--- at the first layer ("REJECTED: only loaded 0 of N layers (failed at layer 0)") no matter how //--- healthy the .nnw was, so every restart retrained from era 0 and every best-checkpoint restore //--- silently kept the current weights. Never add parameters to this method - extend //--- CreateElementScaled() and call it explicitly instead. virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); } virtual void IncreaseTotal() { m_data_total++; } virtual int Type(void) const { return defLayer; } virtual bool Load(const int file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "ArrayLayer.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPool : public CNeuronBase { protected: CLayer *OutputLayer; int iWindow; int iStep; virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); public: CNeuronPool(void) {}; ~CNeuronPool(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronPool; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronPool { protected: double param; //PReLU param virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual double activationFunction(double x); virtual bool updateInputWeights(CLayer *prevLayer); public: CNeuronConv() : param(0.01) { }; ~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; virtual double activationFunctionDerivative(double x); virtual int Type(void) const { return defNeuronConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "LayerDescription.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNet { protected: double dLogitAdjust[3]; bool bLogitAdjust; void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0); bool InitOpenCL(void); bool InitDirectML(void); //--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference. bool feedForwardCPU(CArrayDouble *inputVals); public: CNet(CArrayObj *Description); ~CNet(void); bool feedForward(CArrayDouble *inputVals); //--- sampleWeight scales this example's output-layer gradient before it propagates back through the //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same //--- example multiple times). void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0); //--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class //--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass //--- only. Minimizing softmax CE on adjusted logits is consistent for BALANCED error - which is //--- exactly the metric checkpoint selection already ranks on (macro-recall), so this is the first //--- time the loss and the selection criterion optimize the same thing. //--- It replaces minority REPLAY (which duplicated rare bars up to 28x and made Buy and Sell //--- compete for the same replicated capacity - the measured failure was each model taking one //--- direction to ~50% recall and abandoning the other, with the direction chosen arbitrarily) and //--- the post-hoc inference prior, which becomes double-counting once the offsets are trained in. //--- Offsets are NEGATIVE (log of a probability), so a rare class gets its logit pushed DOWN during //--- training, forcing the weights to produce a larger raw logit to compensate. At inference the //--- offsets are absent, so that surplus becomes the calibrated boost the rare class needs. void SetLogitAdjustment(const double &offsets[]); void ClearLogitAdjustment(void) { bLogitAdjust = false; } void getResults(CArrayDouble *&resultVals) ; double getRecentAverageError() { return recentAverageError; } //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array //--- when there is nothing to persist/restore. bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]); //--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it //--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see //--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud. bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false); //--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). CaptureWeights() snapshots //--- every neuron's weights (base/conv/LSTM) into host arrays; RestoreWeights() writes them back IN //--- PLACE via setWeights - reusing the existing neuron objects and their already-allocated device //--- buffers, exactly like BlendWeightsFrom(). This REPLACED an earlier file-based checkpoint pair //--- (since removed - it had no callers left) for the mid-run stability restore, because a file path RE-CREATES every neuron on //--- load (fresh CLayer + Init), and the multithreaded CPU-DLL backend (CDirectMLMy/WarriorCPU.dll) //--- cannot allocate a second full set of neuron tensors while the live set still exists - so the //--- file restore failed ("read 0 layers"), the model could never roll back a regressed era, and it //--- drifted into a Neutral collapse. In-place weight copy uses only getWeights/setWeights, which the //--- per-era shadow blend already exercises successfully on that backend. Snapshots WEIGHTS only (not //--- Adam moments); the regression handler decays eta on restore and per-step deltas are clipped, so //--- stale moments can't overshoot. In-memory => valid only within a single Train() run (same as the //--- ephemeral _ckpt.tmp was), which is exactly its scope. //--- Per-layer weight-norm change since the previous call - the direct test for "is this stage //--- receiving gradient at all". See the definition for why a loss curve cannot answer that. string LayerLearningReport(void); bool CaptureWeights(void); bool RestoreWeights(void); //--- MINI-BATCH CONTROL (2026-08-09 audit, F4). //--- SetBatchSize() is how training asks for accumulation; 1 restores the exact per-sample path the //--- engine used before. The REQUEST is not always granted - see BatchSize(), which refuses when the //--- backend cannot accumulate. FlushBatch() applies whatever a partial batch has accumulated and is //--- what every save/checkpoint/scoring boundary must call so no caller ever observes weights with //--- unapplied gradients behind them. void SetBatchSize(int size) { m_batchSizeRequested = (size > 1 ? size : 1); } int BatchSize(void); bool FlushBatch(void); int PendingBatchSamples(void) const { return m_batchCount; } //--- Zero every neuron's optimizer state - Adam first/second moments, SGD momentum deltas, the //--- per-neuron bias-correction step counters, and batch-norm's gamma/beta moment slots - while //--- leaving weights, activations and batch-norm running STATISTICS untouched (those belong to the //--- model, not the optimizer). 2026-08-09 audit, F3: RestoreWeights() rolls the WEIGHTS back to the //--- best checkpoint but the moments still encode the rejected trajectory, so the first updates //--- after a restore push straight back toward the state that was just rolled back - observed as the //--- restore -> regress-again -> restore oscillation. Called after every mid-run checkpoint restore //--- and at every plateau warm restart (a restart is a new schedule; stale momentum is not part of //--- it). Resetting t restarts Adam's bias-correction warm-up, whose larger early effective rate is //--- bounded by MAX_WEIGHT_DELTA like every other step. bool ResetOptimizerState(void); //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight + //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation. //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) - //--- true whenever the shadow was cloned from live via Save()/Load() and never independently //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers. bool BlendWeightsFrom(CNet &live, double tau); //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly //--- initialized and carry the real learning signal) of each output neuron's incoming weight block, //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more //--- non-majority classes than the true base rate warrants until backProp corrects it over many //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns //--- false (no-op) rather than corrupt anything if that assumption doesn't hold. bool SeedOutputLayerBias(const double &biasValues[]); //--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Set BEFORE Load() in an inference-only //--- backtest: it makes InitOpenCL()/InitDirectML() no-op (both backends stay NULL), so the OCL //--- neurons load their weights host-side only and feedForward() runs the double-precision MQL5 //--- path (feedForwardCPU) reading those same host buffers. Training/optimization never set this //--- (they always want a backend), so their behaviour is unchanged. See ExpertSignalAIBase.mqh's //--- inference-only wiring and the deploy-time validation that gates it. void SetCpuInference(bool v) { m_cpuInference = v; } bool CpuInference(void) const { return m_cpuInference; } //--- Re-assert the output layer's activation after a Load(), and report what it used to be. //--- WHY THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. CNeuronBase::Save/ //--- CNeuronBaseOCL::Save write (int)activation per neuron and the matching Load() reads it straight //--- back into the live object, so the activation chosen in BuildFreshTopology() only ever applies to //--- a genuinely NEW topology. Every reload restores whatever is on disk and the next Save() writes it //--- back out - a wrong value can never heal on its own, while the source file reads as though it were //--- already fixed. That is exactly how models kept training with an unbounded NONE classification head //--- for a full day after BuildFreshTopology() had been reverted to SIGMOID (2026-07-29): confirmed by //--- parsing the binaries, `layer N: BaseOCL act=NONE out=3`, while a freshly reset model of the same //--- config read act=SIGMOID. Symptom was negative "OOS raw out" values (impossible under sigmoid) //--- escalating to a 4.1e13 logit spread with all three classes numerically identical. //--- Only the output layer is repaired here: it is always a plain dense layer whose activation is a //--- single unambiguous expression in BuildFreshTopology(). Hidden layers are deliberately left alone - //--- they legitimately differ per stage (PRELU dense, PRELU conv, NONE pool, TANH LSTM), so blanket //--- re-assertion there would corrupt exactly the topologies it was meant to protect. //--- Returns true when a repair was actually made, and reports the stale value through `previous`. bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous); //--- Receptive field of the first conv layer as LOADED, for the stale-architecture check in //--- CExpertSignalAIBase::EnforceTopologyContract. 0 when the net has no conv layer. uint FirstConvWindow(void); //--- Freeze/unfreeze every batch-normalization layer's running statistics (AI\NeuronBatchNorm.mqh). //--- Frozen, a forward pass is a pure function of its input; unfrozen (the default) it also advances //--- the statistics. Anything that COMPARES two forward passes must freeze first or it measures its //--- own side effect. No-op on a net with no normalization layers. void SetBatchNormFrozen(bool frozen); //--- static double recentAverageSmoothingFactor; private: CArrayLayer *layers; COpenCLMy *opencl; CDirectMLMy *directml; double recentAverageError; bool m_cpuInference; //--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per //--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture. CArrayObj *m_weightSnapshot; bool m_haveWeightSnapshot; //--- Previous call's per-layer weight L2 norms, for LayerLearningReport(). Sized lazily to the layer //--- count; -1 marks "no baseline yet" so the first report says (init) instead of a bogus 0% change. double m_prevLayerNorm[]; //--- Previous call's per-layer weight VECTORS, for the |dW| term of LayerLearningReport(). One //--- CArrayDouble per layer index (empty for layers that own no weights), host-only. //--- WHY BOTH: |d|W||/|W| - the change in NORM - cannot distinguish "this layer only shrank under //--- weight decay" from "this layer moved somewhere useful". Pure decay and a genuine rotation of a //--- constant-norm weight vector can print the same number, and on 2026-07-31 the LSTM layers printed //--- a suspiciously constant ~1.05%/era while a sibling conv oscillated - a difference the norm-change //--- statistic could only hint at. |dW|/|W| - the norm of the CHANGE - separates them outright: under //--- decay alone it equals the decay rate exactly, while any gradient component adds in quadrature //--- (sqrt(decay^2 + (g/|W|)^2)). Reading them side by side is the whole diagnostic: |dW| >> |d|W|| //--- means the layer is rotating (learning); |dW| ~= |d|W|| with the norm falling means it is only //--- being decayed away. See [[feedback_verify_in_situ_not_offline]] - this is the in-situ check. CArrayObj *m_prevLayerWeights; //--- One-shot latch for BlendWeightsFrom's skip warning - it runs every era, and the condition it //--- reports is permanent, so the second print would only be noise. bool m_blendSkipLogged; //--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). //--- A run legitimately builds SEVERAL CNet objects - the main Net, the EMA shadow net, the OOS-sim //--- clone, the deploy-time MQL5-inference self-check clone - and each one probed the backends //--- independently. On a host without OpenCL that reprinted the same 3-line banner per net //--- ("OpenCL not found, error code=5100" comes from the STDLIB COpenCL::Initialize, so it can't be //--- silenced at our call site), which read in the log like the EA was initializing twice. //--- s_openclUnavailable: latched only on FAILURE, and only ever skips a probe that is already known //--- to fail - OpenCL availability cannot change inside a process. A host that HAS OpenCL never //--- latches, so every CNet still gets its own COpenCLMy. Skipping also keeps GetLastError() free of //--- the harmless 5100 for later callers. //--- s_computeTierLogged: suppresses only the repeat of the informational "tier active" line (the //--- tier is a property of the HOST, identical for every net). Failure messages stay loud every time. static bool s_openclUnavailable; static bool s_computeTierLogged; //--- Mini-batch state. m_batchSizeRequested is what training asked for; m_batchCount is how many //--- samples have been accumulated into the current batch. m_batchKernelsOk records whether this //--- net's OpenCL device actually built the accumulation kernels - false forces per-sample updates //--- rather than failing, so an old device trains exactly as it did before (see InitOpenCL). //--- m_batchBegun guards BeginBatch so the first sample of a batch zeroes the accumulators exactly //--- once. m_batchWarned latches the one-time "cannot batch on this tier" notice. int m_batchSizeRequested; int m_batchCount; bool m_batchKernelsOk; //--- Whether the DEVICE-SIDE apply kernels built (def_k_ApplyAccumAdam). Separate from //--- m_batchKernelsOk because they degrade differently: without the accumulation kernels the net //--- cannot batch at all and must fall back to per-sample updates, whereas without these it batches //--- normally and merely pays a host round-trip per batch. Conflating them would turn a missing //--- optimisation into a change of optimizer. bool m_applyKernelsOk; bool m_batchBegun; bool m_batchWarned; //--- Zero every accumulator, starting a fresh batch. bool BeginBatch(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronPool { protected: CLayer *ForgetGate; CLayer *InputGate; CLayer *OutputGate; CLayer *NewContent; CArrayDouble *Memory; CArrayDouble *PrevMemory; CArrayDouble *Input; CArrayDouble *InputGradient; //--- virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual bool updateInputWeights(CLayer *prevLayer); virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data); virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type); virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence); public: CNeuronLSTM(void); ~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = -1; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return -1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "BufferDouble.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: COpenCLMy *OpenCL; CDirectMLMy *DirectML; CBufferDouble *Output; CBufferDouble *PrevOutput; CBufferDouble *Weights; CBufferDouble *DeltaWeights; CBufferDouble *Gradient; CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; //--- MINI-BATCH ACCUMULATOR (2026-08-09 audit, F4). Same shape as Weights; holds the SUM of this //--- weight block's per-sample gradients over the current batch. Allocated lazily by //--- EnsureGradAccum() the first time a batched update runs, so a batch size of 1 (or a net that //--- only ever runs inference) pays nothing. Deliberately NOT persisted: it is transient within a //--- batch, and every save point flushes the batch first (see CNet::FlushBatch). CBufferDouble *GradAccum; //--- const double alpha; int t; //--- ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); //--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only //--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists - //--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU). bool BackendBufferCreate(CBufferDouble *buf) { if(CheckPointer(buf) == POINTER_INVALID) return false; if(CheckPointer(OpenCL) != POINTER_INVALID) return buf.BufferCreate(OpenCL); if(CheckPointer(DirectML) != POINTER_INVALID) return buf.BufferCreate(DirectML); return true; // no backend: keep host m_data, skip device allocation } //--- THE optimizer step, and the only copy of it in the batched path: one weight block, one Adam or //--- SGD+momentum update on `acc * scale`, then the accumulator is zeroed. Dense, conv and LSTM all //--- route their blocks through this, so the four compute tiers and the three layer types cannot //--- drift apart the way four separate kernel copies have in this engine before. Does NOT advance //--- t - the caller does that once per batch, after all of its blocks are stepped. //--- OpenCL fast path for the above: one dispatch, nothing crosses the bus. Returns false without //--- reporting when the kernels or buffers are not there, so the caller falls back to the host step. bool ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale, int total); bool ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale); //--- Lazily allocate the mini-batch accumulator to match `src` (Weights / WeightsConv / WeightsLSTM, //--- whichever block the caller accumulates into). Returns false when there is nothing to shadow, //--- which is the normal case for an output-layer neuron that owns no weight block at all. //--- General form - `target` is taken by reference-to-pointer so a subclass can allocate its OWN //--- accumulator member (conv's kernel block, for one) through the same code path. bool EnsureGradAccumFor(CBufferDouble *&target, CBufferDouble *src) { if(CheckPointer(src) == POINTER_INVALID || src.Total() <= 0) return false; if(CheckPointer(target) != POINTER_INVALID && target.Total() == src.Total()) return true; if(CheckPointer(target) != POINTER_INVALID) delete target; target = new CBufferDouble(); if(CheckPointer(target) == POINTER_INVALID) return false; if(!target.BufferInit(src.Total(), 0.0)) return false; return BackendBufferCreate(target); } bool EnsureGradAccum(CBufferDouble *src) { return EnsureGradAccumFor(GradAccum, src); } public: CNeuronBaseOCL(void); ~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- virtual int getOutputIndex(void) { return Output.GetIndex(); } virtual int getPrevOutIndex(void) { return PrevOutput.GetIndex(); } virtual int getGradientIndex(void) { return Gradient.GetIndex(); } virtual int getWeightsIndex(void) { return Weights.GetIndex(); } virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); } virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); } virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();} //--- virtual int getOutputVal(double &values[]) { return Output.GetData(values); } virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); } virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); } virtual int getGradient(double &values[]) { return Gradient.GetData(values); } //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same //--- buffer to propagate into the hidden layers. virtual bool setGradient(const double &values[]) { int count = ArraySize(values); for(int i = 0; i < count; i++) if(!Gradient.Update(i, values[i])) return false; return Gradient.BufferWrite(); } // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created, // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here. virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); } // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment // (see that method's declaration comment) - writes a full replacement weight array back to this // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element, // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment // then pushes it to the device - the shape callers use when blending two already-read-out arrays. virtual bool setWeights(double &values[]) { if(CheckPointer(Weights) == POINTER_INVALID) return false; if(!Weights.AssignArray(values)) return false; return Weights.BufferWrite(); } //--- Forget the optimizer's trajectory memory: Adam first/second moments, SGD's previous-delta //--- buffer, and the bias-correction step counter, weights untouched. Subclasses that own extra //--- weight blocks (conv/LSTM/batch-norm) extend this with their own moment storage. See //--- CNet::ResetOptimizerState for the restore/warm-restart rationale. virtual bool ResetOptimizerState(void) { t = 1; bool ok = ZeroOptimizerBuffer(FirstMomentum); ok = ZeroOptimizerBuffer(SecondMomentum) && ok; ok = ZeroOptimizerBuffer(DeltaWeights) && ok; return ok; } virtual int Neurons(void) { return Output.Total(); } virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); } //--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to //--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/ //--- getOutputVal() which route through BufferRead() (a device read that fails without a backend). double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; } double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; } int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; } //--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only //--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense //--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math. virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); //--- Host-side input write for the CPU inference path's layer 0, and host-side output read for //--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates. bool SetInputsCPU(CArrayDouble *inputVals); int GetOutputsCPU(CArrayDouble *values); //--- virtual bool feedForward(CObject *SourceObject); virtual bool calcHiddenGradients(CObject *TargetObject); virtual bool calcOutputGradients(CArrayDouble *Target); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CObject *SourceObject); //--- MINI-BATCH PAIR (2026-08-09 audit, F4). accumulateInputWeightGrads() adds THIS sample's //--- per-weight gradient into GradAccum without touching the weights; ApplyAccumulatedGradients() //--- then takes ONE optimizer step on the batch mean and clears the accumulator. Together they are //--- the batched equivalent of updateInputWeights(), and at batch size 1 they must reproduce it //--- exactly - the accumulation kernels are copied from the Adam update kernels for that reason. //--- The dispatch pair mirrors updateInputWeights: the accumulate half is native (it is the O(n^2) //--- outer product); the apply half is a kernel on OpenCL and host-side MQL5 everywhere else. It was //--- host-side on ALL tiers until 2026-08-09, so that the four backends shared one optimizer //--- implementation - a good instinct that turned out to cost eight full weight-matrix transfers per //--- batch per block, which made the GPU tier several times slower than a CPU thread pool. Since //--- Market builds forbid DLL imports, that tier is the one clients run. The host copy remains the //--- reference; ApplyAccumOnDevice is transcribed from it and the two must be edited together. virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL); virtual bool accumulateInputWeightGrads(CObject *SourceObject); virtual bool ApplyAccumulatedGradients(double scale); //--- Zero the accumulator at the start of a batch. Separate from ApplyAccumulatedGradients so a //--- discarded partial batch (a stopped run) can be cleared without taking a step from it. virtual bool BeginGradAccum(void); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBaseOCL; } }; //+------------------------------------------------------------------+ #include "NeuronOCLConvPool.mqh" #include "NeuronBatchNorm.mqh" //+------------------------------------------------------------------+ //--- Marks a .nnw LSTM record as the SEQUENCE format. Chosen so it cannot collide with any value the //--- pre-sequence format could have written in that slot (an input width, i.e. -1 or a positive count). #define LSTM_SEQ_SAVE_TAG (-424242) //--- Initial bias of the FORGET gate (gate 0). Every other weight starts near zero; this one must not. //--- See the long rationale and the measured numbers at its use in CNeuronLSTMOCL::SetInputs - in //--- short, a zero bias means sigmoid(0)=0.5, which halves the cell state every bar and leaves a //--- 20-bar window with the memory and the gradient reach of a single bar. #define LSTM_FORGET_BIAS_INIT (1.0) //+------------------------------------------------------------------+ //| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from | //| scratch from the standard LSTM equations - NOT ported from the | //| NeuroNet_DNG reference (see the note above the LSTM kernels in | //| Network.cl for why). Single-timestep-truncated BPTT: gradient | //| does not flow back into h_prev/c_prev from an earlier step. | //| Adam-only - Init fails for any other optimization type. | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: int m_iInputs; //--- Mini-batch running total of WeightsGradient across the samples of one batch - see the note on //--- accumulateInputWeightGrads below for why WeightsGradient itself cannot serve as it. CBufferDouble *GradAccumLSTM; //--- Sequence shape. m_iStepInputs is the width of ONE timestep (the per-bar feature count reaching //--- this layer), set from the layer descriptor by SetStepWidth() before the first feedForward; //--- m_iSteps is then m_iInputs / m_iStepInputs. When m_iStepInputs <= 0 the layer falls back to the //--- LEGACY single-timestep behaviour (whole input as one step), which is what every .nnw written //--- before the sequence rewrite contains. int m_iStepInputs; int m_iSteps; //--- Per-timestep caches, required by backpropagation-through-time: the backward pass needs each //--- step's gate activations, cell state and hidden state, which the single-buffer Concatenated/ //--- Memory/HiddenCache trio below cannot hold because every step overwrites the last. CBufferDouble *CacheGates; // T * 4H, gate order [f,i,o,g] CBufferDouble *CacheCell; // T * H, c_t CBufferDouble *CacheHidden; // T * H, h_t CBufferDouble *WeightsLSTM; CBufferDouble *FirstMomentumLSTM; CBufferDouble *SecondMomentumLSTM; CBufferDouble *DeltaWeightsLSTM; CBufferDouble *WeightsGradient; CBufferDouble *Concatenated; CBufferDouble *ConcatenatedGradient; CBufferDouble *Memory; CBufferDouble *HiddenCache; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool SetInputs(int count); bool AllocateSequenceCaches(void); public: CNeuronLSTMOCL(void) : m_iInputs(-1), m_iStepInputs(-1), m_iSteps(-1) { GradAccumLSTM = NULL; CacheGates = NULL; CacheCell = NULL; CacheHidden = NULL; WeightsLSTM = NULL; FirstMomentumLSTM = NULL; SecondMomentumLSTM = NULL; DeltaWeightsLSTM = NULL; WeightsGradient = NULL; Concatenated = NULL; ConcatenatedGradient = NULL; Memory = NULL; HiddenCache = NULL; } ~CNeuronLSTMOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); //--- Per-timestep input width, from CLayerDescription::window (see AddLstmStage). Must be called //--- between Init() and the first feedForward; <= 0 keeps the legacy single-timestep behaviour. //--- Not persisted from here - Save/Load carry it, so a loaded model does not depend on call order. void SetStepWidth(int stepInputs) { m_iStepInputs = (stepInputs > 0 ? stepInputs : -1); } bool IsSequenceMode(void) const { return (m_iStepInputs > 0 && m_iSteps > 1); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTMOCL; } // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment. virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); } //--- Build this layer's weight block to match `src`, for a net that was cloned from one whose LSTM //--- had not yet run a forward pass. Save() writes m_iInputs = -1 and omits EVERY LSTM buffer in that //--- state, so the clone comes back with WeightsLSTM == NULL - and if that clone is an EMA shadow, //--- which only ever receives BlendWeightsFrom and never runs forward itself, nothing would ever //--- allocate it. m_iStepInputs must be copied FIRST: SetInputs reads it to decide whether the block //--- is 4H(H + stepInputs + 1) (sequence) or 4H(H + inputs + 1) (single timestep). virtual bool AdoptShapeFrom(CNeuronLSTMOCL &src) { if(src.m_iInputs <= 0 || Neurons() != src.Neurons()) return false; m_iStepInputs = src.m_iStepInputs; return SetInputs(src.m_iInputs); } virtual bool setWeightsLSTM(double &values[]) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; if(!WeightsLSTM.AssignArray(values)) return false; return WeightsLSTM.BufferWrite(); } //--- The LSTM's gate-weight block keeps its own moment/momentum buffers beside the base class's - //--- see CNet::ResetOptimizerState. virtual bool ResetOptimizerState(void) { bool ok = CNeuronBaseOCL::ResetOptimizerState(); ok = ZeroOptimizerBuffer(FirstMomentumLSTM) && ok; ok = ZeroOptimizerBuffer(SecondMomentumLSTM) && ok; ok = ZeroOptimizerBuffer(DeltaWeightsLSTM) && ok; return ok; } //--- MINI-BATCH. This layer needs no outer-product accumulation kernel of its own: by the time the //--- update pass reaches it, WeightsGradient already holds THIS sample's complete dW, summed over //--- the T timesteps of BPTT by calcInputGradients. Batching it is therefore just a running total. //--- It cannot be done by leaving WeightsGradient un-zeroed between samples, which was the obvious //--- approach and is wrong: CPU_LSTMSeqBackward/DML_LSTMSeqBackward memset that buffer at the top of //--- every call, so the DLL would wipe the batch on each sample no matter what the host did. Hence a //--- separate accumulator plus a generic elementwise add (AccumulateBufferInto), which also avoids //--- changing the signature of an already-deployed export. virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL); virtual bool BeginGradAccum(void) { bool ok = CNeuronBaseOCL::BeginGradAccum(); if(CheckPointer(GradAccumLSTM) != POINTER_INVALID && GradAccumLSTM.Total() > 0) ok = ZeroOptimizerBuffer(GradAccumLSTM) && ok; return ok; } virtual bool ApplyAccumulatedGradients(double scale) { bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale); ok = ApplyAccumToBlock(WeightsLSTM, GradAccumLSTM, FirstMomentumLSTM, SecondMomentumLSTM, DeltaWeightsLSTM, scale) && ok; if(optimization == ADAM) t++; return ok; } }; //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| Implementation bodies. | //| | //| Everything above this line is DECLARATIONS - the nine classes | //| plus the include chain that orders them (each nested include | //| sits exactly where its base class becomes visible, so the order | //| here is a dependency graph, not a preference). | //| | //| Everything below is method BODIES, grouped by the class they | //| belong to. They were interleaved with the declarations in one | //| 6266-line file; splitting them out is behaviour-neutral by | //| construction, since a body cannot run during compilation and | //| every declaration it could need is already visible above. | //+------------------------------------------------------------------+ #include "Impl\NeuronBase.mqh" #include "Impl\NeuronConvPool.mqh" #include "Impl\NeuronLSTM.mqh" #include "Impl\Layer.mqh" #include "Impl\NetBuild.mqh" #include "Impl\NetForward.mqh" #include "Impl\NetPersistence.mqh" #include "Impl\NetWeights.mqh" #include "Impl\NeuronOCLBase.mqh" #include "Impl\NeuronOCLLSTM.mqh" //+------------------------------------------------------------------+