//+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include #include #include #include #include "..\System\AtomicFile.mqh" //--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available, //--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Sizes //--- WarriorCPU.dll's worker thread pool - no effect at all when a GPU tier (OpenCL or DirectML) is //--- active, since neither one calls into WarriorCPU.dll. //--- //--- A FIXED SMALL THREAD COUNT PER NETWORK, not a share of the machine. This replaced a TargetCPULoad //--- input divided by the live chart count, which was wrong twice over: //--- - The count is a SNAPSHOT taken when each net's pool is built, and charts are attached one at a //--- time. Measured 2026-07-29 with five charts: they took 10/6/5/4/4% of the same budget, because //--- the first chart only ever saw itself and the last saw all five. So the earliest chart got //--- several times the threads of the latest - which silently skews any cross-topology comparison //--- run on those charts, the exact thing the setting existed to make fair. //--- - Nothing rebalances when a chart is added or removed, and rebalancing would mean tearing down a //--- DLL context underneath a running trainer. //--- Neither problem exists once the answer stops depending on how many charts are running. //--- //--- Why 2 threads is not a compromise: since the topology became data-derived the widest dense layer //--- is 64 units (ExpertSignalAIBase.mqh's ComputeFirstLayerWidth), so each ParallelFor has almost //--- nothing to split and per-dispatch overhead dominates. The measurements agree - an MLP era cost //--- ~66s at a wildly oversubscribed 12 threads and ~80s at 1 thread, a 20% spread across a 12x //--- difference in thread count. Two per net also lands six concurrent charts exactly on a 12-core box. //--- //--- Removing the input costs nothing on the product side: a Market build has no DLL tier at all (MQL5 //--- Market rule IV strips the #import blocks - see AI\NeuronDirectML.mqh), so it was already compiled //--- out to a constant there and no buyer could ever reach it. #define CPU_THREADS_PER_NETWORK 2 //+------------------------------------------------------------------+ //| The percentage CDirectMLMy needs in order to land on | //| CPU_THREADS_PER_NETWORK workers, given the detected core count. | //| (WarriorCPU.dll takes a percentage of cores, not a thread count.) | //+------------------------------------------------------------------+ int EffectiveCpuLoadPercent() { int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES); //--- Unknown core count: assume a small machine rather than a large one. Guessing high here would //--- reintroduce exactly the oversubscription this function exists to prevent. if(cores <= 0) cores = 4; //--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the //--- DLL, so this is the one value that needs no arithmetic. if(cores <= CPU_THREADS_PER_NETWORK) return 100; //--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread //--- short of the target is a worse error than landing one over. int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores); return (int)MathMax(1, MathMin(100, pct)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference //--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's //--- abstract "0.001" mention, which the book's own worked examples don't actually use either. //--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005) //--- this project had separately tuned lr to before this input existed, so no behavior conflict. //--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak //--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to //--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this //--- reverts to the literature/book default. input double AdamLearningRate = 0.0003; // Adam learning rate input double AdamBeta1 = 0.9; // Adam beta1 input double AdamBeta2 = 0.999; // Adam beta2 //--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method //--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses //--- Adam's book-default rate as its starting point too. The book also states no numeric default for //--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses //--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for. input double SgdLearningRate = 0.0003; // SGD learning rate input double SgdMomentum = 0.9; // SGD momentum #define lr AdamLearningRate #define b1 AdamBeta1 #define b2 AdamBeta2 #define momentum SgdMomentum double eta = lr; #define defConnect 0x7781 #define defArrayConnects 0x7782 #define defNeuronBase 0x7783 #define defNeuron 0x7784 #define defNeuronConv 0x7785 #define defNeuronPool 0x7786 #define defLayer 0x7787 #define defArrayLayer 0x7788 #define defNet 0x7789 #define defNeuronLSTM 0x7791 //--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like //--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps //--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh. #define defNeuronBatchNorm 0x7792 //--- #define defBufferDouble 0x7882 #define defNeuronBaseOCL 0x7883 #define defNeuronLSTMOCL 0x7884 #define defNeuronConvOCL 0x7885 #define defNeuronPoolOCL 0x7886 #define defNeuronBatchNormOCL 0x7887 //--- #define def_k_FeedForward 0 #define def_k_ff_matrix_w 0 #define def_k_ff_matrix_i 1 #define def_k_ff_matrix_o 2 #define def_k_ff_inputs 3 #define def_k_ff_activation 4 //--- #define def_k_CaclOutputGradient 1 #define def_k_cog_matrix_t 0 #define def_k_cog_matrix_o 1 #define def_k_cog_matrix_ig 2 #define def_k_cog_activation 3 //--- #define def_k_CaclHiddenGradient 2 #define def_k_chg_matrix_w 0 #define def_k_chg_matrix_g 1 #define def_k_chg_matrix_o 2 #define def_k_chg_matrix_ig 3 #define def_k_chg_outputs 4 #define def_k_chg_activation 5 //--- #define def_k_UpdateWeightsMomentum 3 #define def_k_uwm_matrix_w 0 #define def_k_uwm_matrix_g 1 #define def_k_uwm_matrix_i 2 #define def_k_uwm_matrix_dw 3 #define def_k_uwm_inputs 4 #define def_k_uwm_learning_rates 5 #define def_k_uwm_momentum 6 #define def_k_uwm_optimizer 7 //--- #define def_k_UpdateWeightsAdam 4 #define def_k_uwa_matrix_w 0 #define def_k_uwa_matrix_g 1 #define def_k_uwa_matrix_i 2 #define def_k_uwa_matrix_m 3 #define def_k_uwa_matrix_v 4 #define def_k_uwa_inputs 5 #define def_k_uwa_l 6 #define def_k_uwa_b1 7 #define def_k_uwa_b2 8 //--- #define def_k_FeedForwardProof 15 #define def_k_ffp_matrix_i 0 #define def_k_ffp_matrix_o 1 #define def_k_ffp_inputs 2 #define def_k_ffp_window 3 #define def_k_ffp_step 4 //--- #define def_k_CalcInputGradientProof 16 #define def_k_cigp_matrix_i 0 #define def_k_cigp_matrix_g 1 #define def_k_cigp_matrix_o 2 #define def_k_cigp_matrix_ig 3 #define def_k_cigp_outputs 4 #define def_k_cigp_window 5 #define def_k_cigp_step 6 //--- #define def_k_FeedForwardConv 5 #define def_k_ffc_matrix_w 0 #define def_k_ffc_matrix_i 1 #define def_k_ffc_matrix_o 2 #define def_k_ffc_inputs 3 #define def_k_ffc_step 4 #define def_k_ffc_window_in 5 #define def_k_ffc_window_out 6 #define def_k_ffc_activation 7 //--- #define def_k_CalcHiddenGradientConv 6 #define def_k_chgc_matrix_w 0 #define def_k_chgc_matrix_g 1 #define def_k_chgc_matrix_o 2 #define def_k_chgc_matrix_ig 3 #define def_k_chgc_outputs 4 #define def_k_chgc_step 5 #define def_k_chgc_window_in 6 #define def_k_chgc_window_out 7 #define def_k_chgc_activation 8 //--- #define def_k_UpdateWeightsConvMomentum 7 #define def_k_uwcm_matrix_w 0 #define def_k_uwcm_matrix_g 1 #define def_k_uwcm_matrix_i 2 #define def_k_uwcm_matrix_dw 3 #define def_k_uwcm_inputs 4 #define def_k_uwcm_learning_rates 5 #define def_k_uwcm_momentum 6 #define def_k_uwcm_window_in 7 #define def_k_uwcm_window_out 8 #define def_k_uwcm_step 9 #define def_k_uwcm_optimizer 10 //--- #define def_k_UpdateWeightsConvAdam 8 #define def_k_uwca_matrix_w 0 #define def_k_uwca_matrix_g 1 #define def_k_uwca_matrix_i 2 #define def_k_uwca_matrix_m 3 #define def_k_uwca_matrix_v 4 #define def_k_uwca_inputs 5 #define def_k_uwca_l 6 #define def_k_uwca_b1 7 #define def_k_uwca_b2 8 #define def_k_uwca_window_in 9 #define def_k_uwca_window_out 10 #define def_k_uwca_step 11 //--- // LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient // flows back into h_prev/c_prev from a prior step). Supports both Adam and // SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) - // see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch. // Derived from scratch from the standard LSTM equations - NOT ported from // the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites // the live weights buffer instead of writing to weights_gradient. #define def_k_LSTM_Gates 9 #define def_k_lstmg_matrix_w 0 #define def_k_lstmg_hidden_prev 1 #define def_k_lstmg_inputs 2 #define def_k_lstmg_concatenated 3 #define def_k_lstmg_hidden_size 4 #define def_k_lstmg_input_size 5 //--- #define def_k_LSTM_State 10 #define def_k_lstms_concatenated 0 #define def_k_lstms_memory 1 #define def_k_lstms_hidden_prev 2 #define def_k_lstms_hidden_cache 3 #define def_k_lstms_output 4 #define def_k_lstms_hidden_size 5 //--- #define def_k_LSTM_GateGradient 11 #define def_k_lstmgg_gradient 0 #define def_k_lstmgg_memory 1 #define def_k_lstmgg_concatenated 2 #define def_k_lstmgg_concatenated_gradient 3 #define def_k_lstmgg_hidden_size 4 //--- #define def_k_LSTM_WeightsGradient 12 #define def_k_lstmwg_concatenated_gradient 0 #define def_k_lstmwg_hidden_cache 1 #define def_k_lstmwg_inputs 2 #define def_k_lstmwg_weights_gradient 3 #define def_k_lstmwg_hidden_size 4 #define def_k_lstmwg_input_size 5 //--- #define def_k_LSTM_InputsGradient 13 #define def_k_lstmig_concatenated_gradient 0 #define def_k_lstmig_matrix_w 1 #define def_k_lstmig_inputs_gradient 2 #define def_k_lstmig_hidden_size 3 #define def_k_lstmig_input_size 4 //--- #define def_k_LSTM_UpdateWeightsAdam 14 #define def_k_lstmuwa_matrix_w 0 #define def_k_lstmuwa_weights_gradient 1 #define def_k_lstmuwa_matrix_m 2 #define def_k_lstmuwa_matrix_v 3 #define def_k_lstmuwa_l 4 #define def_k_lstmuwa_b1 5 #define def_k_lstmuwa_b2 6 //--- // SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see // AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body. #define def_k_LSTM_UpdateWeightsMomentum 17 #define def_k_lstmuwm_matrix_w 0 #define def_k_lstmuwm_weights_gradient 1 #define def_k_lstmuwm_matrix_dw 2 #define def_k_lstmuwm_learning_rates 3 #define def_k_lstmuwm_momentum 4 #define def_k_lstmuwm_optimizer 5 //--- // Sequence LSTM. One launch PER TIMESTEP - the recurrence is sequential and OpenCL // barriers only span a work-group, so the host loop is what provides the global // ordering. See the block comment above LSTM_SeqStepForward in AI\Network.cl. #define def_k_LSTM_SeqStepForward 18 #define def_k_lsf_matrix_w 0 #define def_k_lsf_inputs 1 #define def_k_lsf_cache_gates 2 #define def_k_lsf_cache_cell 3 #define def_k_lsf_cache_hidden 4 #define def_k_lsf_output 5 #define def_k_lsf_hidden_size 6 #define def_k_lsf_step_inputs 7 #define def_k_lsf_steps 8 #define def_k_lsf_t 9 //--- #define def_k_LSTM_SeqStepGateGrad 19 #define def_k_lsgg_out_gradient 0 #define def_k_lsgg_dh_buf 1 #define def_k_lsgg_dc_buf 2 #define def_k_lsgg_cache_gates 3 #define def_k_lsgg_cache_cell 4 #define def_k_lsgg_gate_grad 5 #define def_k_lsgg_hidden_size 6 #define def_k_lsgg_steps 7 #define def_k_lsgg_t 8 //--- #define def_k_LSTM_SeqStepWeightGrad 20 #define def_k_lswg_gate_grad 0 #define def_k_lswg_cache_hidden 1 #define def_k_lswg_inputs 2 #define def_k_lswg_weights_gradient 3 #define def_k_lswg_hidden_size 4 #define def_k_lswg_step_inputs 5 #define def_k_lswg_t 6 //--- #define def_k_LSTM_SeqStepInputGrad 21 #define def_k_lsig_gate_grad 0 #define def_k_lsig_matrix_w 1 #define def_k_lsig_inputs_gradient 2 #define def_k_lsig_dh_buf 3 #define def_k_lsig_hidden_size 4 #define def_k_lsig_step_inputs 5 #define def_k_lsig_t 6 //--- // b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see // AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era // same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root // cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE // gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever // could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled // kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam // signatures), so they're safe to expose as ordinary inputs. // Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had // already gone unstable (real collapses were happening at weight magnitudes several orders of // magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh // engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually // reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below. #define MAX_WEIGHT 100.0 // Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in // DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see // WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright // +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of // training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi- // hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. // 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates, // ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's // sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a // weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the // discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the // calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the // spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt // dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay // strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins // argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth. #define WEIGHT_DECAY 0.001 // Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full // rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden // Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras), // most likely from 5x back-to-back oversampling replay building artificially correlated momentum. // Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the // post-update weight value and is far too loose (1e6) to prevent this. #define MAX_WEIGHT_DELTA 0.1 // Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see // SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its // activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that // neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated // unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in // Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine. // 2026-07-27: Increased from 1.0e-4 to 1.0e-3 — must stay in sync with AI\Network.cl's matching // constant. See that file's comment for the full rationale (fp32 OpenCL saturation floor fix). #define MIN_ACTIVATION_DERIVATIVE 1.0e-3 // Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND // read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a // different distribution than it was trained on). The classification outputs are SIGMOID-bounded to // [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 - // the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training // can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three // outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the // ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal // modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays // (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the // loss, keeping gradient magnitudes (and thus eta tuning) unchanged. #define CLASS_LOGIT_SCALE 6.0 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "Network.cl" as string cl_program //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { NONE, TANH, SIGMOID, PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction }; //+------------------------------------------------------------------+ //| Translates ENUM_ACTIVATION to the "activation" int code every | //| native compute backend actually understands (Network.cl's | //| kernels, and the mirrored Activation()/inline switches in | //| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and | //| deliberately anything else (incl. NONE) falls through every one | //| of those switches unmatched, which is exactly linear passthrough -| //| there's no case 3 anywhere on the native side, so PRELU relies on | //| the FeedForwardConv-family kernels specifically, and NONE never | //| needs a case at all. This is NOT the same numbering as | //| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a | //| raw (int)activation cast at a kernel call site silently sends the | //| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native | //| SIGMOID). Only use this at actual kernel-dispatch call sites - | //| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw | //| ENUM_ACTIVATION value instead, and must keep using (int)activation| //| directly so saved topology files round-trip through Load() as-is. | //+------------------------------------------------------------------+ int NativeActivationCode(ENUM_ACTIVATION value) { switch(value) { case TANH: return 0; case SIGMOID: return 1; case PRELU: return 2; default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches } } //+------------------------------------------------------------------+ //| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the | //| load-time architecture repair (CNet::EnforceOutputActivation) so | //| the log names the stale value it found instead of printing a bare | //| integer nobody can decode months later. | //+------------------------------------------------------------------+ string ActivationName(ENUM_ACTIVATION value) { switch(value) { case NONE: return "NONE"; case TANH: return "TANH"; case SIGMOID: return "SIGMOID"; case PRELU: return "PRELU"; } return "UNKNOWN(" + IntegerToString((int)value) + ")"; } //--- //--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\ //--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead of //--- this AI header - without a duplicate-definition error. Whichever file is parsed first defines it; //--- the other's copy is skipped. Keep the two definitions in sync. #ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED #define WARRIOR_ENUM_OPTIMIZATION_DEFINED //--- 2026-07-28: a third DFA entry was removed. It was never Direct Feedback Alignment: its feedback //--- signal multiplied dL/dw by a DETERMINISTIC sign pattern (connectionIndex % 2), which makes half of //--- every weight tensor perform gradient ASCENT permanently - it diverges by construction, with no //--- hyperparameter able to rescue it. Real DFA (Nokland 2016) works because a FIXED RANDOM matrix gives //--- a consistent feedback direction the forward weights can align to; an index-parity sign flip has no //--- such alignment property. Its backward pass was also structurally incompatible with the OpenCL/ //--- DirectML neuron model this project actually runs on (one CNeuronBaseOCL object holds a whole layer //--- in a device buffer, so the per-neuron host-scalar loops it used saw layer.Total()==1 and updated //--- nothing). SGD/ADAM keep their ordinal values 0/1 - m_optimizationAlgo feeds the weights-filename //--- fingerprint, so these must never be renumbered. enum ENUM_OPTIMIZATION { SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras) ADAM // Adam (adaptive step, faster convergence, can overfit) }; #endif //--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronPrimitives.mqh" class CLayer; //--- class CNeuronBase : public CObject { protected: double outputVal; double prevVal; uint m_myIndex; double gradient; CArrayCon *Connections; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; int t; //--- virtual bool feedForward(CLayer *prevLayer) { return false; } virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; } virtual double activationFunction(double x); virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); } virtual double TanhFunction(double x) { return tanh(x); } virtual CLayer *getOutputLayer(void) { return NULL; } public: CNeuronBase(void); ~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back //--- what a Load() restored from disk without caring which neuron model the net was built from. virtual ENUM_ACTIVATION Activation(void) { return activation; } //--- static double alpha; //--- virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; } virtual double getOutputVal() { return outputVal; } virtual double getPrevVal() { return prevVal; } virtual void setGradient(double val) { gradient = val; } virtual double getGradient() { return gradient; } virtual CArrayCon *getConnections() { return Connections;} virtual double activationFunctionDerivative(double x); virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); } virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); } //--- virtual bool feedForward(CObject *&SourceObject); virtual bool calcHiddenGradients(CObject *&TargetObject); virtual bool updateInputWeights(CLayer *prevLayer) { return false; } virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle) { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- virtual int Type(void) const { return defNeuronBase; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronCPU.mqh" class COpenCLMy : public COpenCL { public: COpenCLMy(void) {}; ~COpenCLMy(void) {}; template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); }; #include "NeuronDirectML.mqh" class CLayer: public CArrayObj { private: uint iOutputs; int iFileHandle; COpenCLMy *OpenCL; CDirectMLMy *DirectML; public: CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL); ~CLayer(void) {}; //--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below. bool CreateElementScaled(int const index, double weighScale); //--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool //--- CreateElement(const int index)` - because it is the real virtual override that CArrayObj::Load() //--- dispatches through, and CArrayObj::Load() is how EVERY saved layer is read back (CLayer::Load -> //--- CNet::Load). In MQL5 an override must match the base parameter list //--- exactly; adding even a DEFAULTED parameter makes it a separate method that merely hides the base //--- one, silently leaving the base's `return(false)` stub in the vtable slot. That is exactly what a //--- `double weighScale = -1.0` parameter added here did: from then on every single model load failed //--- at the first layer ("REJECTED: only loaded 0 of N layers (failed at layer 0)") no matter how //--- healthy the .nnw was, so every restart retrained from era 0 and every best-checkpoint restore //--- silently kept the current weights. Never add parameters to this method - extend //--- CreateElementScaled() and call it explicitly instead. virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); } virtual void IncreaseTotal() { m_data_total++; } virtual int Type(void) const { return defLayer; } virtual bool Load(const int file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "ArrayLayer.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPool : public CNeuronBase { protected: CLayer *OutputLayer; int iWindow; int iStep; virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); public: CNeuronPool(void) {}; ~CNeuronPool(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronPool; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronPool { protected: double param; //PReLU param virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual double activationFunction(double x); virtual bool updateInputWeights(CLayer *prevLayer); public: CNeuronConv() : param(0.01) { }; ~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; virtual double activationFunctionDerivative(double x); virtual int Type(void) const { return defNeuronConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "LayerDescription.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNet { protected: double dLogitAdjust[3]; bool bLogitAdjust; void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0); bool InitOpenCL(void); bool InitDirectML(void); //--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference. bool feedForwardCPU(CArrayDouble *inputVals); public: CNet(CArrayObj *Description); ~CNet(void); bool feedForward(CArrayDouble *inputVals); //--- sampleWeight scales this example's output-layer gradient before it propagates back through the //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same //--- example multiple times). void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0); //--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class //--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass //--- only. Minimizing softmax CE on adjusted logits is consistent for BALANCED error - which is //--- exactly the metric checkpoint selection already ranks on (macro-recall), so this is the first //--- time the loss and the selection criterion optimize the same thing. //--- It replaces minority REPLAY (which duplicated rare bars up to 28x and made Buy and Sell //--- compete for the same replicated capacity - the measured failure was each model taking one //--- direction to ~50% recall and abandoning the other, with the direction chosen arbitrarily) and //--- the post-hoc inference prior, which becomes double-counting once the offsets are trained in. //--- Offsets are NEGATIVE (log of a probability), so a rare class gets its logit pushed DOWN during //--- training, forcing the weights to produce a larger raw logit to compensate. At inference the //--- offsets are absent, so that surplus becomes the calibrated boost the rare class needs. void SetLogitAdjustment(const double &offsets[]); void ClearLogitAdjustment(void) { bLogitAdjust = false; } void getResults(CArrayDouble *&resultVals) ; double getRecentAverageError() { return recentAverageError; } //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array //--- when there is nothing to persist/restore. bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]); //--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it //--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see //--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud. bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false); //--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). CaptureWeights() snapshots //--- every neuron's weights (base/conv/LSTM) into host arrays; RestoreWeights() writes them back IN //--- PLACE via setWeights - reusing the existing neuron objects and their already-allocated device //--- buffers, exactly like BlendWeightsFrom(). This REPLACED an earlier file-based checkpoint pair //--- (since removed - it had no callers left) for the mid-run stability restore, because a file path RE-CREATES every neuron on //--- load (fresh CLayer + Init), and the multithreaded CPU-DLL backend (CDirectMLMy/WarriorCPU.dll) //--- cannot allocate a second full set of neuron tensors while the live set still exists - so the //--- file restore failed ("read 0 layers"), the model could never roll back a regressed era, and it //--- drifted into a Neutral collapse. In-place weight copy uses only getWeights/setWeights, which the //--- per-era shadow blend already exercises successfully on that backend. Snapshots WEIGHTS only (not //--- Adam moments); the regression handler decays eta on restore and per-step deltas are clipped, so //--- stale moments can't overshoot. In-memory => valid only within a single Train() run (same as the //--- ephemeral _ckpt.tmp was), which is exactly its scope. //--- Per-layer weight-norm change since the previous call - the direct test for "is this stage //--- receiving gradient at all". See the definition for why a loss curve cannot answer that. string LayerLearningReport(void); bool CaptureWeights(void); bool RestoreWeights(void); //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight + //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation. //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) - //--- true whenever the shadow was cloned from live via Save()/Load() and never independently //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers. bool BlendWeightsFrom(CNet &live, double tau); //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly //--- initialized and carry the real learning signal) of each output neuron's incoming weight block, //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more //--- non-majority classes than the true base rate warrants until backProp corrects it over many //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns //--- false (no-op) rather than corrupt anything if that assumption doesn't hold. bool SeedOutputLayerBias(const double &biasValues[]); //--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Set BEFORE Load() in an inference-only //--- backtest: it makes InitOpenCL()/InitDirectML() no-op (both backends stay NULL), so the OCL //--- neurons load their weights host-side only and feedForward() runs the double-precision MQL5 //--- path (feedForwardCPU) reading those same host buffers. Training/optimization never set this //--- (they always want a backend), so their behaviour is unchanged. See ExpertSignalAIBase.mqh's //--- inference-only wiring and the deploy-time validation that gates it. void SetCpuInference(bool v) { m_cpuInference = v; } bool CpuInference(void) const { return m_cpuInference; } //--- Re-assert the output layer's activation after a Load(), and report what it used to be. //--- WHY THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. CNeuronBase::Save/ //--- CNeuronBaseOCL::Save write (int)activation per neuron and the matching Load() reads it straight //--- back into the live object, so the activation chosen in BuildFreshTopology() only ever applies to //--- a genuinely NEW topology. Every reload restores whatever is on disk and the next Save() writes it //--- back out - a wrong value can never heal on its own, while the source file reads as though it were //--- already fixed. That is exactly how models kept training with an unbounded NONE classification head //--- for a full day after BuildFreshTopology() had been reverted to SIGMOID (2026-07-29): confirmed by //--- parsing the binaries, `layer N: BaseOCL act=NONE out=3`, while a freshly reset model of the same //--- config read act=SIGMOID. Symptom was negative "OOS raw out" values (impossible under sigmoid) //--- escalating to a 4.1e13 logit spread with all three classes numerically identical. //--- Only the output layer is repaired here: it is always a plain dense layer whose activation is a //--- single unambiguous expression in BuildFreshTopology(). Hidden layers are deliberately left alone - //--- they legitimately differ per stage (PRELU dense, PRELU conv, NONE pool, TANH LSTM), so blanket //--- re-assertion there would corrupt exactly the topologies it was meant to protect. //--- Returns true when a repair was actually made, and reports the stale value through `previous`. bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous); //--- Receptive field of the first conv layer as LOADED, for the stale-architecture check in //--- CExpertSignalAIBase::EnforceTopologyContract. 0 when the net has no conv layer. uint FirstConvWindow(void); //--- Freeze/unfreeze every batch-normalization layer's running statistics (AI\NeuronBatchNorm.mqh). //--- Frozen, a forward pass is a pure function of its input; unfrozen (the default) it also advances //--- the statistics. Anything that COMPARES two forward passes must freeze first or it measures its //--- own side effect. No-op on a net with no normalization layers. void SetBatchNormFrozen(bool frozen); //--- static double recentAverageSmoothingFactor; private: CArrayLayer *layers; COpenCLMy *opencl; CDirectMLMy *directml; double recentAverageError; bool m_cpuInference; //--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per //--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture. CArrayObj *m_weightSnapshot; bool m_haveWeightSnapshot; //--- Previous call's per-layer weight L2 norms, for LayerLearningReport(). Sized lazily to the layer //--- count; -1 marks "no baseline yet" so the first report says (init) instead of a bogus 0% change. double m_prevLayerNorm[]; //--- Previous call's per-layer weight VECTORS, for the |dW| term of LayerLearningReport(). One //--- CArrayDouble per layer index (empty for layers that own no weights), host-only. //--- WHY BOTH: |d|W||/|W| - the change in NORM - cannot distinguish "this layer only shrank under //--- weight decay" from "this layer moved somewhere useful". Pure decay and a genuine rotation of a //--- constant-norm weight vector can print the same number, and on 2026-07-31 the LSTM layers printed //--- a suspiciously constant ~1.05%/era while a sibling conv oscillated - a difference the norm-change //--- statistic could only hint at. |dW|/|W| - the norm of the CHANGE - separates them outright: under //--- decay alone it equals the decay rate exactly, while any gradient component adds in quadrature //--- (sqrt(decay^2 + (g/|W|)^2)). Reading them side by side is the whole diagnostic: |dW| >> |d|W|| //--- means the layer is rotating (learning); |dW| ~= |d|W|| with the norm falling means it is only //--- being decayed away. See [[feedback_verify_in_situ_not_offline]] - this is the in-situ check. CArrayObj *m_prevLayerWeights; //--- One-shot latch for BlendWeightsFrom's skip warning - it runs every era, and the condition it //--- reports is permanent, so the second print would only be noise. bool m_blendSkipLogged; //--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). //--- A run legitimately builds SEVERAL CNet objects - the main Net, the EMA shadow net, the OOS-sim //--- clone, the deploy-time MQL5-inference self-check clone - and each one probed the backends //--- independently. On a host without OpenCL that reprinted the same 3-line banner per net //--- ("OpenCL not found, error code=5100" comes from the STDLIB COpenCL::Initialize, so it can't be //--- silenced at our call site), which read in the log like the EA was initializing twice. //--- s_openclUnavailable: latched only on FAILURE, and only ever skips a probe that is already known //--- to fail - OpenCL availability cannot change inside a process. A host that HAS OpenCL never //--- latches, so every CNet still gets its own COpenCLMy. Skipping also keeps GetLastError() free of //--- the harmless 5100 for later callers. //--- s_computeTierLogged: suppresses only the repeat of the informational "tier active" line (the //--- tier is a property of the HOST, identical for every net). Failure messages stay loud every time. static bool s_openclUnavailable; static bool s_computeTierLogged; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronPool { protected: CLayer *ForgetGate; CLayer *InputGate; CLayer *OutputGate; CLayer *NewContent; CArrayDouble *Memory; CArrayDouble *PrevMemory; CArrayDouble *Input; CArrayDouble *InputGradient; //--- virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual bool updateInputWeights(CLayer *prevLayer); virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data); virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type); virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence); public: CNeuronLSTM(void); ~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = -1; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return -1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "BufferDouble.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: COpenCLMy *OpenCL; CDirectMLMy *DirectML; CBufferDouble *Output; CBufferDouble *PrevOutput; CBufferDouble *Weights; CBufferDouble *DeltaWeights; CBufferDouble *Gradient; CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; //--- const double alpha; int t; //--- ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); //--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only //--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists - //--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU). bool BackendBufferCreate(CBufferDouble *buf) { if(CheckPointer(buf) == POINTER_INVALID) return false; if(CheckPointer(OpenCL) != POINTER_INVALID) return buf.BufferCreate(OpenCL); if(CheckPointer(DirectML) != POINTER_INVALID) return buf.BufferCreate(DirectML); return true; // no backend: keep host m_data, skip device allocation } public: CNeuronBaseOCL(void); ~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- virtual int getOutputIndex(void) { return Output.GetIndex(); } virtual int getPrevOutIndex(void) { return PrevOutput.GetIndex(); } virtual int getGradientIndex(void) { return Gradient.GetIndex(); } virtual int getWeightsIndex(void) { return Weights.GetIndex(); } virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); } virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); } virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();} //--- virtual int getOutputVal(double &values[]) { return Output.GetData(values); } virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); } virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); } virtual int getGradient(double &values[]) { return Gradient.GetData(values); } //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same //--- buffer to propagate into the hidden layers. virtual bool setGradient(const double &values[]) { int count = ArraySize(values); for(int i = 0; i < count; i++) if(!Gradient.Update(i, values[i])) return false; return Gradient.BufferWrite(); } // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created, // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here. virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); } // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment // (see that method's declaration comment) - writes a full replacement weight array back to this // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element, // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment // then pushes it to the device - the shape callers use when blending two already-read-out arrays. virtual bool setWeights(double &values[]) { if(CheckPointer(Weights) == POINTER_INVALID) return false; if(!Weights.AssignArray(values)) return false; return Weights.BufferWrite(); } virtual int Neurons(void) { return Output.Total(); } virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); } //--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to //--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/ //--- getOutputVal() which route through BufferRead() (a device read that fails without a backend). double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; } double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; } int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; } //--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only //--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense //--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math. virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); //--- Host-side input write for the CPU inference path's layer 0, and host-side output read for //--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates. bool SetInputsCPU(CArrayDouble *inputVals); int GetOutputsCPU(CArrayDouble *values); //--- virtual bool feedForward(CObject *SourceObject); virtual bool calcHiddenGradients(CObject *TargetObject); virtual bool calcOutputGradients(CArrayDouble *Target); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBaseOCL; } }; //+------------------------------------------------------------------+ #include "NeuronOCLConvPool.mqh" #include "NeuronBatchNorm.mqh" //+------------------------------------------------------------------+ //--- Marks a .nnw LSTM record as the SEQUENCE format. Chosen so it cannot collide with any value the //--- pre-sequence format could have written in that slot (an input width, i.e. -1 or a positive count). #define LSTM_SEQ_SAVE_TAG (-424242) //--- Initial bias of the FORGET gate (gate 0). Every other weight starts near zero; this one must not. //--- See the long rationale and the measured numbers at its use in CNeuronLSTMOCL::SetInputs - in //--- short, a zero bias means sigmoid(0)=0.5, which halves the cell state every bar and leaves a //--- 20-bar window with the memory and the gradient reach of a single bar. #define LSTM_FORGET_BIAS_INIT (1.0) //+------------------------------------------------------------------+ //| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from | //| scratch from the standard LSTM equations - NOT ported from the | //| NeuroNet_DNG reference (see the note above the LSTM kernels in | //| Network.cl for why). Single-timestep-truncated BPTT: gradient | //| does not flow back into h_prev/c_prev from an earlier step. | //| Adam-only - Init fails for any other optimization type. | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: int m_iInputs; //--- Sequence shape. m_iStepInputs is the width of ONE timestep (the per-bar feature count reaching //--- this layer), set from the layer descriptor by SetStepWidth() before the first feedForward; //--- m_iSteps is then m_iInputs / m_iStepInputs. When m_iStepInputs <= 0 the layer falls back to the //--- LEGACY single-timestep behaviour (whole input as one step), which is what every .nnw written //--- before the sequence rewrite contains. int m_iStepInputs; int m_iSteps; //--- Per-timestep caches, required by backpropagation-through-time: the backward pass needs each //--- step's gate activations, cell state and hidden state, which the single-buffer Concatenated/ //--- Memory/HiddenCache trio below cannot hold because every step overwrites the last. CBufferDouble *CacheGates; // T * 4H, gate order [f,i,o,g] CBufferDouble *CacheCell; // T * H, c_t CBufferDouble *CacheHidden; // T * H, h_t CBufferDouble *WeightsLSTM; CBufferDouble *FirstMomentumLSTM; CBufferDouble *SecondMomentumLSTM; CBufferDouble *DeltaWeightsLSTM; CBufferDouble *WeightsGradient; CBufferDouble *Concatenated; CBufferDouble *ConcatenatedGradient; CBufferDouble *Memory; CBufferDouble *HiddenCache; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool SetInputs(int count); bool AllocateSequenceCaches(void); public: CNeuronLSTMOCL(void) : m_iInputs(-1), m_iStepInputs(-1), m_iSteps(-1) { CacheGates = NULL; CacheCell = NULL; CacheHidden = NULL; WeightsLSTM = NULL; FirstMomentumLSTM = NULL; SecondMomentumLSTM = NULL; DeltaWeightsLSTM = NULL; WeightsGradient = NULL; Concatenated = NULL; ConcatenatedGradient = NULL; Memory = NULL; HiddenCache = NULL; } ~CNeuronLSTMOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); //--- Per-timestep input width, from CLayerDescription::window (see AddLstmStage). Must be called //--- between Init() and the first feedForward; <= 0 keeps the legacy single-timestep behaviour. //--- Not persisted from here - Save/Load carry it, so a loaded model does not depend on call order. void SetStepWidth(int stepInputs) { m_iStepInputs = (stepInputs > 0 ? stepInputs : -1); } bool IsSequenceMode(void) const { return (m_iStepInputs > 0 && m_iSteps > 1); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTMOCL; } // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment. virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); } //--- Build this layer's weight block to match `src`, for a net that was cloned from one whose LSTM //--- had not yet run a forward pass. Save() writes m_iInputs = -1 and omits EVERY LSTM buffer in that //--- state, so the clone comes back with WeightsLSTM == NULL - and if that clone is an EMA shadow, //--- which only ever receives BlendWeightsFrom and never runs forward itself, nothing would ever //--- allocate it. m_iStepInputs must be copied FIRST: SetInputs reads it to decide whether the block //--- is 4H(H + stepInputs + 1) (sequence) or 4H(H + inputs + 1) (single timestep). virtual bool AdoptShapeFrom(CNeuronLSTMOCL &src) { if(src.m_iInputs <= 0 || Neurons() != src.Neurons()) return false; m_iStepInputs = src.m_iStepInputs; return SetInputs(src.m_iInputs); } virtual bool setWeightsLSTM(double &values[]) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; if(!WeightsLSTM.AssignArray(values)) return false; return WeightsLSTM.BufferWrite(); } }; //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| Implementation bodies. | //| | //| Everything above this line is DECLARATIONS - the nine classes | //| plus the include chain that orders them (each nested include | //| sits exactly where its base class becomes visible, so the order | //| here is a dependency graph, not a preference). | //| | //| Everything below is method BODIES, grouped by the class they | //| belong to. They were interleaved with the declarations in one | //| 6266-line file; splitting them out is behaviour-neutral by | //| construction, since a body cannot run during compilation and | //| every declaration it could need is already visible above. | //+------------------------------------------------------------------+ #include "Impl\NeuronBase.mqh" #include "Impl\NeuronConvPool.mqh" #include "Impl\NeuronLSTM.mqh" #include "Impl\Layer.mqh" #include "Impl\NetBuild.mqh" #include "Impl\NetForward.mqh" #include "Impl\NetPersistence.mqh" #include "Impl\NetWeights.mqh" #include "Impl\NeuronOCLBase.mqh" #include "Impl\NeuronOCLLSTM.mqh" //+------------------------------------------------------------------+