//+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include #include #include #include #include "..\System\AtomicFile.mqh" //--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available, //--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). #define CPU_THREADS_PER_NETWORK 2 //+------------------------------------------------------------------+ //| The percentage CDirectMLMy needs in order to land on | //| CPU_THREADS_PER_NETWORK workers, given the detected core count. | //| (WarriorCPU.dll takes a percentage of cores, not a thread count.) | //+------------------------------------------------------------------+ int EffectiveCpuLoadPercent() { int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES); //--- Unknown core count: assume a small machine rather than a large one. Guessing high here would //--- reintroduce exactly the oversubscription this function exists to prevent. if(cores <= 0) cores = 4; //--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the //--- DLL, so this is the one value that needs no arithmetic. if(cores <= CPU_THREADS_PER_NETWORK) return 100; //--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread //--- short of the target is a worse error than landing one over. int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores); return (int)MathMax(1, MathMin(100, pct)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //--- Adam (Kingma & Ba, 2014) hyperparameters. 3.0e-4 also happens to sit inside the noisy/non- //--- stationary-trading-data range (0.0003-0.0005) this project had separately tuned lr to before //--- this input existed, so no behavior conflict. input double AdamLearningRate = 0.0003; // Adam learning rate input double AdamBeta1 = 0.9; // Adam beta1 input double AdamBeta2 = 0.999; // Adam beta2 //--- SGD+momentum hyperparameters. input double SgdLearningRate = 0.0003; // SGD learning rate input double SgdMomentum = 0.9; // SGD momentum //--- Live learning rate: starts at the Adam input, then decayed and restored by the training loop, //--- so it is a mutable global rather than a constant. //--- g_-PREFIXED, and it has to be: as a bare `eta` this collided with a local of the same name in //--- the standard library's Math\Stat\Math.mqh (the incomplete-gamma branch), which the compiler //--- reports as "declaration of 'eta' hides global variable". Same fault as the b1/b2/lr/momentum //--- macros retired in ea2552e - a single-token global name in a header that library code is //--- compiled beside. The ETA_DECAY_FACTOR/ETA_MIN/m_etaCeiling vocabulary around it is unchanged, //--- so the symbol still reads as the learning rate everywhere it appears. double g_eta = AdamLearningRate; #define defConnect 0x7781 #define defArrayConnects 0x7782 #define defNeuronBase 0x7783 #define defNeuron 0x7784 #define defNeuronConv 0x7785 #define defNeuronPool 0x7786 #define defLayer 0x7787 #define defArrayLayer 0x7788 #define defNet 0x7789 #define defNeuronLSTM 0x7791 //--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like //--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps //--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh. #define defNeuronBatchNorm 0x7792 //--- #define defBufferDouble 0x7882 #define defNeuronBaseOCL 0x7883 #define defNeuronLSTMOCL 0x7884 #define defNeuronConvOCL 0x7885 #define defNeuronPoolOCL 0x7886 #define defNeuronBatchNormOCL 0x7887 //--- #define def_k_FeedForward 0 #define def_k_ff_matrix_w 0 #define def_k_ff_matrix_i 1 #define def_k_ff_matrix_o 2 #define def_k_ff_inputs 3 #define def_k_ff_activation 4 //--- #define def_k_CaclOutputGradient 1 #define def_k_cog_matrix_t 0 #define def_k_cog_matrix_o 1 #define def_k_cog_matrix_ig 2 #define def_k_cog_activation 3 //--- #define def_k_CaclHiddenGradient 2 #define def_k_chg_matrix_w 0 #define def_k_chg_matrix_g 1 #define def_k_chg_matrix_o 2 #define def_k_chg_matrix_ig 3 #define def_k_chg_outputs 4 #define def_k_chg_activation 5 //--- #define def_k_UpdateWeightsMomentum 3 #define def_k_uwm_matrix_w 0 #define def_k_uwm_matrix_g 1 #define def_k_uwm_matrix_i 2 #define def_k_uwm_matrix_dw 3 #define def_k_uwm_inputs 4 #define def_k_uwm_learning_rates 5 #define def_k_uwm_momentum 6 #define def_k_uwm_optimizer 7 //--- #define def_k_UpdateWeightsAdam 4 #define def_k_uwa_matrix_w 0 #define def_k_uwa_matrix_g 1 #define def_k_uwa_matrix_i 2 #define def_k_uwa_matrix_m 3 #define def_k_uwa_matrix_v 4 #define def_k_uwa_inputs 5 #define def_k_uwa_l 6 #define def_k_uwa_b1 7 #define def_k_uwa_b2 8 //--- #define def_k_FeedForwardProof 15 #define def_k_ffp_matrix_i 0 #define def_k_ffp_matrix_o 1 #define def_k_ffp_inputs 2 #define def_k_ffp_window 3 #define def_k_ffp_step 4 //--- #define def_k_CalcInputGradientProof 16 #define def_k_cigp_matrix_i 0 #define def_k_cigp_matrix_g 1 #define def_k_cigp_matrix_o 2 #define def_k_cigp_matrix_ig 3 #define def_k_cigp_outputs 4 #define def_k_cigp_window 5 #define def_k_cigp_step 6 //--- #define def_k_FeedForwardConv 5 #define def_k_ffc_matrix_w 0 #define def_k_ffc_matrix_i 1 #define def_k_ffc_matrix_o 2 #define def_k_ffc_inputs 3 #define def_k_ffc_step 4 #define def_k_ffc_window_in 5 #define def_k_ffc_window_out 6 #define def_k_ffc_activation 7 //--- #define def_k_CalcHiddenGradientConv 6 #define def_k_chgc_matrix_w 0 #define def_k_chgc_matrix_g 1 #define def_k_chgc_matrix_o 2 #define def_k_chgc_matrix_ig 3 #define def_k_chgc_outputs 4 #define def_k_chgc_step 5 #define def_k_chgc_window_in 6 #define def_k_chgc_window_out 7 #define def_k_chgc_activation 8 //--- #define def_k_UpdateWeightsConvMomentum 7 #define def_k_uwcm_matrix_w 0 #define def_k_uwcm_matrix_g 1 #define def_k_uwcm_matrix_i 2 #define def_k_uwcm_matrix_dw 3 #define def_k_uwcm_inputs 4 #define def_k_uwcm_learning_rates 5 #define def_k_uwcm_momentum 6 #define def_k_uwcm_window_in 7 #define def_k_uwcm_window_out 8 #define def_k_uwcm_step 9 #define def_k_uwcm_optimizer 10 //--- #define def_k_UpdateWeightsConvAdam 8 #define def_k_uwca_matrix_w 0 #define def_k_uwca_matrix_g 1 #define def_k_uwca_matrix_i 2 #define def_k_uwca_matrix_m 3 #define def_k_uwca_matrix_v 4 #define def_k_uwca_inputs 5 #define def_k_uwca_l 6 #define def_k_uwca_b1 7 #define def_k_uwca_b2 8 #define def_k_uwca_window_in 9 #define def_k_uwca_window_out 10 #define def_k_uwca_step 11 //--- //--- LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient flows back into //--- h_prev/c_prev from a prior step). #define def_k_LSTM_Gates 9 #define def_k_lstmg_matrix_w 0 #define def_k_lstmg_hidden_prev 1 #define def_k_lstmg_inputs 2 #define def_k_lstmg_concatenated 3 #define def_k_lstmg_hidden_size 4 #define def_k_lstmg_input_size 5 //--- #define def_k_LSTM_State 10 #define def_k_lstms_concatenated 0 #define def_k_lstms_memory 1 #define def_k_lstms_hidden_prev 2 #define def_k_lstms_hidden_cache 3 #define def_k_lstms_output 4 #define def_k_lstms_hidden_size 5 //--- #define def_k_LSTM_GateGradient 11 #define def_k_lstmgg_gradient 0 #define def_k_lstmgg_memory 1 #define def_k_lstmgg_concatenated 2 #define def_k_lstmgg_concatenated_gradient 3 #define def_k_lstmgg_hidden_size 4 //--- #define def_k_LSTM_WeightsGradient 12 #define def_k_lstmwg_concatenated_gradient 0 #define def_k_lstmwg_hidden_cache 1 #define def_k_lstmwg_inputs 2 #define def_k_lstmwg_weights_gradient 3 #define def_k_lstmwg_hidden_size 4 #define def_k_lstmwg_input_size 5 //--- #define def_k_LSTM_InputsGradient 13 #define def_k_lstmig_concatenated_gradient 0 #define def_k_lstmig_matrix_w 1 #define def_k_lstmig_inputs_gradient 2 #define def_k_lstmig_hidden_size 3 #define def_k_lstmig_input_size 4 //--- #define def_k_LSTM_UpdateWeightsAdam 14 #define def_k_lstmuwa_matrix_w 0 #define def_k_lstmuwa_weights_gradient 1 #define def_k_lstmuwa_matrix_m 2 #define def_k_lstmuwa_matrix_v 3 #define def_k_lstmuwa_l 4 #define def_k_lstmuwa_b1 5 #define def_k_lstmuwa_b2 6 //--- // SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see // AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body. #define def_k_LSTM_UpdateWeightsMomentum 17 #define def_k_lstmuwm_matrix_w 0 #define def_k_lstmuwm_weights_gradient 1 #define def_k_lstmuwm_matrix_dw 2 #define def_k_lstmuwm_learning_rates 3 #define def_k_lstmuwm_momentum 4 #define def_k_lstmuwm_optimizer 5 //--- // Sequence LSTM. One launch PER TIMESTEP - the recurrence is sequential and OpenCL // barriers only span a work-group, so the host loop is what provides the global // ordering. See the block comment above LSTM_SeqStepForward in AI\Network.cl. #define def_k_LSTM_SeqStepForward 18 #define def_k_lsf_matrix_w 0 #define def_k_lsf_inputs 1 #define def_k_lsf_cache_gates 2 #define def_k_lsf_cache_cell 3 #define def_k_lsf_cache_hidden 4 #define def_k_lsf_output 5 #define def_k_lsf_hidden_size 6 #define def_k_lsf_step_inputs 7 #define def_k_lsf_steps 8 #define def_k_lsf_t 9 //--- #define def_k_LSTM_SeqStepGateGrad 19 #define def_k_lsgg_out_gradient 0 #define def_k_lsgg_dh_buf 1 #define def_k_lsgg_dc_buf 2 #define def_k_lsgg_cache_gates 3 #define def_k_lsgg_cache_cell 4 #define def_k_lsgg_gate_grad 5 #define def_k_lsgg_hidden_size 6 #define def_k_lsgg_steps 7 #define def_k_lsgg_t 8 //--- #define def_k_LSTM_SeqStepWeightGrad 20 #define def_k_lswg_gate_grad 0 #define def_k_lswg_cache_hidden 1 #define def_k_lswg_inputs 2 #define def_k_lswg_weights_gradient 3 #define def_k_lswg_hidden_size 4 #define def_k_lswg_step_inputs 5 #define def_k_lswg_t 6 //--- #define def_k_LSTM_SeqStepInputGrad 21 #define def_k_lsig_gate_grad 0 #define def_k_lsig_matrix_w 1 #define def_k_lsig_inputs_gradient 2 #define def_k_lsig_dh_buf 3 #define def_k_lsig_hidden_size 4 #define def_k_lsig_step_inputs 5 #define def_k_lsig_t 6 //--- //--- Mini-batch gradient accumulation - see the block comment above AccumulateWeightGrad in //--- AI\Network.cl. #define def_k_AccumulateWeightGrad 22 #define def_k_awg_matrix_acc 0 #define def_k_awg_matrix_g 1 #define def_k_awg_matrix_i 2 #define def_k_awg_inputs 3 //--- #define def_k_AccumulateWeightGradConv 23 #define def_k_awgc_matrix_acc 0 #define def_k_awgc_matrix_g 1 #define def_k_awgc_matrix_i 2 #define def_k_awgc_inputs 3 #define def_k_awgc_window_in 4 #define def_k_awgc_window_out 5 #define def_k_awgc_step 6 //--- #define def_k_AccumulateBufferInto 24 #define def_k_abi_dst 0 #define def_k_abi_src 1 //--- //--- Mini-batch APPLY. #define def_k_ApplyAccumAdam 25 #define def_k_aaa_matrix_w 0 #define def_k_aaa_matrix_acc 1 #define def_k_aaa_matrix_m 2 #define def_k_aaa_matrix_v 3 #define def_k_aaa_scale 4 #define def_k_aaa_l 5 #define def_k_aaa_b1 6 #define def_k_aaa_b2 7 //--- #define def_k_ApplyAccumMomentum 26 #define def_k_aam_matrix_w 0 #define def_k_aam_matrix_acc 1 #define def_k_aam_matrix_dw 2 #define def_k_aam_scale 3 #define def_k_aam_lr 4 #define def_k_aam_momentum 5 //--- One latch for the whole process, not per net: the apply kernels either built on this machine's //--- OpenCL device or they did not, and that answer cannot change mid-run. bool g_applyAccumKernelUsable = true; //--- //--- Batch-norm kernels (2026-08-09) - see the BATCH NORM block in AI\Network.cl. #define def_k_BatchNormForward 27 #define def_k_bnf_matrix_i 0 #define def_k_bnf_matrix_o 1 #define def_k_bnf_options 2 #define def_k_bnf_w 3 #define def_k_bnf_frozen 4 //--- #define def_k_BatchNormHiddenGrad 28 #define def_k_bnh_matrix_g 0 #define def_k_bnh_prev_o 1 #define def_k_bnh_prev_g 2 #define def_k_bnh_options 3 #define def_k_bnh_activation 4 //--- #define def_k_BatchNormAccumGammaBeta 29 #define def_k_bna_matrix_g 0 #define def_k_bna_options 1 #define def_k_bna_acc 2 //--- #define def_k_BatchNormApplyGammaBeta 30 #define def_k_bnp_options 0 #define def_k_bnp_acc 1 #define def_k_bnp_scale 2 #define def_k_bnp_lt 3 #define def_k_bnp_b1 4 #define def_k_bnp_b2 5 #define def_k_bnp_lr 6 #define def_k_bnp_momentum 7 #define def_k_bnp_optimizer 8 bool g_bnKernelUsable = true; //--- // The Adam betas are ordinary inputs (book defaults 0.9/0.999) - see // AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era // same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root // cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE // gradient in backProp()/backPropOCL(), which addresses it more directly than shortening beta1 ever // could. Both are passed as runtime parameters into every backend (not baked into compiled // kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam // signatures), so they're safe to expose as ordinary inputs. // Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had // already gone unstable (real collapses were happening at weight magnitudes several orders of // magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh // engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually // reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below. #define MAX_WEIGHT 100.0 // Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in // DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see // WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright // +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of // training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi- // hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. // 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates, // ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's // sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a // weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the // discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the // calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the // spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt // dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay // strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins // argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth. #define WEIGHT_DECAY 0.001 // Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full // rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden // Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras), // most likely from 5x back-to-back oversampling replay building artificially correlated momentum. // Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the // post-update weight value and is far too loose (1e6) to prevent this. #define MAX_WEIGHT_DELTA 0.1 //--- Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see //--- SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near //--- its activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes //--- that neuron's entire backprop gradient contribution regardless of how wrong its output is. #define MIN_ACTIVATION_DERIVATIVE 1.0e-3 //--- Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND //--- read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored //--- against a different distribution than it was trained on). #define CLASS_LOGIT_SCALE 6.0 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "Network.cl" as string cl_program //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { NONE, TANH, SIGMOID, PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction }; //+------------------------------------------------------------------+ //| Translates ENUM_ACTIVATION to the "activation" int code every | //| native compute backend actually understands (Network.cl's | //| kernels, and the mirrored Activation()/inline switches in | //| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, | //| and deliberately anything else (incl. | //+------------------------------------------------------------------+ int NativeActivationCode(ENUM_ACTIVATION value) { switch(value) { case TANH: return 0; case SIGMOID: return 1; case PRELU: return 2; default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches } } //+------------------------------------------------------------------+ //| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the | //| load-time architecture repair (CNet::EnforceOutputActivation) so | //| the log names the stale value it found instead of printing a bare | //| integer nobody can decode months later. | //+------------------------------------------------------------------+ string ActivationName(ENUM_ACTIVATION value) { switch(value) { case NONE: return "NONE"; case TANH: return "TANH"; case SIGMOID: return "SIGMOID"; case PRELU: return "PRELU"; } return "UNKNOWN(" + IntegerToString((int)value) + ")"; } //--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\ //--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead //--- of this AI header - without a duplicate-definition error. #ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED #define WARRIOR_ENUM_OPTIMIZATION_DEFINED //--- 2026-07-28: a third DFA entry was removed. SGD/ADAM keep their ordinal values 0/1 - //--- m_optimizationAlgo feeds the weights-filename fingerprint, so these must never be renumbered. enum ENUM_OPTIMIZATION { SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras) ADAM // Adam (adaptive step, faster convergence, can overfit) }; #endif //--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronPrimitives.mqh" class CLayer; //--- class CNeuronBase : public CObject { protected: double outputVal; double prevVal; uint m_myIndex; double gradient; CArrayCon *Connections; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; int t; //--- virtual bool feedForward(CLayer *prevLayer) { return false; } virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; } virtual double activationFunction(double x); virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); } virtual double TanhFunction(double x) { return tanh(x); } virtual CLayer *getOutputLayer(void) { return NULL; } public: CNeuronBase(void); ~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back //--- what a Load() restored from disk without caring which neuron model the net was built from. virtual ENUM_ACTIVATION Activation(void) { return activation; } //--- static double alpha; //--- virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; } virtual double getOutputVal() { return outputVal; } virtual double getPrevVal() { return prevVal; } virtual void setGradient(double val) { gradient = val; } virtual double getGradient() { return gradient; } virtual CArrayCon *getConnections() { return Connections;} virtual double activationFunctionDerivative(double x); virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); } virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); } //--- virtual bool feedForward(CObject *&SourceObject); virtual bool calcHiddenGradients(CObject *&TargetObject); virtual bool updateInputWeights(CLayer *prevLayer) { return false; } virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle) { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- Forget the optimizer's trajectory memory (Adam m/v, SGD momentum, step counter) while //--- leaving the weights untouched - see CNet::ResetOptimizerState for when and why. virtual bool ResetOptimizerState(void) { t = 1; if(CheckPointer(Connections) == POINTER_INVALID) return true; for(int i = 0; i < Connections.Total(); i++) { CConnection *con = Connections.At(i); if(CheckPointer(con) == POINTER_INVALID) continue; con.deltaWeight = 0.0; con.mt = 0.0; con.vt = 0.0; } return true; } //--- virtual int Type(void) const { return defNeuronBase; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronCPU.mqh" class COpenCLMy : public COpenCL { public: COpenCLMy(void) {}; ~COpenCLMy(void) {}; template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); }; #include "NeuronDirectML.mqh" class CLayer: public CArrayObj { private: uint iOutputs; int iFileHandle; COpenCLMy *OpenCL; CDirectMLMy *DirectML; public: CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL); ~CLayer(void) {}; //--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below. bool CreateElementScaled(int const index, double weighScale); //--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool //--- CreateElement(const int index)` - because it is the real virtual override that //--- CArrayObj::Load() dispatches through, and CArrayObj::Load() is how EVERY saved layer is //--- read back (CLayer::Load -> CNet::Load). virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); } virtual void IncreaseTotal() { m_data_total++; } virtual int Type(void) const { return defLayer; } virtual bool Load(const int file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "ArrayLayer.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPool : public CNeuronBase { protected: CLayer *OutputLayer; int iWindow; int iStep; virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); public: CNeuronPool(void) {}; ~CNeuronPool(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronPool; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronPool { protected: double param; //PReLU param virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual double activationFunction(double x); virtual bool updateInputWeights(CLayer *prevLayer); public: CNeuronConv() : param(0.01) { }; ~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; virtual double activationFunctionDerivative(double x); virtual int Type(void) const { return defNeuronConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "LayerDescription.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNet { protected: double dLogitAdjust[3]; bool bLogitAdjust; void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0); bool InitOpenCL(void); bool InitDirectML(void); //--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference. bool feedForwardCPU(CArrayDouble *inputVals); public: CNet(CArrayObj *Description); ~CNet(void); bool feedForward(CArrayDouble *inputVals); //--- sampleWeight scales this example's output-layer gradient before it propagates back through the //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same //--- example multiple times). void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0); //--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class //--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass //--- only. void SetLogitAdjustment(const double &offsets[]); void ClearLogitAdjustment(void) { bLogitAdjust = false; } void getResults(CArrayDouble *&resultVals) ; double getRecentAverageError() { return recentAverageError; } //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array //--- when there is nothing to persist/restore. bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]); //--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it //--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see //--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud. bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false); //--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). In-place weight copy //--- uses only getWeights/setWeights, which the per-era shadow blend already exercises //--- successfully on that backend. string LayerLearningReport(void); bool CaptureWeights(void); bool RestoreWeights(void); //--- MINI-BATCH CONTROL (2026-08-09 audit, F4). SetBatchSize() is how training asks for //--- accumulation; 1 restores the exact per-sample path the engine used before. void SetBatchSize(int size) { m_batchSizeRequested = (size > 1 ? size : 1); } int BatchSize(void); bool FlushBatch(void); //--- Zero every neuron's optimizer state - Adam first/second moments, SGD momentum deltas, the //--- per-neuron bias-correction step counters, and batch-norm's gamma/beta moment slots - while //--- leaving weights, activations and batch-norm running STATISTICS untouched (those belong to //--- the model, not the optimizer). bool ResetOptimizerState(void); //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight + //--- tau*live.weight. bool BlendWeightsFrom(CNet &live, double tau); //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly //--- initialized and carry the real learning signal) of each output neuron's incoming weight block, //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more //--- non-majority classes than the true base rate warrants until backProp corrects it over many //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns //--- false (no-op) rather than corrupt anything if that assumption doesn't hold. bool SeedOutputLayerBias(const double &biasValues[]); //--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Training/optimization never set this //--- (they always want a backend), so their behaviour is unchanged. void SetCpuInference(bool v) { m_cpuInference = v; } bool CpuInference(void) const { return m_cpuInference; } //--- Re-assert the output layer's activation after a Load(), and report what it used to be. WHY //--- THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. Every reload restores //--- whatever is on disk and the next Save() writes it back out - a wrong value can never heal //--- on its own, while the source file reads as though it were already fixed. bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous); //--- Receptive field of the first conv layer as LOADED, for the stale-architecture check in //--- CExpertSignalAIBase::EnforceTopologyContract. 0 when the net has no conv layer. uint FirstConvWindow(void); //--- Freeze/unfreeze every batch-normalization layer's running statistics //--- (AI\NeuronBatchNorm.mqh). Frozen, a forward pass is a pure function of its input; unfrozen //--- (the default) it also advances the statistics. void SetBatchNormFrozen(bool frozen); //--- Current freeze state (the first normalization layer's flag; they only ever move together). //--- False on a net with no normalization layers. bool GetBatchNormFrozen(void); //--- static double recentAverageSmoothingFactor; private: CArrayLayer *layers; COpenCLMy *opencl; CDirectMLMy *directml; double recentAverageError; bool m_cpuInference; //--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per //--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture. CArrayObj *m_weightSnapshot; bool m_haveWeightSnapshot; //--- Previous call's per-layer weight L2 norms, for LayerLearningReport(). Sized lazily to the layer //--- count; -1 marks "no baseline yet" so the first report says (init) instead of a bogus 0% change. double m_prevLayerNorm[]; //--- Previous call's per-layer weight VECTORS, for the |dW| term of LayerLearningReport(). One //--- CArrayDouble per layer index (empty for layers that own no weights), host-only. CArrayObj *m_prevLayerWeights; //--- One-shot latch for BlendWeightsFrom's skip warning - it runs every era, and the condition it //--- reports is permanent, so the second print would only be noise. bool m_blendSkipLogged; //--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). A host //--- that HAS OpenCL never latches, so every CNet still gets its own COpenCLMy. Failure messages //--- stay loud every time. static bool s_openclUnavailable; static bool s_computeTierLogged; //--- Mini-batch state. m_batchSizeRequested is what training asked for; m_batchCount is how many //--- samples have been accumulated into the current batch. m_batchKernelsOk records whether this //--- net's OpenCL device actually built the accumulation kernels - false forces per-sample updates //--- rather than failing, so an old device trains exactly as it did before (see InitOpenCL). //--- m_batchBegun guards BeginBatch so the first sample of a batch zeroes the accumulators exactly //--- once. m_batchWarned latches the one-time "cannot batch on this tier" notice. int m_batchSizeRequested; int m_batchCount; bool m_batchKernelsOk; //--- Whether the DEVICE-SIDE apply kernels built (def_k_ApplyAccumAdam). Conflating them would //--- turn a missing optimisation into a change of optimizer. bool m_applyKernelsOk; bool m_batchBegun; bool m_batchWarned; //--- Zero every accumulator, starting a fresh batch. bool BeginBatch(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronPool { protected: CLayer *ForgetGate; CLayer *InputGate; CLayer *OutputGate; CLayer *NewContent; CArrayDouble *Memory; CArrayDouble *PrevMemory; CArrayDouble *Input; CArrayDouble *InputGradient; //--- virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual bool updateInputWeights(CLayer *prevLayer); virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data); virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type); //--- Fills `result` rather than returning a fresh CArrayDouble. The old shape handed every caller //--- an object to delete on each of its own error paths, and none of them did - see feedForward(). virtual bool CalculateGate(CLayer *gate, CArrayDouble *sequence, CArrayDouble &result); bool AccumulateGateInputGradient(CLayer *gate, const int i, const int n, double &value); public: CNeuronLSTM(void); ~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = -1; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return -1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "BufferDouble.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: COpenCLMy *OpenCL; CDirectMLMy *DirectML; CBufferDouble *Output; CBufferDouble *PrevOutput; CBufferDouble *Weights; CBufferDouble *DeltaWeights; CBufferDouble *Gradient; CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; //--- MINI-BATCH ACCUMULATOR (2026-08-09 audit, F4). Same shape as Weights; holds the SUM of this //--- weight block's per-sample gradients over the current batch. CBufferDouble *GradAccum; //--- const double alpha; int t; //--- ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); //--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only //--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists - //--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU). bool BackendBufferCreate(CBufferDouble *buf) { if(CheckPointer(buf) == POINTER_INVALID) return false; if(CheckPointer(OpenCL) != POINTER_INVALID) return buf.BufferCreate(OpenCL); if(CheckPointer(DirectML) != POINTER_INVALID) return buf.BufferCreate(DirectML); return true; // no backend: keep host m_data, skip device allocation } //--- THE optimizer step, and the only copy of it in the batched path: one weight block, one Adam //--- or SGD+momentum update on `acc * scale`, then the accumulator is zeroed. bool ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale, int total); bool ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale); //--- Lazily allocate the mini-batch accumulator to match `src` (Weights / WeightsConv / //--- WeightsLSTM, whichever block the caller accumulates into). bool EnsureGradAccumFor(CBufferDouble *&target, CBufferDouble *src) { if(CheckPointer(src) == POINTER_INVALID || src.Total() <= 0) return false; if(CheckPointer(target) != POINTER_INVALID && target.Total() == src.Total()) return true; if(CheckPointer(target) != POINTER_INVALID) delete target; target = new CBufferDouble(); if(CheckPointer(target) == POINTER_INVALID) return false; if(!target.BufferInit(src.Total(), 0.0)) return false; return BackendBufferCreate(target); } bool EnsureGradAccum(CBufferDouble *src) { return EnsureGradAccumFor(GradAccum, src); } public: CNeuronBaseOCL(void); ~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- virtual int getOutputIndex(void) { return Output.GetIndex(); } virtual int getGradientIndex(void) { return Gradient.GetIndex(); } virtual int getWeightsIndex(void) { return Weights.GetIndex(); } virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); } virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); } virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();} //--- virtual int getOutputVal(double &values[]) { return Output.GetData(values); } virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); } virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); } virtual int getGradient(double &values[]) { return Gradient.GetData(values); } //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same //--- buffer to propagate into the hidden layers. virtual bool setGradient(const double &values[]) { int count = ArraySize(values); for(int i = 0; i < count; i++) if(!Gradient.Update(i, values[i])) return false; return Gradient.BufferWrite(); } // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created, // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here. virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); } //--- Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment //--- (see that method's declaration comment) - writes a full replacement weight array back to //--- this buffer's device-side (DLL/OpenCL/DirectML) storage. virtual bool setWeights(double &values[]) { if(CheckPointer(Weights) == POINTER_INVALID) return false; if(!Weights.AssignArray(values)) return false; return Weights.BufferWrite(); } //--- Forget the optimizer's trajectory memory: Adam first/second moments, SGD's previous-delta //--- buffer, and the bias-correction step counter, weights untouched. See //--- CNet::ResetOptimizerState for the restore/warm-restart rationale. virtual bool ResetOptimizerState(void) { t = 1; bool ok = ZeroOptimizerBuffer(FirstMomentum); ok = ZeroOptimizerBuffer(SecondMomentum) && ok; ok = ZeroOptimizerBuffer(DeltaWeights) && ok; return ok; } virtual int Neurons(void) { return Output.Total(); } virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); } //--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to //--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/ //--- getOutputVal() which route through BufferRead() (a device read that fails without a backend). double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; } double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; } int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; } //--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only //--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense //--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math. virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); //--- Host-side input write for the CPU inference path's layer 0, and host-side output read for //--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates. bool SetInputsCPU(CArrayDouble *inputVals); int GetOutputsCPU(CArrayDouble *values); //--- virtual bool feedForward(CObject *SourceObject); virtual bool calcHiddenGradients(CObject *TargetObject); virtual bool calcOutputGradients(CArrayDouble *Target); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CObject *SourceObject); //--- MINI-BATCH PAIR (2026-08-09 audit, F4). accumulateInputWeightGrads() adds THIS sample's //--- per-weight gradient into GradAccum without touching the weights; //--- ApplyAccumulatedGradients() then takes ONE optimizer step on the batch mean and clears the //--- accumulator. virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL); virtual bool accumulateInputWeightGrads(CObject *SourceObject); virtual bool ApplyAccumulatedGradients(double scale); //--- Zero the accumulator at the start of a batch. Separate from ApplyAccumulatedGradients so a //--- discarded partial batch (a stopped run) can be cleared without taking a step from it. virtual bool BeginGradAccum(void); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBaseOCL; } }; //+------------------------------------------------------------------+ #include "NeuronOCLConvPool.mqh" #include "NeuronBatchNorm.mqh" //+------------------------------------------------------------------+ //--- Marks a .nnw LSTM record as the SEQUENCE format. Chosen so it cannot collide with any value the //--- pre-sequence format could have written in that slot (an input width, i.e. -1 or a positive count). #define LSTM_SEQ_SAVE_TAG (-424242) //--- Initial bias of the FORGET gate (gate 0). Every other weight starts near zero; this one must //--- not. #define LSTM_FORGET_BIAS_INIT (1.0) //+------------------------------------------------------------------+ //| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from | //| scratch from the standard LSTM equations - NOT ported from the | //| NeuroNet_DNG reference (see the note above the LSTM kernels in | //| Network.cl for why). Single-timestep-truncated BPTT: gradient | //| does not flow back into h_prev/c_prev from an earlier step. | //| Adam-only - Init fails for any other optimization type. | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: int m_iInputs; //--- Mini-batch running total of WeightsGradient across the samples of one batch - see the note on //--- accumulateInputWeightGrads below for why WeightsGradient itself cannot serve as it. CBufferDouble *GradAccumLSTM; //--- Sequence shape. m_iStepInputs is the width of ONE timestep (the per-bar feature count //--- reaching this layer), set from the layer descriptor by SetStepWidth() before the first //--- feedForward; m_iSteps is then m_iInputs / m_iStepInputs. int m_iStepInputs; int m_iSteps; //--- Per-timestep caches, required by backpropagation-through-time: the backward pass needs each //--- step's gate activations, cell state and hidden state, which the single-buffer Concatenated/ //--- Memory/HiddenCache trio below cannot hold because every step overwrites the last. CBufferDouble *CacheGates; // T * 4H, gate order [f,i,o,g] CBufferDouble *CacheCell; // T * H, c_t CBufferDouble *CacheHidden; // T * H, h_t CBufferDouble *WeightsLSTM; CBufferDouble *FirstMomentumLSTM; CBufferDouble *SecondMomentumLSTM; CBufferDouble *DeltaWeightsLSTM; CBufferDouble *WeightsGradient; CBufferDouble *Concatenated; CBufferDouble *ConcatenatedGradient; CBufferDouble *Memory; CBufferDouble *HiddenCache; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool SetInputs(int count); bool AllocateSequenceCaches(void); public: CNeuronLSTMOCL(void) : m_iInputs(-1), m_iStepInputs(-1), m_iSteps(-1) { GradAccumLSTM = NULL; CacheGates = NULL; CacheCell = NULL; CacheHidden = NULL; WeightsLSTM = NULL; FirstMomentumLSTM = NULL; SecondMomentumLSTM = NULL; DeltaWeightsLSTM = NULL; WeightsGradient = NULL; Concatenated = NULL; ConcatenatedGradient = NULL; Memory = NULL; HiddenCache = NULL; } ~CNeuronLSTMOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); //--- Per-timestep input width, from CLayerDescription::window (see AddLstmStage). Must be called //--- between Init() and the first feedForward; <= 0 keeps the legacy single-timestep behaviour. //--- Not persisted from here - Save/Load carry it, so a loaded model does not depend on call order. void SetStepWidth(int stepInputs) { m_iStepInputs = (stepInputs > 0 ? stepInputs : -1); } bool IsSequenceMode(void) const { return (m_iStepInputs > 0 && m_iSteps > 1); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTMOCL; } // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment. virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); } //--- Build this layer's weight block to match `src`, for a net that was cloned from one whose //--- LSTM had not yet run a forward pass. virtual bool AdoptShapeFrom(CNeuronLSTMOCL &src) { if(src.m_iInputs <= 0 || Neurons() != src.Neurons()) return false; m_iStepInputs = src.m_iStepInputs; return SetInputs(src.m_iInputs); } virtual bool setWeightsLSTM(double &values[]) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; if(!WeightsLSTM.AssignArray(values)) return false; return WeightsLSTM.BufferWrite(); } //--- The LSTM's gate-weight block keeps its own moment/momentum buffers beside the base class's - //--- see CNet::ResetOptimizerState. virtual bool ResetOptimizerState(void) { bool ok = CNeuronBaseOCL::ResetOptimizerState(); ok = ZeroOptimizerBuffer(FirstMomentumLSTM) && ok; ok = ZeroOptimizerBuffer(SecondMomentumLSTM) && ok; ok = ZeroOptimizerBuffer(DeltaWeightsLSTM) && ok; return ok; } //--- MINI-BATCH. Batching it is therefore just a running total. Hence a separate accumulator //--- plus a generic elementwise add (AccumulateBufferInto), which also avoids changing the //--- signature of an already-deployed export. virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL); virtual bool BeginGradAccum(void) { bool ok = CNeuronBaseOCL::BeginGradAccum(); if(CheckPointer(GradAccumLSTM) != POINTER_INVALID && GradAccumLSTM.Total() > 0) ok = ZeroOptimizerBuffer(GradAccumLSTM) && ok; return ok; } virtual bool ApplyAccumulatedGradients(double scale) { bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale); ok = ApplyAccumToBlock(WeightsLSTM, GradAccumLSTM, FirstMomentumLSTM, SecondMomentumLSTM, DeltaWeightsLSTM, scale) && ok; if(optimization == ADAM) t++; return ok; } }; //+------------------------------------------------------------------+ //| Implementation bodies. | //+------------------------------------------------------------------+ #include "Impl\NeuronBase.mqh" #include "Impl\NeuronConvPool.mqh" #include "Impl\NeuronLSTM.mqh" #include "Impl\Layer.mqh" #include "Impl\NetBuild.mqh" #include "Impl\NetForward.mqh" #include "Impl\NetPersistence.mqh" #include "Impl\NetWeights.mqh" #include "Impl\NeuronOCLBase.mqh" #include "Impl\NeuronOCLLSTM.mqh" //+------------------------------------------------------------------+