//+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include #include #include #include #include "..\System\AtomicFile.mqh" //--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available, //--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Sizes //--- WarriorCPU.dll's worker thread pool - no effect at all when a GPU tier (OpenCL or DirectML) is //--- active, since neither one calls into WarriorCPU.dll. //--- //--- A FIXED SMALL THREAD COUNT PER NETWORK, not a share of the machine. This replaced a TargetCPULoad //--- input divided by the live chart count, which was wrong twice over: //--- - The count is a SNAPSHOT taken when each net's pool is built, and charts are attached one at a //--- time. Measured 2026-07-29 with five charts: they took 10/6/5/4/4% of the same budget, because //--- the first chart only ever saw itself and the last saw all five. So the earliest chart got //--- several times the threads of the latest - which silently skews any cross-topology comparison //--- run on those charts, the exact thing the setting existed to make fair. //--- - Nothing rebalances when a chart is added or removed, and rebalancing would mean tearing down a //--- DLL context underneath a running trainer. //--- Neither problem exists once the answer stops depending on how many charts are running. //--- //--- Why 2 threads is not a compromise: since the topology became data-derived the widest dense layer //--- is 64 units (ExpertSignalAIBase.mqh's ComputeFirstLayerWidth), so each ParallelFor has almost //--- nothing to split and per-dispatch overhead dominates. The measurements agree - an MLP era cost //--- ~66s at a wildly oversubscribed 12 threads and ~80s at 1 thread, a 20% spread across a 12x //--- difference in thread count. Two per net also lands six concurrent charts exactly on a 12-core box. //--- //--- Removing the input costs nothing on the product side: a Market build has no DLL tier at all (MQL5 //--- Market rule IV strips the #import blocks - see AI\NeuronDirectML.mqh), so it was already compiled //--- out to a constant there and no buyer could ever reach it. #define CPU_THREADS_PER_NETWORK 2 //+------------------------------------------------------------------+ //| The percentage CDirectMLMy needs in order to land on | //| CPU_THREADS_PER_NETWORK workers, given the detected core count. | //| (WarriorCPU.dll takes a percentage of cores, not a thread count.) | //+------------------------------------------------------------------+ int EffectiveCpuLoadPercent() { int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES); //--- Unknown core count: assume a small machine rather than a large one. Guessing high here would //--- reintroduce exactly the oversubscription this function exists to prevent. if(cores <= 0) cores = 4; //--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the //--- DLL, so this is the one value that needs no arithmetic. if(cores <= CPU_THREADS_PER_NETWORK) return 100; //--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread //--- short of the target is a worse error than landing one over. int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores); return (int)MathMax(1, MathMin(100, pct)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference //--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's //--- abstract "0.001" mention, which the book's own worked examples don't actually use either. //--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005) //--- this project had separately tuned lr to before this input existed, so no behavior conflict. //--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak //--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to //--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this //--- reverts to the literature/book default. input double AdamLearningRate = 0.0003; // Adam learning rate input double AdamBeta1 = 0.9; // Adam beta1 input double AdamBeta2 = 0.999; // Adam beta2 //--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method //--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses //--- Adam's book-default rate as its starting point too. The book also states no numeric default for //--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses //--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for. input double SgdLearningRate = 0.0003; // SGD learning rate input double SgdMomentum = 0.9; // SGD momentum #define lr AdamLearningRate #define b1 AdamBeta1 #define b2 AdamBeta2 #define momentum SgdMomentum double eta = lr; #define defConnect 0x7781 #define defArrayConnects 0x7782 #define defNeuronBase 0x7783 #define defNeuron 0x7784 #define defNeuronConv 0x7785 #define defNeuronPool 0x7786 #define defLayer 0x7787 #define defArrayLayer 0x7788 #define defNet 0x7789 #define defNeuronLSTM 0x7791 //--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like //--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps //--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh. #define defNeuronBatchNorm 0x7792 //--- #define defBufferDouble 0x7882 #define defNeuronBaseOCL 0x7883 #define defNeuronLSTMOCL 0x7884 #define defNeuronConvOCL 0x7885 #define defNeuronPoolOCL 0x7886 #define defNeuronBatchNormOCL 0x7887 //--- #define def_k_FeedForward 0 #define def_k_ff_matrix_w 0 #define def_k_ff_matrix_i 1 #define def_k_ff_matrix_o 2 #define def_k_ff_inputs 3 #define def_k_ff_activation 4 //--- #define def_k_CaclOutputGradient 1 #define def_k_cog_matrix_t 0 #define def_k_cog_matrix_o 1 #define def_k_cog_matrix_ig 2 #define def_k_cog_activation 3 //--- #define def_k_CaclHiddenGradient 2 #define def_k_chg_matrix_w 0 #define def_k_chg_matrix_g 1 #define def_k_chg_matrix_o 2 #define def_k_chg_matrix_ig 3 #define def_k_chg_outputs 4 #define def_k_chg_activation 5 //--- #define def_k_UpdateWeightsMomentum 3 #define def_k_uwm_matrix_w 0 #define def_k_uwm_matrix_g 1 #define def_k_uwm_matrix_i 2 #define def_k_uwm_matrix_dw 3 #define def_k_uwm_inputs 4 #define def_k_uwm_learning_rates 5 #define def_k_uwm_momentum 6 #define def_k_uwm_optimizer 7 //--- #define def_k_UpdateWeightsAdam 4 #define def_k_uwa_matrix_w 0 #define def_k_uwa_matrix_g 1 #define def_k_uwa_matrix_i 2 #define def_k_uwa_matrix_m 3 #define def_k_uwa_matrix_v 4 #define def_k_uwa_inputs 5 #define def_k_uwa_l 6 #define def_k_uwa_b1 7 #define def_k_uwa_b2 8 //--- #define def_k_FeedForwardProof 15 #define def_k_ffp_matrix_i 0 #define def_k_ffp_matrix_o 1 #define def_k_ffp_inputs 2 #define def_k_ffp_window 3 #define def_k_ffp_step 4 //--- #define def_k_CalcInputGradientProof 16 #define def_k_cigp_matrix_i 0 #define def_k_cigp_matrix_g 1 #define def_k_cigp_matrix_o 2 #define def_k_cigp_matrix_ig 3 #define def_k_cigp_outputs 4 #define def_k_cigp_window 5 #define def_k_cigp_step 6 //--- #define def_k_FeedForwardConv 5 #define def_k_ffc_matrix_w 0 #define def_k_ffc_matrix_i 1 #define def_k_ffc_matrix_o 2 #define def_k_ffc_inputs 3 #define def_k_ffc_step 4 #define def_k_ffc_window_in 5 #define def_k_ffc_window_out 6 #define def_k_ffc_activation 7 //--- #define def_k_CalcHiddenGradientConv 6 #define def_k_chgc_matrix_w 0 #define def_k_chgc_matrix_g 1 #define def_k_chgc_matrix_o 2 #define def_k_chgc_matrix_ig 3 #define def_k_chgc_outputs 4 #define def_k_chgc_step 5 #define def_k_chgc_window_in 6 #define def_k_chgc_window_out 7 #define def_k_chgc_activation 8 //--- #define def_k_UpdateWeightsConvMomentum 7 #define def_k_uwcm_matrix_w 0 #define def_k_uwcm_matrix_g 1 #define def_k_uwcm_matrix_i 2 #define def_k_uwcm_matrix_dw 3 #define def_k_uwcm_inputs 4 #define def_k_uwcm_learning_rates 5 #define def_k_uwcm_momentum 6 #define def_k_uwcm_window_in 7 #define def_k_uwcm_window_out 8 #define def_k_uwcm_step 9 #define def_k_uwcm_optimizer 10 //--- #define def_k_UpdateWeightsConvAdam 8 #define def_k_uwca_matrix_w 0 #define def_k_uwca_matrix_g 1 #define def_k_uwca_matrix_i 2 #define def_k_uwca_matrix_m 3 #define def_k_uwca_matrix_v 4 #define def_k_uwca_inputs 5 #define def_k_uwca_l 6 #define def_k_uwca_b1 7 #define def_k_uwca_b2 8 #define def_k_uwca_window_in 9 #define def_k_uwca_window_out 10 #define def_k_uwca_step 11 //--- // LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient // flows back into h_prev/c_prev from a prior step). Supports both Adam and // SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) - // see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch. // Derived from scratch from the standard LSTM equations - NOT ported from // the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites // the live weights buffer instead of writing to weights_gradient. #define def_k_LSTM_Gates 9 #define def_k_lstmg_matrix_w 0 #define def_k_lstmg_hidden_prev 1 #define def_k_lstmg_inputs 2 #define def_k_lstmg_concatenated 3 #define def_k_lstmg_hidden_size 4 #define def_k_lstmg_input_size 5 //--- #define def_k_LSTM_State 10 #define def_k_lstms_concatenated 0 #define def_k_lstms_memory 1 #define def_k_lstms_hidden_prev 2 #define def_k_lstms_hidden_cache 3 #define def_k_lstms_output 4 #define def_k_lstms_hidden_size 5 //--- #define def_k_LSTM_GateGradient 11 #define def_k_lstmgg_gradient 0 #define def_k_lstmgg_memory 1 #define def_k_lstmgg_concatenated 2 #define def_k_lstmgg_concatenated_gradient 3 #define def_k_lstmgg_hidden_size 4 //--- #define def_k_LSTM_WeightsGradient 12 #define def_k_lstmwg_concatenated_gradient 0 #define def_k_lstmwg_hidden_cache 1 #define def_k_lstmwg_inputs 2 #define def_k_lstmwg_weights_gradient 3 #define def_k_lstmwg_hidden_size 4 #define def_k_lstmwg_input_size 5 //--- #define def_k_LSTM_InputsGradient 13 #define def_k_lstmig_concatenated_gradient 0 #define def_k_lstmig_matrix_w 1 #define def_k_lstmig_inputs_gradient 2 #define def_k_lstmig_hidden_size 3 #define def_k_lstmig_input_size 4 //--- #define def_k_LSTM_UpdateWeightsAdam 14 #define def_k_lstmuwa_matrix_w 0 #define def_k_lstmuwa_weights_gradient 1 #define def_k_lstmuwa_matrix_m 2 #define def_k_lstmuwa_matrix_v 3 #define def_k_lstmuwa_l 4 #define def_k_lstmuwa_b1 5 #define def_k_lstmuwa_b2 6 //--- // SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see // AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body. #define def_k_LSTM_UpdateWeightsMomentum 17 #define def_k_lstmuwm_matrix_w 0 #define def_k_lstmuwm_weights_gradient 1 #define def_k_lstmuwm_matrix_dw 2 #define def_k_lstmuwm_learning_rates 3 #define def_k_lstmuwm_momentum 4 #define def_k_lstmuwm_optimizer 5 //--- // b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see // AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era // same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root // cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE // gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever // could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled // kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam // signatures), so they're safe to expose as ordinary inputs. // Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had // already gone unstable (real collapses were happening at weight magnitudes several orders of // magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh // engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually // reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below. #define MAX_WEIGHT 100.0 // Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in // DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see // WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright // +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of // training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi- // hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. // 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates, // ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's // sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a // weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the // discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the // calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the // spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt // dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay // strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins // argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth. #define WEIGHT_DECAY 0.001 // Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full // rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden // Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras), // most likely from 5x back-to-back oversampling replay building artificially correlated momentum. // Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the // post-update weight value and is far too loose (1e6) to prevent this. #define MAX_WEIGHT_DELTA 0.1 // Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see // SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its // activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that // neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated // unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in // Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine. // 2026-07-27: Increased from 1.0e-4 to 1.0e-3 — must stay in sync with AI\Network.cl's matching // constant. See that file's comment for the full rationale (fp32 OpenCL saturation floor fix). #define MIN_ACTIVATION_DERIVATIVE 1.0e-3 // Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND // read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a // different distribution than it was trained on). The classification outputs are SIGMOID-bounded to // [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 - // the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training // can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three // outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the // ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal // modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays // (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the // loss, keeping gradient magnitudes (and thus eta tuning) unchanged. #define CLASS_LOGIT_SCALE 6.0 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "Network.cl" as string cl_program //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { NONE, TANH, SIGMOID, PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction }; //+------------------------------------------------------------------+ //| Translates ENUM_ACTIVATION to the "activation" int code every | //| native compute backend actually understands (Network.cl's | //| kernels, and the mirrored Activation()/inline switches in | //| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and | //| deliberately anything else (incl. NONE) falls through every one | //| of those switches unmatched, which is exactly linear passthrough -| //| there's no case 3 anywhere on the native side, so PRELU relies on | //| the FeedForwardConv-family kernels specifically, and NONE never | //| needs a case at all. This is NOT the same numbering as | //| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a | //| raw (int)activation cast at a kernel call site silently sends the | //| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native | //| SIGMOID). Only use this at actual kernel-dispatch call sites - | //| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw | //| ENUM_ACTIVATION value instead, and must keep using (int)activation | //| directly so saved topology files round-trip through Load() as-is. | //+------------------------------------------------------------------+ int NativeActivationCode(ENUM_ACTIVATION value) { switch(value) { case TANH: return 0; case SIGMOID: return 1; case PRELU: return 2; default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches } } //+------------------------------------------------------------------+ //| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the | //| load-time architecture repair (CNet::EnforceOutputActivation) so | //| the log names the stale value it found instead of printing a bare | //| integer nobody can decode months later. | //+------------------------------------------------------------------+ string ActivationName(ENUM_ACTIVATION value) { switch(value) { case NONE: return "NONE"; case TANH: return "TANH"; case SIGMOID: return "SIGMOID"; case PRELU: return "PRELU"; } return "UNKNOWN(" + IntegerToString((int)value) + ")"; } //--- //--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\ //--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead of //--- this AI header - without a duplicate-definition error. Whichever file is parsed first defines it; //--- the other's copy is skipped. Keep the two definitions in sync. #ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED #define WARRIOR_ENUM_OPTIMIZATION_DEFINED //--- 2026-07-28: a third DFA entry was removed. It was never Direct Feedback Alignment: its feedback //--- signal multiplied dL/dw by a DETERMINISTIC sign pattern (connectionIndex % 2), which makes half of //--- every weight tensor perform gradient ASCENT permanently - it diverges by construction, with no //--- hyperparameter able to rescue it. Real DFA (Nokland 2016) works because a FIXED RANDOM matrix gives //--- a consistent feedback direction the forward weights can align to; an index-parity sign flip has no //--- such alignment property. Its backward pass was also structurally incompatible with the OpenCL/ //--- DirectML neuron model this project actually runs on (one CNeuronBaseOCL object holds a whole layer //--- in a device buffer, so the per-neuron host-scalar loops it used saw layer.Total()==1 and updated //--- nothing). SGD/ADAM keep their ordinal values 0/1 - m_optimizationAlgo feeds the weights-filename //--- fingerprint, so these must never be renumbered. enum ENUM_OPTIMIZATION { SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras) ADAM // Adam (adaptive step, faster convergence, can overfit) }; #endif //--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronPrimitives.mqh" class CLayer; //--- class CNeuronBase : public CObject { protected: double outputVal; double prevVal; uint m_myIndex; double gradient; CArrayCon *Connections; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; int t; //--- virtual bool feedForward(CLayer *prevLayer) { return false; } virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; } virtual double activationFunction(double x); virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); } virtual double TanhFunction(double x) { return tanh(x); } virtual CLayer *getOutputLayer(void) { return NULL; } public: CNeuronBase(void); ~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back //--- what a Load() restored from disk without caring which neuron model the net was built from. virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual void SetOptimization(ENUM_OPTIMIZATION value) { optimization = value; } virtual ENUM_OPTIMIZATION Optimization(void) const { return optimization; } //--- static double alpha; //--- virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; } virtual double getOutputVal() { return outputVal; } virtual double getPrevVal() { return prevVal; } virtual void setGradient(double val) { gradient = val; } virtual double getGradient() { return gradient; } virtual CArrayCon *getConnections() { return Connections;} virtual double activationFunctionDerivative(double x); virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); } virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); } //--- virtual bool feedForward(CObject *&SourceObject); virtual bool calcHiddenGradients(CObject *&TargetObject); virtual bool updateInputWeights(CLayer *prevLayer) { return false; } virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle) { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- virtual int Type(void) const { return defNeuronBase; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::alpha = momentum; // momentum //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::CNeuronBase(void) : outputVal(1), gradient(0), activation(TANH), t(1), optimization(SGD) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::~CNeuronBase(void) { if(CheckPointer(Connections) != POINTER_INVALID) delete Connections; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0) { if(CheckPointer(Connections) == POINTER_INVALID) { Connections = new CArrayCon(); if(CheckPointer(Connections) == POINTER_INVALID) return false; } //--- if(!Connections.Reserve(fmax(numOutputs, 1))) { Print(__FUNCTION__ + ": Connections.Reserve failed (allocation failure?) - neuron would silently end up with 0 connections"); return false; } for(uint c = 0; c < numOutputs; c++) { if(!Connections.CreateElementScaled(c, weighScale)) return false; Connections.IncreaseTotal(); } //--- m_myIndex = myIndex; optimization = optimization_type; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronCPU.mqh" class COpenCLMy : public COpenCL { public: COpenCLMy(void) {}; ~COpenCLMy(void) {}; template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); }; #include "NeuronDirectML.mqh" class CLayer: public CArrayObj { private: uint iOutputs; int iFileHandle; int hWeights; int hDeltaWeights; int hOutput; int hGradient; COpenCLMy *OpenCL; CDirectMLMy *DirectML; public: CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL); ~CLayer(void) {}; //--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below. bool CreateElementScaled(int const index, double weighScale); //--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool //--- CreateElement(const int index)` - because it is the real virtual override that CArrayObj::Load() //--- dispatches through, and CArrayObj::Load() is how EVERY saved layer is read back (CLayer::Load -> //--- CNet::Load). In MQL5 an override must match the base parameter list //--- exactly; adding even a DEFAULTED parameter makes it a separate method that merely hides the base //--- one, silently leaving the base's `return(false)` stub in the vtable slot. That is exactly what a //--- `double weighScale = -1.0` parameter added here did: from then on every single model load failed //--- at the first layer ("REJECTED: only loaded 0 of N layers (failed at layer 0)") no matter how //--- healthy the .nnw was, so every restart retrained from era 0 and every best-checkpoint restore //--- silently kept the current weights. Never add parameters to this method - extend //--- CreateElementScaled() and call it explicitly instead. virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); } virtual void IncreaseTotal() { m_data_total++; } virtual int Type(void) const { return defLayer; } virtual bool Load(const int file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::CreateElementScaled(int index, double weighScale) { if(index >= m_data_max) return false; //--- bool result = false; CNeuronBase *temp = NULL; CNeuronPool *temp_p = NULL; CNeuronBaseOCL *temp_ocl = NULL; if(iFileHandle <= 0) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD, weighScale)) return false; result = true; } else { int type = FileReadInteger(iFileHandle); switch(type) { case defNeuron: temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) { result = false; break; } result = temp.Init(iOutputs, index, ADAM); break; case defNeuronPool: temp_p = new CNeuronPool(); if(CheckPointer(temp_p) == POINTER_INVALID) { result = false; break; } if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronConv: temp_p = new CNeuronConv(); if(CheckPointer(temp_p) == POINTER_INVALID) { result = false; break; } if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronLSTM: temp_p = new CNeuronLSTM(); if(CheckPointer(temp_p) == POINTER_INVALID) { result = false; break; } if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronBaseOCL: temp_ocl = new CNeuronBaseOCL(); if(CheckPointer(temp_ocl) == POINTER_INVALID) { result = false; break; } //--- Pure-MQL5 inference (no backend): construct host-only. Load() restores the weights into //--- the host buffers and CNeuronBaseOCL::feedForwardCPU() computes on them. The device Init() //--- below REQUIRES a backend, so it only runs when one exists (training/optimization/GPU run). if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) { m_data[index] = temp_ocl; return true; } if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM) : temp_ocl.Init(iOutputs, index, DirectML, 1, ADAM)) { m_data[index] = temp_ocl; return true; } break; case defNeuronConvOCL: { //--- placeholder dims; the real window/step/units are restored by Load() right after CNeuronConvOCL *temp_conv = new CNeuronConvOCL(); if(CheckPointer(temp_conv) == POINTER_INVALID) { result = false; break; } //--- Pure-MQL5 inference (no backend): construct host-only, Load() fills the host buffers, //--- CNeuronConvOCL::feedForwardCPU() computes on them (device Init below needs a backend). if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) { m_data[index] = temp_conv; return true; } if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM) : temp_conv.Init(iOutputs, index, DirectML, 1, 1, 1, 1, ADAM)) { m_data[index] = temp_conv; return true; } break; } case defNeuronBatchNormOCL: { //--- Placeholder width of 1; CNeuronBatchNormOCL::Load() replaces both the buffers and the //--- whole gamma/beta/statistics block with the real ones, then verifies they agree. CNeuronBatchNormOCL *temp_bn = new CNeuronBatchNormOCL(); if(CheckPointer(temp_bn) == POINTER_INVALID) { result = false; break; } //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note). if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) { m_data[index] = temp_bn; return true; } if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_bn.Init(iOutputs, index, OpenCL, 1, 1, ADAM) : temp_bn.Init(iOutputs, index, DirectML, 1, 1, ADAM)) { m_data[index] = temp_bn; return true; } break; } case defNeuronPoolOCL: { CNeuronPoolOCL *temp_pool = new CNeuronPoolOCL(); if(CheckPointer(temp_pool) == POINTER_INVALID) { result = false; break; } //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note). if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) { m_data[index] = temp_pool; return true; } if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_pool.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM) : temp_pool.Init(iOutputs, index, DirectML, 1, 1, 1, ADAM)) { m_data[index] = temp_pool; return true; } break; } case defNeuronLSTMOCL: { CNeuronLSTMOCL *temp_lstm = new CNeuronLSTMOCL(); if(CheckPointer(temp_lstm) == POINTER_INVALID) { result = false; break; } //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note). if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) { m_data[index] = temp_lstm; return true; } if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_lstm.Init(iOutputs, index, OpenCL, 1, ADAM) : temp_lstm.Init(iOutputs, index, DirectML, 1, ADAM)) { m_data[index] = temp_lstm; return true; } break; } default: result = false; break; } } if(result) m_data[index] = temp; //--- return (result); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "ArrayLayer.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPool : public CNeuronBase { protected: CLayer *OutputLayer; int iWindow; int iStep; virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); public: CNeuronPool(void) {}; ~CNeuronPool(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronPool; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronPool { protected: double param; //PReLU param virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual double activationFunction(double x); virtual bool updateInputWeights(CLayer *prevLayer); public: CNeuronConv() : param(0.01) { }; ~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; virtual double activationFunctionDerivative(double x); virtual int Type(void) const { return defNeuronConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::feedForward(CObject *&SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = feedForward(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: temp_n = SourceObject; result = feedForward(temp_n.getOutputLayer()); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::updateInputWeights(CObject *SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = updateInputWeights(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: temp_n = SourceObject; temp_l = temp_n.getOutputLayer(); result = updateInputWeights(temp_l); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::feedForward(CLayer *prevLayer) { bool result = false; //--- if(CheckPointer(prevLayer) == POINTER_INVALID) return result; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; CConnection *con; result = true; for(int i = 0; (i < total && result); i += iStep) { double sum = 0; for(int j = 0; (j < iWindow && result); j++) { temp = prevLayer.At(i + j); con = Connections.At(j); if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) return false; double val = temp.getOutputVal(); sum += val * con.weight; } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setOutputVal(activationFunction(sum)); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronConv::activationFunction(double x) { if(x >= 0) return x; return param * x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject) { bool result = false; //--- if(CheckPointer(TargetObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(TargetObject.Type()) { case defLayer: temp_l = TargetObject; result = calcHiddenGradients(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: switch(Type()) { case defNeuron: temp_n = TargetObject; result = temp_n.calcInputGradients(GetPointer(this), m_myIndex); break; case defNeuronLSTM: temp_n = TargetObject; temp_l = getOutputLayer(); if(!temp_n.calcInputGradients(temp_l)) { result = false; break; } result = calcHiddenGradients(temp_l); break; default: temp_l =getOutputLayer(); temp_n = TargetObject; result = temp_n.calcInputGradients(temp_l); break; } break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) return false; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal())); } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronConv::activationFunctionDerivative(double x) { if(x >= 0) return 1; return param; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::updateInputWeights(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- CConnection *con; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int n = 0; n < iWindow && !IsStopped(); n++) { con = Connections.At(n); if(CheckPointer(con) == POINTER_INVALID) continue; double delta = 0; int total_i = OutputLayer.Total(); CNeuron *prev, *out; for(int i = 0; i < total_i; i++) { prev = prevLayer.At(n * iStep + i); out = OutputLayer.At(total_i - i - 1); if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) continue; delta += prev.getOutputVal() * out.getGradient(); } if(optimization == SGD) con.weight += con.deltaWeight = (delta != 0 ? eta*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0); else { con.mt = b1 * con.mt + (1 - b1) * delta; con.vt = b2 * con.vt + (1 - b2) * delta * delta + 0.00000001; con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight)); // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why. con.weight += con.deltaWeight; } // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches // AI\Network.cl's UpdateWeightsConvMomentum/UpdateWeightsConvAdam MAX_WEIGHT clamp. con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight)); } if(optimization == ADAM) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { iWindow = window; iStep = step; //--- LeCun-uniform init, matching CNeuronConvOCL::Init's rationale - fan-in is the conv window size. if(!CNeuronBase::Init(window, myIndex, optimization_type, 1.0 / MathSqrt((double)window + 1.0))) return false; OutputLayer = new CLayer(numOutputs); if(CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- He-scaled init for the OutputLayer's own dense units - fan-in is this pool/conv unit's own //--- sibling count (units_count); no OCL/DLL equivalent exists to mirror since those tiers use flat //--- buffers instead of this per-unit object representation, so this follows the same dense rationale //--- as CNet::CNet()'s defNeuron case above. double outputScale = MathSqrt(2.0 / ((double)units_count + 1.0)); if(!OutputLayer.Reserve(units_count)) { Print(__FUNCTION__ + ": OutputLayer.Reserve failed (allocation failure?) - neuron would silently end up with 0 outputs"); return false; } for(int i = 0; i < units_count; i++) { if(!OutputLayer.CreateElementScaled(i, outputScale)) return false; OutputLayer.IncreaseTotal(); } //--- if(Type() == defNeuronPool) { if(CheckPointer(Connections) != POINTER_INVALID) Connections.Clear(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronPool::~CNeuronPool(void) { delete OutputLayer; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) return false; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; for(int i = 0; i <= total; i += iStep) { double sum = 0; for(int j = 0; j < iWindow; j++) { temp = prevLayer.At(i + j); if(CheckPointer(temp) == POINTER_INVALID) continue; sum += temp.getOutputVal(); } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setOutputVal(sum / iWindow); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) return false; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || CheckPointer(prevLayer.At(0)) == POINTER_INVALID) return false; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronPool *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) return false; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) return false; } //--- CNeuronBase *prevNeuron, *outputNeuron; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; double prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (i - i % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevNeuron.Type() != defNeuron) { CNeuronPool *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; double prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (index - index % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronPool *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) return false; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) return false; } //--- CNeuronBase *prevNeuron, *outputNeuron; CConnection *con; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; double prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (i - i % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep; con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevNeuron.Type() != defNeuron) { CNeuronPool *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; CConnection *con; double prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (index - index % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep); con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Save(int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE) return false; //--- return Connections.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "LayerDescription.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNet { protected: double dLogitAdjust[3]; bool bLogitAdjust; void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0); bool InitOpenCL(void); bool InitDirectML(void); //--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference. bool feedForwardCPU(CArrayDouble *inputVals); public: CNet(CArrayObj *Description); ~CNet(void); bool feedForward(CArrayDouble *inputVals); //--- sampleWeight scales this example's output-layer gradient before it propagates back through the //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same //--- example multiple times). void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0); //--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class //--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass //--- only. Minimizing softmax CE on adjusted logits is consistent for BALANCED error - which is //--- exactly the metric checkpoint selection already ranks on (macro-recall), so this is the first //--- time the loss and the selection criterion optimize the same thing. //--- It replaces minority REPLAY (which duplicated rare bars up to 28x and made Buy and Sell //--- compete for the same replicated capacity - the measured failure was each model taking one //--- direction to ~50% recall and abandoning the other, with the direction chosen arbitrarily) and //--- the post-hoc inference prior, which becomes double-counting once the offsets are trained in. //--- Offsets are NEGATIVE (log of a probability), so a rare class gets its logit pushed DOWN during //--- training, forcing the weights to produce a larger raw logit to compensate. At inference the //--- offsets are absent, so that surplus becomes the calibrated boost the rare class needs. void SetLogitAdjustment(const double &offsets[]); void ClearLogitAdjustment(void) { bLogitAdjust = false; } bool HasLogitAdjustment(void) { return bLogitAdjust; } void getResults(CArrayDouble *&resultVals) ; double getRecentAverageError() { return recentAverageError; } //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array //--- when there is nothing to persist/restore. bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]); //--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it //--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see //--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud. bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false); //--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). CaptureWeights() snapshots //--- every neuron's weights (base/conv/LSTM) into host arrays; RestoreWeights() writes them back IN //--- PLACE via setWeights - reusing the existing neuron objects and their already-allocated device //--- buffers, exactly like BlendWeightsFrom(). This REPLACED an earlier file-based checkpoint pair //--- (since removed - it had no callers left) for the mid-run stability restore, because a file path RE-CREATES every neuron on //--- load (fresh CLayer + Init), and the multithreaded CPU-DLL backend (CDirectMLMy/WarriorCPU.dll) //--- cannot allocate a second full set of neuron tensors while the live set still exists - so the //--- file restore failed ("read 0 layers"), the model could never roll back a regressed era, and it //--- drifted into a Neutral collapse. In-place weight copy uses only getWeights/setWeights, which the //--- per-era shadow blend already exercises successfully on that backend. Snapshots WEIGHTS only (not //--- Adam moments); the regression handler decays eta on restore and per-step deltas are clipped, so //--- stale moments can't overshoot. In-memory => valid only within a single Train() run (same as the //--- ephemeral _ckpt.tmp was), which is exactly its scope. bool CaptureWeights(void); bool RestoreWeights(void); bool HaveWeightSnapshot(void) const { return m_haveWeightSnapshot; } //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight + //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation. //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) - //--- true whenever the shadow was cloned from live via Save()/Load() and never independently //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers. bool BlendWeightsFrom(CNet &live, double tau); //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly //--- initialized and carry the real learning signal) of each output neuron's incoming weight block, //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more //--- non-majority classes than the true base rate warrants until backProp corrects it over many //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns //--- false (no-op) rather than corrupt anything if that assumption doesn't hold. bool SeedOutputLayerBias(const double &biasValues[]); //--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Set BEFORE Load() in an inference-only //--- backtest: it makes InitOpenCL()/InitDirectML() no-op (both backends stay NULL), so the OCL //--- neurons load their weights host-side only and feedForward() runs the double-precision MQL5 //--- path (feedForwardCPU) reading those same host buffers. Training/optimization never set this //--- (they always want a backend), so their behaviour is unchanged. See ExpertSignalAIBase.mqh's //--- inference-only wiring and the deploy-time validation that gates it. void SetCpuInference(bool v) { m_cpuInference = v; } bool CpuInference(void) const { return m_cpuInference; } //--- Re-assert the output layer's activation after a Load(), and report what it used to be. //--- WHY THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. CNeuronBase::Save/ //--- CNeuronBaseOCL::Save write (int)activation per neuron and the matching Load() reads it straight //--- back into the live object, so the activation chosen in BuildFreshTopology() only ever applies to //--- a genuinely NEW topology. Every reload restores whatever is on disk and the next Save() writes it //--- back out - a wrong value can never heal on its own, while the source file reads as though it were //--- already fixed. That is exactly how models kept training with an unbounded NONE classification head //--- for a full day after BuildFreshTopology() had been reverted to SIGMOID (2026-07-29): confirmed by //--- parsing the binaries, `layer N: BaseOCL act=NONE out=3`, while a freshly reset model of the same //--- config read act=SIGMOID. Symptom was negative "OOS raw out" values (impossible under sigmoid) //--- escalating to a 4.1e13 logit spread with all three classes numerically identical. //--- Only the output layer is repaired here: it is always a plain dense layer whose activation is a //--- single unambiguous expression in BuildFreshTopology(). Hidden layers are deliberately left alone - //--- they legitimately differ per stage (PRELU dense, PRELU conv, NONE pool, TANH LSTM), so blanket //--- re-assertion there would corrupt exactly the topologies it was meant to protect. //--- Returns true when a repair was actually made, and reports the stale value through `previous`. bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous); //--- Freeze/unfreeze every batch-normalization layer's running statistics (AI\NeuronBatchNorm.mqh). //--- Frozen, a forward pass is a pure function of its input; unfrozen (the default) it also advances //--- the statistics. Anything that COMPARES two forward passes must freeze first or it measures its //--- own side effect. No-op on a net with no normalization layers. void SetBatchNormFrozen(bool frozen); //--- static double recentAverageSmoothingFactor; private: CArrayLayer *layers; COpenCLMy *opencl; CDirectMLMy *directml; double recentAverageError; bool m_cpuInference; //--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per //--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture. CArrayObj *m_weightSnapshot; bool m_haveWeightSnapshot; //--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). //--- A run legitimately builds SEVERAL CNet objects - the main Net, the EMA shadow net, the OOS-sim //--- clone, the deploy-time MQL5-inference self-check clone - and each one probed the backends //--- independently. On a host without OpenCL that reprinted the same 3-line banner per net //--- ("OpenCL not found, error code=5100" comes from the STDLIB COpenCL::Initialize, so it can't be //--- silenced at our call site), which read in the log like the EA was initializing twice. //--- s_openclUnavailable: latched only on FAILURE, and only ever skips a probe that is already known //--- to fail - OpenCL availability cannot change inside a process. A host that HAS OpenCL never //--- latches, so every CNet still gets its own COpenCLMy. Skipping also keeps GetLastError() free of //--- the harmless 5100 for later callers. //--- s_computeTierLogged: suppresses only the repeat of the informational "tier active" line (the //--- tier is a property of the HOST, identical for every net). Failure messages stay loud every time. static bool s_openclUnavailable; static bool s_computeTierLogged; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over bool CNet::s_openclUnavailable = false; bool CNet::s_computeTierLogged = false; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNet::CNet(CArrayObj *Description) { //--- Before the early returns below: CNet(NULL) is a legitimate construction (see //--- InitNeuralNetwork) and must still leave the adjustment cleanly disabled. bLogitAdjust = false; ArrayInitialize(dLogitAdjust, 0.0); if(CheckPointer(Description) == POINTER_INVALID) return; //--- int total = Description.Total(); if(total <= 0) return; //--- layers = new CArrayLayer(); if(CheckPointer(layers) == POINTER_INVALID) return; //--- CLayer *temp; CLayerDescription *desc = NULL, *next = NULL, *prev = NULL; CNeuronBase *neuron = NULL; CNeuronPool *neuron_p = NULL; int output_count = 0; int temp_count = 0; //--- next = Description.At(1); if(CheckPointer(next) != POINTER_INVALID && (next.type == defNeuron || next.type == defNeuronBaseOCL || next.type == defNeuronConv || next.type == defNeuronConvOCL || next.type == defNeuronLSTM || next.type == defNeuronBatchNorm || next.type == defNeuronBatchNormOCL)) { //--- OpenCL first, DirectML/D3D12 next, plain CPU as the final fallback if(!InitOpenCL()) InitDirectML(); } //--- Batch normalization exists only in the OCL neuron model (AI\NeuronBatchNorm.mqh); the legacy //--- scalar CNeuron path below has no counterpart. Rather than silently build a DIFFERENT network //--- than the topology asked for - which is precisely the class of bug that had models training //--- against a stale output head for a day - refuse, loudly, and leave an empty net behind. Callers //--- already treat a 0-layer net as a hard failure. Reaching here means OpenCL, DirectML AND the //--- CPU DLL all failed to initialize, which this project does not support for training anyway. if(CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID) { for(int i = 0; i < total; i++) { CLayerDescription *probe = Description.At(i); if(CheckPointer(probe) != POINTER_INVALID && (probe.type == defNeuronBatchNorm || probe.type == defNeuronBatchNormOCL)) { Print(__FUNCTION__ + ": REFUSED - topology requests a batch-normalization layer but no compute" " backend initialized (no OpenCL, no DirectML, no CPU DLL). Batch norm has no scalar-CPU" " implementation; building without it would silently train a different architecture."); return; } } } //--- for(int i = 0; i < total; i++) { prev = desc; desc = Description.At(i); if((i + 1) < total) { next = Description.At(i + 1); if(CheckPointer(next) == POINTER_INVALID) return; } else next = NULL; //--- How many outgoing weights this layer carries. The convention in this engine is that the //--- weight matrix feeding layer L is stored ON layer L-1, so only a DENSE successor claims one: //--- conv/pool/LSTM own their weights internally, and batch norm is elementwise and has none at //--- all (just gamma/beta, which live in its own parameter block). Getting this wrong for the new //--- type would allocate a full dense matrix that nothing ever reads or trains. int outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL) ? 0 : next.count); temp = new CLayer(outputs); int neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL ? 1 : 0)); if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { CNeuronBaseOCL *neuron_ocl = NULL; switch(desc.type) { case defNeuron: case defNeuronBaseOCL: neuron_ocl = new CNeuronBaseOCL(); if(CheckPointer(neuron_ocl) == POINTER_INVALID) { delete temp; return; } if(CheckPointer(opencl) != POINTER_INVALID ? !neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization) : !neuron_ocl.Init(outputs, 0, directml, desc.count, desc.optimization)) { delete temp; return; } neuron_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_ocl)) { delete neuron_ocl; delete temp; return; } neuron_ocl = NULL; break; case defNeuronBatchNorm: case defNeuronBatchNormOCL: { CNeuronBatchNormOCL *neuron_bn = new CNeuronBatchNormOCL(); if(CheckPointer(neuron_bn) == POINTER_INVALID) { delete temp; return; } //--- Elementwise, so this layer is exactly as wide as the one below it. Take that width //--- from the layer already built rather than from desc.count: a conv or pool stage's //--- output size is derived HERE (the sliding-window arithmetic above), so the topology //--- builder in ExpertSignalAIBase.mqh has no way to know it and cannot state it. Falls //--- back to desc.count only for the impossible case of a batch-norm layer at index 0. int bnUnits = desc.count; if(layers.Total() > 0) { CLayer *below = layers.At(layers.Total() - 1); if(CheckPointer(below) != POINTER_INVALID && below.Total() > 0) { CNeuronBaseOCL *belowNeuron = below.At(0); if(CheckPointer(belowNeuron) != POINTER_INVALID && belowNeuron.Neurons() > 0) bnUnits = belowNeuron.Neurons(); } } bool bnInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_bn.Init(outputs, 0, opencl, bnUnits, desc.batch, desc.optimization) : neuron_bn.Init(outputs, 0, directml, bnUnits, desc.batch, desc.optimization)); if(!bnInit) { delete neuron_bn; delete temp; return; } if(!temp.Add(neuron_bn)) { delete neuron_bn; delete temp; return; } neuron_bn = NULL; //--- Keep the running conv/pool sizing cursor pointing at this layer's real width, so a //--- conv or pool stage placed ABOVE a batch-norm layer still sizes correctly. output_count = bnUnits; break; } case defNeuronConv: case defNeuronConvOCL: { CNeuronConvOCL *neuron_conv = new CNeuronConvOCL(); if(CheckPointer(neuron_conv) == POINTER_INVALID) { delete temp; return; } //--- number of sliding positions - same formula the CPU CNeuronConv path uses if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL) { int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count); temp_count = (prevCount - desc.window) % desc.step; output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } else { temp_count = (output_count - desc.window) % desc.step; output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } bool convInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, output_count, desc.optimization) : neuron_conv.Init(outputs, 0, directml, desc.window, desc.step, desc.count, output_count, desc.optimization)); if(!convInit) { delete neuron_conv; delete temp; return; } neuron_conv.SetActivationFunction(desc.activation); if(!temp.Add(neuron_conv)) { delete neuron_conv; delete temp; return; } neuron_conv = NULL; break; } case defNeuronPool: case defNeuronPoolOCL: { CNeuronPoolOCL *neuron_pool = new CNeuronPoolOCL(); if(CheckPointer(neuron_pool) == POINTER_INVALID) { delete temp; return; } //--- number of sliding positions - same formula the CPU CNeuronPool path uses if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL) { int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count); temp_count = (prevCount - desc.window) % desc.step; output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } else { temp_count = (output_count - desc.window) % desc.step; output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } bool poolInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_pool.Init(outputs, 0, opencl, desc.window, desc.step, output_count, desc.optimization) : neuron_pool.Init(outputs, 0, directml, desc.window, desc.step, output_count, desc.optimization)); if(!poolInit) { delete neuron_pool; delete temp; return; } if(!temp.Add(neuron_pool)) { delete neuron_pool; delete temp; return; } neuron_pool = NULL; break; } case defNeuronLSTM: case defNeuronLSTMOCL: { CNeuronLSTMOCL *neuron_lstm = new CNeuronLSTMOCL(); if(CheckPointer(neuron_lstm) == POINTER_INVALID) { delete temp; return; } bool lstmInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_lstm.Init(outputs, 0, opencl, desc.count, desc.optimization) : neuron_lstm.Init(outputs, 0, directml, desc.count, desc.optimization)); if(!lstmInit) { delete neuron_lstm; delete temp; return; } if(!temp.Add(neuron_lstm)) { delete neuron_lstm; delete temp; return; } neuron_lstm = NULL; break; } default: return; break; } } else for(int n = 0; n < neurons; n++) { switch(desc.type) { case defNeuron: neuron = new CNeuron(); if(CheckPointer(neuron) == POINTER_INVALID) { delete temp; delete layers; return; } //--- He-scaled init, matching CNeuronBaseOCL::Init's rationale - fan-in is this //--- layer's own neuron count (bias already included via the `neurons` count above). neuron.Init(outputs, n, desc.optimization, MathSqrt(2.0 / (double)neurons)); neuron.SetActivationFunction(desc.activation); break; case defNeuronConv: neuron_p = new CNeuronConv(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronPool: neuron_p = new CNeuronPool(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronLSTM: neuron_p = new CNeuronLSTM(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } output_count = (next != NULL ? next.window : desc.step); if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization)) neuron = neuron_p; break; } if(!temp.Add(neuron)) { delete temp; delete layers; return; } neuron = NULL; } if(!layers.Add(temp)) { delete temp; delete layers; return; } } //--- } //+------------------------------------------------------------------+ //| Tries to initialize OpenCL; on any failure (no GPU, driver | //| missing, kernel build error) frees it and leaves opencl==NULL | //| so the rest of CNet transparently runs its CPU code path. | //+------------------------------------------------------------------+ bool CNet::InitOpenCL(void) { //--- pure-MQL5 inference: deliberately refuse a backend so the OCL neurons compute host-side. if(m_cpuInference) return false; if(CheckPointer(opencl) != POINTER_INVALID) return true; //--- already established (by an earlier CNet in this process) that this host has no OpenCL - skip the //--- probe rather than re-run a known failure and reprint the stdlib's banner. See s_openclUnavailable. if(s_openclUnavailable) return false; //--- opencl = new COpenCLMy(); if(CheckPointer(opencl) == POINTER_INVALID || !opencl.Initialize(cl_program, true)) { if(CheckPointer(opencl) != POINTER_INVALID) delete opencl; opencl = NULL; s_openclUnavailable = true; PrintFormat("%s: OpenCL unavailable, falling back to CPU", __FUNCTION__); return false; } //--- create kernels opencl.SetKernelsCount(18); opencl.KernelCreate(def_k_FeedForward, "FeedForward"); opencl.KernelCreate(def_k_CaclOutputGradient, "CaclOutputGradient"); opencl.KernelCreate(def_k_CaclHiddenGradient, "CaclHiddenGradient"); opencl.KernelCreate(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum"); opencl.KernelCreate(def_k_UpdateWeightsAdam, "UpdateWeightsAdam"); opencl.KernelCreate(def_k_FeedForwardConv, "FeedForwardConv"); opencl.KernelCreate(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv"); opencl.KernelCreate(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum"); opencl.KernelCreate(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam"); opencl.KernelCreate(def_k_LSTM_Gates, "LSTM_Gates"); opencl.KernelCreate(def_k_LSTM_State, "LSTM_State"); opencl.KernelCreate(def_k_LSTM_GateGradient, "LSTM_GateGradient"); opencl.KernelCreate(def_k_LSTM_WeightsGradient, "LSTM_WeightsGradient"); opencl.KernelCreate(def_k_LSTM_InputsGradient, "LSTM_InputsGradient"); opencl.KernelCreate(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam"); opencl.KernelCreate(def_k_LSTM_UpdateWeightsMomentum, "LSTM_UpdateWeightsMomentum"); opencl.KernelCreate(def_k_FeedForwardProof, "FeedForwardProof"); opencl.KernelCreate(def_k_CalcInputGradientProof, "CalcInputGradientProof"); return true; } //+------------------------------------------------------------------+ //| Second-tier GPU fallback: only tried when OpenCL init failed. | //| Requires DirectML\WarriorDML.dll in the terminal's Libraries | //| folder (build it with DirectML\build.bat); on any failure frees | //| itself and leaves directml==NULL so CNet falls through to CPU. | //+------------------------------------------------------------------+ bool CNet::InitDirectML(void) { //--- pure-MQL5 inference: deliberately refuse a backend so the OCL neurons compute host-side. if(m_cpuInference) return false; //--- Tester/optimization/forward runs must not touch WarriorDML/WarriorCPU DLL imports. //--- This avoids agent-side file-lock/synchronization failures on rapid stop/restart cycles. if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD)) return false; if(CheckPointer(directml) != POINTER_INVALID) return true; //--- directml = new CDirectMLMy(); if(CheckPointer(directml) == POINTER_INVALID) return false; directml.SetCpuLoadPercent(EffectiveCpuLoadPercent()); if(!directml.Initialize()) { int err = directml.LastError(); delete directml; directml = NULL; string reason; switch(err) { case 1: reason = "CreateDXGIFactory1 failed"; break; case 2: reason = "no DX12 hardware adapter found (feature level 11_0)"; break; case 3: reason = "compute command queue creation failed"; break; case 4: reason = "command allocator creation failed"; break; case 5: reason = "command list creation failed"; break; case 6: reason = "fence creation failed"; break; case 7: reason = "fence event creation failed"; break; case 8: reason = "HLSL kernel compile/PSO creation failed"; break; default: reason = "neither WarriorDML.dll nor WarriorCPU.dll loaded (check Libraries folder / \"Allow DLL imports\")"; } PrintFormat("%s: DirectML/D3D12 and CPU DLL both unavailable (%s), falling back to slow per-object CPU path", __FUNCTION__, reason); return false; } //--- Announce the tier ONCE per process: it describes the host, not this particular net, and a run builds //--- several nets (main + EMA shadow + sim/self-check clones). See s_computeTierLogged. if(!s_computeTierLogged) { s_computeTierLogged = true; if(directml.Tier() == COMPUTE_TIER_CPU) { //--- Report the SPLIT, not just the result. When several charts train at once this is the //--- number that explains their speed, and it is the one that was silently wrong before. int share = EffectiveCpuLoadPercent(); PrintFormat("%s: DirectML/D3D12 unavailable, using multithreaded CPU DLL fallback (%d threads per network, target %d; %d%% of %d detected cores - fixed, independent of how many charts run)", __FUNCTION__, directml.CpuThreadsUsed(), CPU_THREADS_PER_NETWORK, share, (int)TerminalInfoInteger(TERMINAL_CPU_CORES)); } else PrintFormat("%s: DirectML/D3D12 GPU tier active", __FUNCTION__); } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CArrayDouble *inputVals) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1) return false; //--- Pure-MQL5 inference: OCL-format neurons loaded host-only, computed in double precision. Separate //--- path because the branches below assume either a device backend or the plain CNeuronBase model. if(m_cpuInference) return feedForwardCPU(inputVals); //--- CLayer *previous = NULL; CLayer *current = layers.At(0); int total = MathMin(current.Total(), inputVals.Total()); CNeuronBase *neuron = NULL; bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID); if(!gpuActive) { for(int i = 0; i < total; i++) { neuron = current.At(i); if(CheckPointer(neuron) == POINTER_INVALID) return false; neuron.setOutputVal(inputVals.At(i)); } } else { CNeuronBaseOCL *neuron_ocl = current.At(0); int total_data = inputVals.Total(); bool written; if(CheckPointer(opencl) != POINTER_INVALID) { //--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own //--- BufferWrite(), this call bypasses that class entirely (it writes straight into the //--- first layer's Output buffer by index), so the narrow-to-float has to happen here too. float array[]; if(ArrayResize(array, total_data) < 0) return false; for(int d = 0; d < total_data; d++) array[d] = (float)inputVals.At(d); written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data); } else { double array[]; if(ArrayResize(array, total_data) < 0) return false; for(int d = 0; d < total_data; d++) array[d] = inputVals.At(d); written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data); } if(!written) return false; } //--- CObject *temp = NULL; for(int l = 1; l < layers.Total(); l++) { previous = current; current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID) return false; //--- if(gpuActive) { CNeuronBaseOCL *current_ocl = current.At(0); if(!current_ocl.feedForward(previous.At(0))) return false; continue; } //--- total = current.Total(); if(current.At(0).Type() == defNeuron) total--; //--- for(int n = 0; n < total; n++) { neuron = current.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return false; if(previous.At(0).Type() == defNeuron) { temp = previous; if(!neuron.feedForward(temp)) return false; continue; } if(neuron.Type() == defNeuron) { if(n == 0) { CLayer *temp_l = new CLayer(total); if(CheckPointer(temp_l) == POINTER_INVALID) return false; CNeuronPool *Pool = NULL; for(int p = 0; p < previous.Total(); p++) { Pool = previous.At(p); if(CheckPointer(Pool) == POINTER_INVALID) return false; temp_l.AddArray(Pool.getOutputLayer()); } temp = temp_l; } if(!neuron.feedForward(temp)) return false; if(n == total - 1) { CLayer *temp_l = temp; temp_l.FreeMode(false); temp_l.Shutdown(); delete temp_l; } continue; } temp = previous.At(n); if(CheckPointer(temp) == POINTER_INVALID) return false; if(!neuron.feedForward(temp)) return false; } } //--- return true; } //+------------------------------------------------------------------+ //| Pure-MQL5 forward pass over an OCL-format network loaded host-only | //| (no OpenCL/DirectML/DLL). Layer 0 is fed from inputVals; each | //| subsequent layer's OCL neuron computes via its virtual | //| feedForwardCPU() (dense/conv/pool/LSTM). See SetCpuInference(). | //+------------------------------------------------------------------+ bool CNet::feedForwardCPU(CArrayDouble *inputVals) { CLayer *current = layers.At(0); if(CheckPointer(current) == POINTER_INVALID) return false; CNeuronBaseOCL *in0 = current.At(0); if(CheckPointer(in0) == POINTER_INVALID || !in0.SetInputsCPU(inputVals)) return false; for(int l = 1; l < layers.Total(); l++) { CLayer *previous = current; current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID) return false; CNeuronBaseOCL *cur = current.At(0); CNeuronBaseOCL *prev = previous.At(0); if(CheckPointer(cur) == POINTER_INVALID || CheckPointer(prev) == POINTER_INVALID) return false; if(!cur.feedForwardCPU(prev)) return false; } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::backProp(CArrayDouble *targetVals, double sampleWeight) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID) return; if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { backPropOCL(targetVals, sampleWeight); return; } //--- CLayer *outputLayer = layers.At(layers.Total() - 1); if(CheckPointer(outputLayer) == POINTER_INVALID) return; //--- //--- Defensive: this is the pure-MQL5 CPU backward pass and it walks CNeuron/CNeuronBase objects //--- with SCALAR getOutputVal()/getGradient()/setGradient() accessors. CNeuronBaseOCL & friends do //--- NOT derive from CNeuronBase and expose only ARRAY accessors over a device buffer, so they can //--- never be walked here. That is normally impossible to reach: the early return above hands any //--- backend-backed net to backPropOCL(), and CNet::Create() only ever constructs OCL neurons when //--- a backend exists. The one way to hold OCL neurons with no backend is pure-MQL5 inference mode //--- (SetCpuInference -> CLayer::CreateElementScaled's host-only branch), which is inference-only //--- and never backprops (OnlineLearnStep() guards on Net.CpuInference()). Bail out loudly rather //--- than mis-cast if that ever changes. //--- 2026-07-28: a set of inline "handle OCL neuron types" branches was added throughout this //--- function to cover that impossible case. They could not compile - they called the scalar //--- accessors on CNeuronBaseOCL - and were removed; this guard replaces them. CObject *probe = outputLayer.At(0); if(CheckPointer(probe) != POINTER_INVALID) { int t0 = probe.Type(); if(t0 == defNeuronBaseOCL || t0 == defNeuronConvOCL || t0 == defNeuronPoolOCL || t0 == defNeuronLSTMOCL || t0 == defNeuronBatchNormOCL) { Print(__FUNCTION__ + ": REFUSED - CPU backward pass reached a net built from OpenCL/DirectML neurons with no compute backend attached. Nothing was trained this step."); return; } } //--- double error = 0.0; int total = outputLayer.Total() - 1; //--- 3-output classification case: true softmax + categorical-cross-entropy gradient //--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section //--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID //--- (bounded, avoids the historical logit-runaway collapse documented at //--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD //--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly, //--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/ //--- Neutral together during training: raising one class's softmax probability now structurally //--- lowers the other two's (via the shared normalizing sum), giving real competition instead of //--- three independent binary regressions that can all drift toward "predict Neutral" together - //--- root cause of the "overshoot to all-Neutral" convergence failure this replaces. bool useSoftmaxGrad = (total == 3); double smax[3]; if(useSoftmaxGrad) { //--- Logit adjustment added BEFORE the max-subtraction so the shift stays numerically safe. double logit[3]; double maxLogit = -DBL_MAX; for(int n = 0; n < 3; n++) { CNeuron *nrn = outputLayer.At(n); logit[n] = CLASS_LOGIT_SCALE * nrn.getOutputVal() + (bLogitAdjust ? dLogitAdjust[n] : 0.0); maxLogit = MathMax(maxLogit, logit[n]); } double sum = 0.0; for(int n = 0; n < 3; n++) { smax[n] = exp(logit[n] - maxLogit); sum += smax[n]; } for(int n = 0; n < 3; n++) smax[n] /= sum; } for(int n = 0; n < total && !IsStopped(); n++) { CNeuron *neuron = outputLayer.At(n); double target = targetVals.At(n); double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target); double delta = clampedTarget - neuron.getOutputVal(); error += delta * delta; if(useSoftmaxGrad) neuron.setGradient(clampedTarget - smax[n]); else neuron.calcOutputGradients(targetVals.At(n)); //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how //--- sampleWeight is derived) - scales the just-computed output gradient in place, before the //--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted //--- signal without needing its own separate weighting logic. if(sampleWeight != 1.0) neuron.setGradient(neuron.getGradient() * sampleWeight); } error /= total; error = sqrt(error); recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor; //--- CNeuronBase *neuron = NULL; CObject *temp = NULL; for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--) { CLayer *hiddenLayer = layers.At(layerNum); CLayer *nextLayer = layers.At(layerNum + 1); total = hiddenLayer.Total(); for(int n = 0; n < total && !IsStopped(); ++n) { neuron = hiddenLayer.At(n); if(nextLayer.At(0).Type() == defNeuron) { temp = nextLayer; neuron.calcHiddenGradients(temp); continue; } if(neuron.Type() == defNeuron) { double g = 0; for(int i = 0; i < nextLayer.Total(); i++) { temp = nextLayer.At(i); neuron.calcHiddenGradients(temp); g += neuron.getGradient(); } neuron.setGradient(g); continue; } temp = nextLayer.At(n); neuron.calcHiddenGradients(temp); } } //--- for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--) { CLayer *layer = layers.At(layerNum); CLayer *prevLayer = layers.At(layerNum - 1); total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0); int n_conv = 0; for(int n = 0; n < total && !IsStopped(); n++) { neuron = layer.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return; if(neuron.Type() == defNeuronPool) continue; switch(prevLayer.At(0).Type()) { case defNeuron: temp = prevLayer; neuron.updateInputWeights(temp); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: if(neuron.Type() == defNeuron) { for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++) { temp = prevLayer.At(n_conv); neuron.updateInputWeights(temp); } } else { temp = prevLayer.At(n); neuron.updateInputWeights(temp); } break; default: temp = NULL; break; } } } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID || (CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID)) return; CLayer *currentLayer = (CLayer*)layers.At(layers.Total() - 1); if(CheckPointer(currentLayer) == POINTER_INVALID) return; //--- double error = 0.0; int total = targetVals.Total(); double result[]; CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)currentLayer.At(0); if(neuron.getOutputVal(result) < total) return; for(int n = 0; n < total && !IsStopped(); n++) { double target = targetVals.At(n); // Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever // target==0, so a one-hot classification target only ever counted the true class's own // error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric, // which is what CNet::backProp()'s CPU-fallback path computes for every output // unconditionally, and is what actually drives the dError<0.1 convergence gate in // ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp // / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was. double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n]; error += MathPow(delta, 2); } error /= total; error = sqrt(error); recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor; if(!neuron.calcOutputGradients(targetVals)) return; //--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true //--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5 //--- from the raw outputs already read back into result[] above - see the matching CNet::backProp() //--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared //--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted //--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely //--- on the MQL5 side, so none of the 3 compute backends need to change. if(total == 3) { //--- Logit adjustment (see SetLogitAdjustment): tau*log(prior_c) per class, added to the logit //--- before the softmax. Backward pass ONLY - the forward pass and every inference path stay //--- untouched, which is the whole point: the network learns to absorb the offset, so at //--- inference its RAW argmax is already the balanced-error-optimal decision. double logit[3]; double maxLogit = -DBL_MAX; for(int n = 0; n < 3; n++) { logit[n] = CLASS_LOGIT_SCALE * result[n] + (bLogitAdjust ? dLogitAdjust[n] : 0.0); maxLogit = MathMax(maxLogit, logit[n]); } double smax[3]; double sm = 0.0; for(int n = 0; n < 3; n++) { smax[n] = exp(logit[n] - maxLogit); sm += smax[n]; } double gradOverwrite[3]; for(int n = 0; n < 3; n++) { smax[n] /= sm; double target = targetVals.At(n); double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target); gradOverwrite[n] = clampedTarget - smax[n]; } neuron.setGradient(gradOverwrite); } //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how //--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the //--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers //--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends. if(sampleWeight != 1.0) { double gradVals[]; int gradCount = neuron.getGradient(gradVals); if(gradCount > 0) { for(int g = 0; g < gradCount; g++) gradVals[g] *= sampleWeight; neuron.setGradient(gradVals); } } //--- Calc Hidden Gradients CObject *temp = NULL; total = layers.Total(); for(int layerNum = total - 2; layerNum > 0; layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); neuron = currentLayer.At(0); neuron.calcHiddenGradients(nextLayer.At(0)); } //--- Layer-1 LSTM special case. The loop above deliberately stops at layerNum > 0 because the INPUT //--- layer needs no gradient of its own - true for every layer type whose updateInputWeights() derives //--- its own weight deltas from (own gradient x previous output) inside the kernel: dense //--- (UpdateWeightsMomentum/Adam) and conv (UpdateWeightsConvMomentum/Adam) both do. //--- CNeuronLSTMOCL is the ONE exception: LSTM_UpdateWeightsMomentum/Adam do not derive anything, they //--- only CONSUME the WeightsGradient buffer, and that buffer is filled purely as a SIDE EFFECT of //--- CNeuronLSTMOCL::calcInputGradients() - which is invoked by the layer BELOW, via its //--- calcHiddenGradients(target=thisLSTM) dispatch. So an LSTM sitting at layer index 1 (the LSTM_2L //--- preset: input -> LSTM -> dense -> dense -> output) never gets calcInputGradients() called at all: //--- WeightsGradient stays at its BufferInit(total, 0) zeros forever, Adam's mt/vt therefore stay 0 and //--- every weight delta is exactly 0. The LSTM silently never trains - it stays a frozen random //--- recurrent projection while only the dense taper above it learns, which shows up as ~0% Buy/Sell //--- recall and a raw-output range that barely moves off its initialisation. //--- HYBRID_2L (input -> conv -> pool -> LSTM -> dense -> dense -> output) is unaffected: its LSTM is at //--- index 3, so the pool layer below it makes the call in the normal course of the loop. //--- Done here rather than by relaxing the loop bound so the loop's currentLayer/nextLayer bookkeeping //--- is untouched, and so no other topology pays for an extra kernel dispatch it does not need. if(total >= 3) { CLayer *firstHidden = layers.At(1); CLayer *inputLayer = layers.At(0); if(CheckPointer(firstHidden) != POINTER_INVALID && CheckPointer(inputLayer) != POINTER_INVALID && CheckPointer(firstHidden.At(0)) != POINTER_INVALID && CheckPointer(inputLayer.At(0)) != POINTER_INVALID && firstHidden.At(0).Type() == defNeuronLSTMOCL) { CNeuronLSTMOCL *lstm = firstHidden.At(0); CNeuronBaseOCL *inputNeuron = inputLayer.At(0); //--- Safe to run now and only now: the loop above has already filled this LSTM's own Gradient //--- (it processed layerNum == 1), which is exactly what LSTMGateGradient reads. if(!lstm.calcInputGradients(inputNeuron)) printf("%s: LSTM at layer 1 failed to compute its weight gradients - it will not train this step", __FUNCTION__); } } //--- CLayer *prevLayer = layers.At(total - 1); for(int layerNum = total - 1; layerNum > 0; layerNum--) { currentLayer = prevLayer; prevLayer = layers.At(layerNum - 1); neuron = currentLayer.At(0); neuron.updateInputWeights(prevLayer.At(0)); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::getResults(CArrayDouble *&resultVals) { if(CheckPointer(resultVals) == POINTER_INVALID) { resultVals = new CArrayDouble(); if(CheckPointer(resultVals) == POINTER_INVALID) return; } //--- resultVals.Clear(); if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0) return; //--- CLayer *output = layers.At(layers.Total() - 1); if(CheckPointer(output) == POINTER_INVALID) return; //--- if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { switch(output.At(0).Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: { CNeuronBaseOCL *temp = output.At(0); temp.getOutputVal(resultVals); return; } } } //--- pure-MQL5 inference: OCL output neuron computed host-side, read without a device BufferRead. if(m_cpuInference) { switch(output.At(0).Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: { CNeuronBaseOCL *temp = output.At(0); temp.GetOutputsCPU(resultVals); return; } } } CNeuronBase *neuron = NULL; CLayer *temp = NULL; int total = output.Total(); if(output.At(0).Type() == defNeuron) total--; //--- for(int i = 0; i < total; i++) { CObject *obj = output.At(i); if(CheckPointer(obj) == POINTER_INVALID) continue; if(obj.Type() == defNeuron) { neuron = (CNeuronBase*)obj; resultVals.Add(neuron.getOutputVal()); continue; } if(obj.Type() == defNeuronPool) { CNeuronPool *n = (CNeuronPool*)obj; temp = n.getOutputLayer(); for(int ii = 0; ii < temp.Total(); ii++) { CObject *poolObj = temp.At(ii); if(CheckPointer(poolObj) == POINTER_INVALID) continue; CNeuronBase *poolNeuron = (CNeuronBase*)poolObj; resultVals.Add(poolNeuron.getOutputVal()); } } } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]) { //--- only the shared FILE_COMMON production weights are protected from being overwritten by //--- backtest/optimization noise - a LOCAL (common=false) file is exactly what the tester's //--- per-agent cross-pass weight cache uses (see CExpertSignalAIBase::InitNeuralNetwork), and //--- must be allowed to write even inside the tester/optimizer. if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD))) return true; if(file_name == NULL) return false; //--- Never persist a layerless network. FileOpen(FILE_WRITE) below TRUNCATES on open, so writing an //--- empty net here would replace a good ~18MB model on disk with a ~330-byte header-only stub (0 //--- layers) - silent, total loss of trained weights. This guard is checked BEFORE FileOpen so a bad //--- in-memory state (e.g. a failed checkpoint restore that emptied the layers) can't clobber the file. if(CheckPointer(layers) == POINTER_INVALID || layers.Total() == 0) { Print("CNet::Save: refusing to write ", file_name, " - network has 0 layers (would overwrite a valid model with an empty stub)"); return false; } //--- ATOMIC SAVE (crash-safe) - see System\AtomicFile.mqh for why every write here is staged through a //--- temp file. Short version: FileOpen(FILE_WRITE) truncates on open, so writing straight to file_name //--- would zero the good model the instant the save starts, and an interrupted save then leaves a //--- truncated .nnw that the next load rebuilds from era 0 (the recurring data loss). int commonFlag = (common ? FILE_COMMON : 0); string tmpName = ""; int handle = AtomicWriteBegin(file_name, commonFlag, tmpName); if(handle == INVALID_HANDLE) return false; //--- bool ok = true; if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0 || FileWriteLong(handle, era) <= 0 || FileWriteInteger(handle, trainingComplete ? 1 : 0) <= 0) ok = false; //--- AutoTuneIndicators "winning" AD indicator params, same append-only pattern as the era/ //--- trainingComplete fields above: count-prefixed so older readers can still stop before this block int paramsCount = ArraySize(indicatorParams); if(ok && FileWriteInteger(handle, paramsCount) <= 0) ok = false; for(int p = 0; ok && p < paramsCount; p++) if(FileWriteDouble(handle, indicatorParams[p]) <= 0) ok = false; if(ok && !layers.Save(handle)) ok = false; return AtomicWriteEnd(handle, file_name, tmpName, commonFlag, ok, "CNet::Save"); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet) { //--- see the matching comment in Save() - only shared FILE_COMMON production weights are blocked if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD))) return false; //--- if(file_name == NULL) return false; //--- if(!quiet) Print(file_name); //--- FILE_SHARE_READ|FILE_SHARE_WRITE: this is a READ - it never needs exclusive access, and demanding //--- it (the default) makes the open fail with 5004 whenever any other process holds the file, which is //--- the normal state of a deployed model while a live chart runs this EA. See CopySharedFile(). int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ | FILE_SHARE_READ | FILE_SHARE_WRITE); if(handle == INVALID_HANDLE) { //--- Every OTHER rejection below prints why; this one previously didn't, which is how a real cause //--- (a transient Windows sharing violation from a concurrent writer - e.g. a live chart's atomic //--- Save() renaming this same file mid-read) looked identical to "no file yet" in the journal. Name //--- it explicitly so a recurrence is diagnosable instead of a silent fall-through to an untrained net. if(!quiet) Print("CNet::Load: " + file_name + " - FileOpen failed, error " + IntegerToString(GetLastError()) + ". If this file is known to exist, this is likely a transient lock (e.g. another process " + "writing the same file concurrently) rather than a missing/corrupt file."); return false; } //--- Capture the on-disk size up front: it discriminates the two failure modes a rejected read can have. //--- A healthy multi-layer model is multi-MB; a size of only a few hundred bytes/KB = a TRUNCATED file //--- (interrupted save). A full-size file that STILL fails to load layers = a backend ALLOCATION failure //--- (e.g. the CPU-DLL still holding a prior net's tensors after a skipped teardown), NOT truncation. ulong fileSizeBytes = FileSize(handle); //--- error = FileReadDouble(handle); undefine = FileReadDouble(handle); forecast = FileReadDouble(handle); time = (datetime)FileReadLong(handle); era = FileReadLong(handle); // Older save files predate this field - FileReadInteger returns 0 past EOF, // which correctly defaults to "not complete" so a resumed run keeps training // instead of silently trusting an unfinished/unknown model as done. trainingComplete = (FileReadInteger(handle) != 0); // Older save files predate this block too - FileReadInteger returns 0 past EOF, which // correctly yields an empty indicatorParams (nothing to restore) instead of misreading weights. ArrayFree(indicatorParams); int paramsCount = FileReadInteger(handle); if(paramsCount > 0) { ArrayResize(indicatorParams, paramsCount); for(int p = 0; p < paramsCount; p++) indicatorParams[p] = FileReadDouble(handle); } //--- if(CheckPointer(layers) != POINTER_INVALID) layers.Clear(); else layers = new CArrayLayer(); int i = 0, num; //--- if(!InitOpenCL()) InitDirectML(); //--- check //--- read and check start marker - 0xFFFFFFFFFFFFFFFF //--- DIAGNOSTIC: every failure below returns a bare false, and the caller can only surface GetLastError() //--- which on a no-GPU/CPU-DLL box is polluted with the harmless 5100 from the OpenCL probe above - so a //--- genuinely corrupt/partial/incompatible file looked identical to a missing one for several sessions of //--- "restart -> era 0" hunting. Name the exact failure so the next restart is diagnosable at a glance. long temp = FileReadLong(handle); if(temp != -1) { FileClose(handle); if(!quiet) Print("CNet::Load: " + file_name + " - REJECTED: bad start marker (read " + IntegerToString(temp) + ", expected -1) => the header is corrupt or the file was truncated by an interrupted save. This is NOT a compute/OpenCL problem."); return(false); } //--- read and check array type int savedType = FileReadInteger(handle, INT_VALUE); if(savedType != layers.Type()) { FileClose(handle); if(!quiet) Print("CNet::Load: " + file_name + " - REJECTED: layer-array type mismatch (read " + IntegerToString(savedType) + ", expected " + IntegerToString(layers.Type()) + ") => file format/version incompatible. This is NOT a compute/OpenCL problem."); return(false); } //--- read array length num = FileReadInteger(handle, INT_VALUE); //--- read array int failedLayer = -1; // index of the first layer that failed to load, or -1 if all loaded if(num != 0) { for(i = 0; i < num; i++) { //--- create new element CLayer *Layer = new CLayer(0, handle, opencl, directml); if(CheckPointer(Layer) == POINTER_INVALID) { failedLayer = i; break; } //--- On a partial/corrupt file, Layer.Load() or layers.Add() can fail after the CLayer was //--- already allocated. Deleting it before breaking prevents the orphaned-CLayer leak MT5 reports //--- at unload ("1 object of class 'CLayer'") - the layer was never added to `layers`, so the //--- CNet destructor (which frees `layers`) can't reclaim it. if(!Layer.Load(handle)) { delete Layer; failedLayer = i; break; } if(!layers.Add(Layer)) { delete Layer; failedLayer = i; break; } } } FileClose(handle); //--- result: a 0-layer file is not a usable model (it's the empty stub the Save guard now prevents, but //--- older stubs may still be on disk) - report failure so the caller rebuilds/retrains instead of //--- running inference on a layerless network. if(num == 0) { if(!quiet) Print("CNet::Load: " + file_name + " - REJECTED: 0-layer stub (empty model, header says " + IntegerToString(num) + " layers). An older empty-save stub; the current Save guard prevents these. Use reset-weights to start clean. NOT a compute/OpenCL problem."); return(false); } if(failedLayer >= 0 || layers.Total() != num) { if(!quiet) { string backend = (CheckPointer(opencl) != POINTER_INVALID ? "OpenCL" : (CheckPointer(directml) != POINTER_INVALID ? "CPU-DLL" : "pure-MQL5")); //--- Which failure mode this is depends on WHERE it stopped, not just on the file size: //--- - failed at layer 0 with a full-size file: nothing was read yet, so the file cannot be the //--- cause. This is a CODE fault in the read path - historically CLayer::CreateElement losing //--- its virtual override of CArrayObj::CreateElement (see the note on that method), which //--- made CArrayObj::Load() hit the base's `return(false)` stub on the very first neuron. //--- - failed at a LATER layer: the header/earlier layers read fine, so the file really does end //--- early => truncated by an interrupted save (Save() is atomic now, so this should be a //--- legacy file only). //--- - tiny file: truncated, whatever the layer index. string likely; if(fileSizeBytes < 65536) likely = "the file is only " + IntegerToString((int)fileSizeBytes) + " bytes => TRUNCATED (interrupted save, pre-atomic-Save file)"; else if(failedLayer == 0) likely = "the file is " + IntegerToString((int)(fileSizeBytes / 1024)) + " KB and it failed on the FIRST layer, before any layer data mattered" + " => this is a READ-PATH CODE fault, not a bad file and not a " + backend + " allocation problem." + " Check that CLayer::CreateElement still exactly matches CArrayObj::CreateElement's signature (const int) so it overrides it," + " and that CLayer::Load/CNeuronBaseOCL::Load are in sync with Save()"; else likely = "the file is " + IntegerToString((int)(fileSizeBytes / 1024)) + " KB and layers 0.." + IntegerToString(failedLayer - 1) + " read fine => the file ends early (TRUNCATED by an interrupted save) or a neuron tensor could not be allocated on the " + backend; Print("CNet::Load: " + file_name + " - REJECTED: only loaded " + IntegerToString(layers.Total()) + " of " + IntegerToString(num) + " layers (failed at layer " + IntegerToString(failedLayer) + "). Diagnosis: " + likely + "."); } return(false); } return true; } //+------------------------------------------------------------------+ //| See this method's declaration comment for the EMA shadow-weight | //| deployment rationale. | //+------------------------------------------------------------------+ bool CNet::BlendWeightsFrom(CNet &live, double tau) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID) return false; int layerTotal = MathMin(layers.Total(), live.layers.Total()); for(int l = 0; l < layerTotal; l++) { CLayer *shadowLayer = layers.At(l); CLayer *liveLayer = live.layers.At(l); if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID) continue; int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total()); for(int n = 0; n < neuronTotal; n++) { //--- Type() first, through the true common base (CObject) - NOT through a CNeuronBaseOCL*-typed //--- pointer. On the plain-CPU tier (no OpenCL/DirectML - the only option on Marketplace, which //--- forbids DLLs) CNet::CNet() builds legacy CNeuronBase-hierarchy neurons (CNeuron/CNeuronConv/ //--- CNeuronPool/CNeuronLSTM), an UNRELATED class hierarchy from CNeuronBaseOCL/CNeuronConvOCL/ //--- CNeuronLSTMOCL. Assigning one of those objects to a CNeuronBaseOCL* and calling its virtual //--- methods (as this used to do unconditionally) reads the wrong vtable/member layout - undefined //--- behaviour, not just a silent no-op, every single online-learning step on that tier. CObject *shadowObj = shadowLayer.At(n); CObject *liveObj = liveLayer.At(n); if(CheckPointer(shadowObj) == POINTER_INVALID || CheckPointer(liveObj) == POINTER_INVALID) continue; if(shadowObj.Type() != liveObj.Type()) continue; double shadowW[], liveW[]; switch(shadowObj.Type()) { case defNeuronBaseOCL: { CNeuronBaseOCL *shadowNeuron = shadowObj; CNeuronBaseOCL *liveNeuron = liveObj; if(shadowNeuron.getWeights(shadowW) > 0 && liveNeuron.getWeights(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowNeuron.setWeights(shadowW); } } break; case defNeuronBatchNormOCL: { //--- getWeightsBN packs the outgoing dense matrix and gamma/beta/statistics into one //--- flat array, so the shared blend loop below needs no special case of its own. CNeuronBatchNormOCL *shadowBN = shadowObj; CNeuronBatchNormOCL *liveBN = liveObj; if(shadowBN.getWeightsBN(shadowW) > 0 && liveBN.getWeightsBN(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowBN.setWeightsBN(shadowW); } } break; case defNeuronConvOCL: { CNeuronConvOCL *shadowConv = shadowObj; CNeuronConvOCL *liveConv = liveObj; if(shadowConv.getWeightsConv(shadowW) > 0 && liveConv.getWeightsConv(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowConv.setWeightsConv(shadowW); } } break; case defNeuronLSTMOCL: { CNeuronLSTMOCL *shadowLstm = shadowObj; CNeuronLSTMOCL *liveLstm = liveObj; if(shadowLstm.getWeightsLSTM(shadowW) > 0 && liveLstm.getWeightsLSTM(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowLstm.setWeightsLSTM(shadowW); } } break; //--- Plain-CPU tier (no backend at all): dense/conv/pool/LSTM legacy neurons store weights //--- per-connection (CConnection.weight) rather than in one contiguous buffer - blend those //--- directly instead of silently dropping the shadow-EMA deployment on this tier. case defNeuron: case defNeuronConv: case defNeuronPool: case defNeuronLSTM: { CNeuronBase *shadowBase = shadowObj; CNeuronBase *liveBase = liveObj; CArrayCon *shadowCon = shadowBase.getConnections(); CArrayCon *liveCon = liveBase.getConnections(); if(CheckPointer(shadowCon) == POINTER_INVALID || CheckPointer(liveCon) == POINTER_INVALID) break; int ct = MathMin(shadowCon.Total(), liveCon.Total()); for(int ci = 0; ci < ct; ci++) { CConnection *sc = shadowCon.At(ci); CConnection *lc = liveCon.At(ci); if(CheckPointer(sc) == POINTER_INVALID || CheckPointer(lc) == POINTER_INVALID) continue; sc.weight = (1.0 - tau) * sc.weight + tau * lc.weight; } } break; } } } return true; } //+------------------------------------------------------------------+ //| Snapshot every neuron's weights into host memory (see the header | //| declaration). Layer-major order; one CArrayDouble per neuron. Used | //| as the mid-run best-era checkpoint - restored by RestoreWeights(). | //+------------------------------------------------------------------+ bool CNet::CaptureWeights(void) { if(CheckPointer(layers) == POINTER_INVALID) return false; if(CheckPointer(m_weightSnapshot) == POINTER_INVALID) { m_weightSnapshot = new CArrayObj(); if(CheckPointer(m_weightSnapshot) == POINTER_INVALID) return false; } m_weightSnapshot.Clear(); // FreeMode deletes the previous snapshot's per-neuron arrays m_haveWeightSnapshot = false; for(int l = 0; l < layers.Total(); l++) { CLayer *layer = (CLayer*)layers.At(l); if(CheckPointer(layer) == POINTER_INVALID) return false; for(int n = 0; n < layer.Total(); n++) { CObject *obj = layer.At(n); if(CheckPointer(obj) == POINTER_INVALID) continue; double w[]; int got = 0; switch(obj.Type()) { case defNeuronBaseOCL: { CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj; got = neuron.getWeights(w); break; } case defNeuronBatchNormOCL: { //--- Dense matrix + gamma/beta/statistics as one array - see getWeightsBN. Without this //--- the plateau ladder would restore the weights around this layer while leaving its //--- own parameters at whatever the diverged era left behind. CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj; got = bn.getWeightsBN(w); break; } case defNeuronConvOCL: { CNeuronConvOCL *c = (CNeuronConvOCL*)obj; got = c.getWeightsConv(w); break; } case defNeuronLSTMOCL: { CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj; got = ls.getWeightsLSTM(w); break; } case defNeuron: case defNeuronConv: case defNeuronPool: case defNeuronLSTM: { CNeuronBase *neuron = (CNeuronBase*)obj; CArrayCon *conns = neuron.getConnections(); if(CheckPointer(conns) != POINTER_INVALID) { got = conns.Total(); ArrayResize(w, got); for(int i = 0; i < got; i++) { CConnection *con = conns.At(i); w[i] = (CheckPointer(con) != POINTER_INVALID) ? con.weight : 0.0; } } break; } default: break; } CArrayDouble *snap = new CArrayDouble(); if(CheckPointer(snap) == POINTER_INVALID) return false; if(got > 0) snap.AssignArray(w); if(!m_weightSnapshot.Add(snap)) { delete snap; return false; } } } m_haveWeightSnapshot = true; return true; } //+------------------------------------------------------------------+ //| Write the CaptureWeights() snapshot back into the live neurons IN | //| PLACE (setWeights - no neuron re-creation, no new device tensors). | //| Aborts (returning false, live weights untouched past that point) | //| only on a neuron-count mismatch, which never happens within a run | //| (fixed topology). See the header declaration for the full why. | //+------------------------------------------------------------------+ bool CNet::RestoreWeights(void) { if(!m_haveWeightSnapshot || CheckPointer(m_weightSnapshot) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID) return false; int idx = 0; for(int l = 0; l < layers.Total(); l++) { CLayer *layer = (CLayer*)layers.At(l); if(CheckPointer(layer) == POINTER_INVALID) return false; for(int n = 0; n < layer.Total(); n++) { if(idx >= m_weightSnapshot.Total()) return false; // topology/count mismatch - stop rather than mis-map weights CObject *obj = layer.At(n); CArrayDouble *snap = m_weightSnapshot.At(idx); idx++; if(CheckPointer(obj) == POINTER_INVALID || CheckPointer(snap) == POINTER_INVALID) continue; int cnt = snap.Total(); if(cnt <= 0) continue; // no weights captured for this neuron (e.g. output layer) - nothing to restore double w[]; ArrayResize(w, cnt); for(int i = 0; i < cnt; i++) w[i] = snap.At(i); switch(obj.Type()) { case defNeuronBaseOCL: { CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj; neuron.setWeights(w); break; } case defNeuronBatchNormOCL: { CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj; bn.setWeightsBN(w); break; } case defNeuronConvOCL: { CNeuronConvOCL *c = (CNeuronConvOCL*)obj; c.setWeightsConv(w); break; } case defNeuronLSTMOCL: { CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj; ls.setWeightsLSTM(w); break; } case defNeuron: case defNeuronConv: case defNeuronPool: case defNeuronLSTM: { CNeuronBase *neuron = (CNeuronBase*)obj; CArrayCon *conns = neuron.getConnections(); if(CheckPointer(conns) == POINTER_INVALID) break; int count = MathMin(cnt, conns.Total()); for(int i = 0; i < count; i++) { CConnection *con = conns.At(i); if(CheckPointer(con) != POINTER_INVALID) con.weight = w[i]; } break; } default: break; } } } return (idx == m_weightSnapshot.Total()); } //+------------------------------------------------------------------+ //| See this method's declaration comment for the cold-start bias | //| rationale. The weight block that produces the output layer's | //| values is stored on the layer BEFORE it (see CNeuronBaseOCL:: | //| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE | //| neuron, laid out as (sourceNeurons+1) values per destination | //| neuron, the last of which is that neuron's bias term), so this | //| reaches one layer back from the output layer to edit it. | //+------------------------------------------------------------------+ bool CNet::SeedOutputLayerBias(const double &biasValues[]) { int outputs = ArraySize(biasValues); if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2) return false; CLayer *sourceLayer = layers.At(layers.Total() - 2); if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0) return false; CObject *sourceObj = sourceLayer.At(0); //--- Batch norm is accepted here as well as a plain dense layer. It is a CNeuronBaseOCL subclass //--- with the identical (inputs+1)*outputs weight layout - it just happens to be the layer that //--- carries the head's weight matrix once normalization is enabled (see AI\NeuronBatchNorm.mqh and //--- BuildFreshTopology's batch-norm insertion). Without this the exact-type check silently failed //--- and the cold-start output-bias seed stopped being applied for every batch-norm topology. if(CheckPointer(sourceObj) == POINTER_INVALID || (sourceObj.Type() != defNeuronBaseOCL && sourceObj.Type() != defNeuronBatchNormOCL)) return false; CNeuronBaseOCL *sourceNeuron = (CNeuronBaseOCL*)sourceObj; if(CheckPointer(sourceNeuron) == POINTER_INVALID) return false; int inputs = sourceNeuron.Neurons(); double weights[]; int count = sourceNeuron.getWeights(weights); if(count != (inputs + 1) * outputs) return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess for(int i = 0; i < outputs; i++) weights[(inputs + 1) * i + inputs] = biasValues[i]; return sourceNeuron.setWeights(weights); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::SetBatchNormFrozen(bool frozen) { if(CheckPointer(layers) == POINTER_INVALID) return; for(int l = 0; l < layers.Total(); l++) { CLayer *layer = (CLayer*)layers.At(l); if(CheckPointer(layer) == POINTER_INVALID) continue; for(int n = 0; n < layer.Total(); n++) { CObject *obj = layer.At(n); if(CheckPointer(obj) == POINTER_INVALID || obj.Type() != defNeuronBatchNormOCL) continue; CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj; bn.SetStatsFrozen(frozen); } } } //+------------------------------------------------------------------+ //| See the declaration comment for why a loaded model's activation | //| cannot be trusted and must be re-asserted from the topology spec. | //+------------------------------------------------------------------+ bool CNet::EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous) { previous = intended; if(CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2) return false; CLayer *outputLayer = layers.At(layers.Total() - 1); if(CheckPointer(outputLayer) == POINTER_INVALID || outputLayer.Total() <= 0) return false; CObject *obj = outputLayer.At(0); if(CheckPointer(obj) == POINTER_INVALID) return false; //--- Both neuron models expose the same two accessors (CNeuronBase gained Activation() for exactly //--- this), so one branch per family is enough - and anything else (conv/pool/LSTM can never be an //--- output layer in this project's topologies) is left untouched rather than force-cast. int t = obj.Type(); if(t == defNeuronBaseOCL || t == defNeuronConvOCL || t == defNeuronPoolOCL || t == defNeuronLSTMOCL || t == defNeuronBatchNormOCL) { CNeuronBaseOCL *n = (CNeuronBaseOCL*)obj; previous = n.Activation(); if(previous == intended) return false; n.SetActivationFunction(intended); return true; } if(t == defNeuron || t == defNeuronConv || t == defNeuronPool || t == defNeuronLSTM) { //--- The scalar CPU model stores activation per NEURON, not per layer, so every neuron in the //--- output layer has to be corrected - not just index 0. bool repaired = false; for(int i = 0; i < outputLayer.Total(); i++) { CObject *cell = outputLayer.At(i); if(CheckPointer(cell) == POINTER_INVALID || cell.Type() != defNeuron) continue; CNeuronBase *n = (CNeuronBase*)cell; if(n.Activation() == intended) continue; if(!repaired) previous = n.Activation(); n.SetActivationFunction(intended); repaired = true; } return repaired; } return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Save(const int file_handle) { if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle)) return false; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Load(const int file_handle) { if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle)) return false; iWindow = FileReadInteger(file_handle, INT_VALUE); iStep = FileReadInteger(file_handle, INT_VALUE); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Save(const int file_handle) { if(!CNeuronPool::Save(file_handle)) return false; if(FileWriteDouble(file_handle, param) < 8) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Load(const int file_handle) { if(!CNeuronPool::Load(file_handle)) return false; param = FileReadDouble(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronPool { protected: CLayer *ForgetGate; CLayer *InputGate; CLayer *OutputGate; CLayer *NewContent; CArrayDouble *Memory; CArrayDouble *PrevMemory; CArrayDouble *Input; CArrayDouble *InputGradient; //--- virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual bool updateInputWeights(CLayer *prevLayer); virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data); virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type); virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence); public: CNeuronLSTM(void); ~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::CNeuronLSTM(void) { ForgetGate = new CLayer(); InputGate = new CLayer(); OutputGate = new CLayer(); NewContent = new CLayer(); Memory = new CArrayDouble(); PrevMemory = new CArrayDouble(); Input = new CArrayDouble(); InputGradient = new CArrayDouble(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::~CNeuronLSTM(void) { if(CheckPointer(ForgetGate) != POINTER_INVALID) delete ForgetGate; if(CheckPointer(InputGate) != POINTER_INVALID) delete InputGate; if(CheckPointer(OutputGate) != POINTER_INVALID) delete OutputGate; if(CheckPointer(NewContent) != POINTER_INVALID) delete NewContent; if(CheckPointer(Memory) != POINTER_INVALID) delete Memory; if(CheckPointer(PrevMemory) != POINTER_INVALID) delete PrevMemory; if(CheckPointer(Input) != POINTER_INVALID) delete Input; if(CheckPointer(InputGradient) != POINTER_INVALID) delete InputGradient; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { if(units_count <= 0) return false; //--- Init Layers if(!CNeuronPool::Init(numOutputs, myIndex, window, step, units_count, optimization_type)) return false; if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(InputGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(NewContent, units_count, window + units_count, optimization_type)) return false; if(!Memory.Reserve(units_count)) return false; if(!PrevMemory.Reserve(units_count)) return false; CNeuron *temp; for(int i = 0; i < units_count; i++) { if(!Memory.Add(0)) return false; if(!PrevMemory.Add(0)) return false; temp = OutputLayer.At(i); temp.setOutputVal(0); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(layer) == POINTER_INVALID) { layer = new CLayer(numOutputs); if(CheckPointer(layer) == POINTER_INVALID) return false; } else layer.Clear(); if(!layer.Reserve(numUnits)) return false; //--- //--- LeCun-uniform init, matching CNeuronLSTMOCL::Init's gate weighScale rationale - fan-in is //--- hidden units + input width (window+units_count, i.e. this InitLayer() call's own numOutputs //--- param, since each gate neuron's Connections array IS its fan-in weight vector here, not a //--- fan-out to a next layer). double gateScale = 1.0 / MathSqrt((double)numOutputs + 1.0); CNeuron *temp; for(int i = 0; i < numUnits; i++) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) return false; if(!temp.Init(numOutputs + 1, i, optimization_type, gateScale)) return false; if(!layer.Add(temp)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0) return false; CNeuronBase *temp; CConnection *temp_con; if(CheckPointer(Input) == POINTER_INVALID) { Input = new CArrayDouble(); if(CheckPointer(Input) == POINTER_INVALID) return false; } else Input.Clear(); //--- Concatenate input sequence int total = prevLayer.Total(); if(!Input.Reserve(total + OutputLayer.Total())) return false; for(int i = 0; i < total; i++) { temp = prevLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) return false; } total = OutputLayer.Total(); for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) return false; } int total_data = Input.Total(); //--- Calculated forget gate CArrayDouble *forget_gate = CalculateGate(ForgetGate, Input); if(CheckPointer(forget_gate) == POINTER_INVALID) return false; //--- Calculated input gate CArrayDouble *input_gate = CalculateGate(InputGate, Input); if(CheckPointer(input_gate) == POINTER_INVALID) return false; //--- Calculated output gate CArrayDouble *output_gate = CalculateGate(OutputGate, Input); if(CheckPointer(output_gate) == POINTER_INVALID) return false; //--- Calculated new content CArrayDouble *new_content = new CArrayDouble(); if(CheckPointer(new_content) == POINTER_INVALID) return false; total = NewContent.Total(); for(int i = 0; i < total; i++) { temp = NewContent.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; double val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) return false; val += temp_con.weight * Input.At(c); } val = TanhFunction(val); temp.setOutputVal(val); if(!new_content.Add(val)) return false; } //--- Calculated output sequences for(int i = 0; i < total; i++) { if(PrevMemory.Total() <= i) PrevMemory.Add(Memory.At(i)); else PrevMemory.Update(i, Memory.At(i)); double value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i); if(!Memory.Update(i, value)) return false; temp = OutputLayer.At(i); value = TanhFunction(value) * output_gate.At(i); temp.setOutputVal(value); } //--- delete forget_gate; delete input_gate; delete new_content; delete output_gate; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CArrayDouble *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayDouble *sequence) { CNeuronBase *temp; CConnection *temp_con; CArrayDouble *result = new CArrayDouble(); if(CheckPointer(gate) == POINTER_INVALID) return NULL; int total = gate.Total(); int total_data = sequence.Total(); for(int i = 0; i < total; i++) { temp = gate.At(i); if(CheckPointer(temp) == POINTER_INVALID) { delete result; return NULL; } double val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) { delete result; return NULL; } val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c)); } val = SigmoidFunction(val); temp.setOutputVal(val); if(!result.Add(val)) { delete result; return NULL; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(InputGradient) == POINTER_INVALID) { InputGradient = new CArrayDouble(); if(CheckPointer(InputGradient) == POINTER_INVALID) return false; } else InputGradient.Clear(); //--- int total = OutputLayer.Total(); CNeuron *temp; CArrayDouble *MemoryGradient = new CArrayDouble(); CNeuron *gate; CConnection *con; //--- if(nextLayer != OutputLayer) for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)); } //--- Calculated memory and output gate gradients if(CheckPointer(MemoryGradient) == POINTER_INVALID) return false; if(!MemoryGradient.Reserve(total)) return false; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; double value = temp.getGradient() * gate.getOutputVal(); value = TanhFunctionDerivative(Memory.At(i)) * value; if(i >= MemoryGradient.Total()) { if(!MemoryGradient.Add(value)) return false; } else { value = MemoryGradient.At(i) + value; if(!MemoryGradient.Update(i, value)) return false; } gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0); temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0); } //--- Calculated input gradients int total_inp = temp.getConnections().Total(); for(int n = 0; n < total_inp; n++) { double value = 0; for(int i = 0; i < total; i++) { temp = ForgetGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = InputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = OutputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = NewContent.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; } if(InputGradient.Total() >= n) { if(!InputGradient.Add(value)) return false; } else if(!InputGradient.Update(n, value)) return false; } //--- Calculated gradients for prev. state int shift = total_inp - total; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(InputGradient.At(shift + i)); } //--- Calculated memory and output gate gradients for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; double value = temp.getGradient() * gate.getPrevVal(); value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value; if(!MemoryGradient.Update(i, value)) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0)); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0)); } //--- delete MemoryGradient; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID) return false; //--- if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input) || !updateInputWeights(NewContent, Input)) { return false; } if(optimization == ADAM) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayDouble *input_data) { if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID) return false; CNeuronBase *neuron; CConnection *con; int total_n = gate.Total(); int total_data = input_data.Total(); double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int n = 0; n < total_n; n++) { neuron = gate.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return false; double g = neuron.getGradient(); double g2 = g * g; for(int i = 0; i < total_data; i++) { con = neuron.getConnections().At(i); if(CheckPointer(con) == POINTER_INVALID) return false; double data = input_data.At(i); if(optimization == SGD) con.weight += con.deltaWeight = (g != 0 && data != 0 ? eta * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight; else { con.mt = b1 * con.mt + (1 - b1) * g; con.vt = b2 * con.vt + (1 - b2) * g2 + 0.00000001; con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight)); // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why. con.weight += con.deltaWeight; } // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches // AI\Network.cl's LSTM_UpdateWeightsAdam MAX_WEIGHT clamp. con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight)); } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index) return false; //--- prevNeuron.setGradient(InputGradient.At(index)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) return false; //--- int total = prevLayer.Total(); if(total <= 0) return false; CNeuronBase *neuron; bool result = true; for(int i = 0; (i < total && result); i++) { neuron = prevLayer.At(i); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } result = calcInputGradients(neuron, i); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Save(const int file_handle) { if(!CNeuronPool::Save(file_handle)) return false; if(!ForgetGate.Save(file_handle)) return false; if(!InputGate.Save(file_handle)) return false; if(!OutputGate.Save(file_handle)) return false; if(!NewContent.Save(file_handle)) return false; if(!Memory.Save(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Load(const int file_handle) { if(!CNeuronPool::Load(file_handle)) return false; if(!ForgetGate.Load(file_handle)) return false; if(!InputGate.Load(file_handle)) return false; if(!OutputGate.Load(file_handle)) return false; if(!NewContent.Load(file_handle)) return false; if(!Memory.Load(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::activationFunction(double x) { switch(activation) { case NONE: return(x); break; case TANH: return TanhFunction(x); break; case SIGMOID: return SigmoidFunction(x); break; case PRELU: // fixed 0.01 slope, matches CNeuronConv::activationFunction's `param` default - // this case was previously missing here, so any generic Dense (defNeuron) layer // given PRELU silently fell through to the `return x` below (pure linear identity, // no nonlinearity at all) instead of leaky-ReLU. return(x >= 0 ? x : 0.01 * x); break; } //--- return x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::activationFunctionDerivative(double x) { switch(activation) { case NONE: return(1); break; case TANH: return TanhFunctionDerivative(x); break; case SIGMOID: return SigmoidFunctionDerivative(x); break; case PRELU: // See activationFunction() above - was previously missing here too, defaulting // to a derivative of 1 (which happens to be right for x>=0, but wrong for x<0, // where it must be the 0.01 leak slope, not 1). return(x >= 0 ? 1.0 : 0.01); break; } //--- return 1; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = -1; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return -1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL, CDirectMLMy *directml = NULL) : hWeights(-1), hDeltaWeights(-1), hOutput(-1), hGradient(-1) { iOutputs = outputs; iFileHandle = handle; OpenCL = opencl; DirectML = directml; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::Load(const int file_handle) { iFileHandle = file_handle; if(!CArrayObj::Load(file_handle)) return false; // A corrupt/truncated save can produce a 0-element layer here - CArrayObj::Load itself would // still return true for it, so m_data[0] must be guarded before indexing or MQL5 raises an // "array out of range" runtime error that aborts the whole CNet::Load call. if(Total() <= 0 || CheckPointer(m_data[0]) == POINTER_INVALID) { Print(__FUNCTION__ + ": layer loaded with no elements (corrupt or truncated save file)"); return false; } //--- switch(m_data[0].Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: case defNeuronBatchNormOCL: { CNeuronBaseOCL *temp = m_data[0]; iOutputs = temp.getConnections(); break; } default: { CNeuronBase *temp = m_data[0]; iOutputs = temp.getConnections().Total(); break; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| Install the per-class logit offsets - see the declaration. | //| Expects tau*log(prior_c) already computed by the caller, which is | //| the only place that knows the training set's class distribution. | //+------------------------------------------------------------------+ void CNet::SetLogitAdjustment(const double &offsets[]) { if(ArraySize(offsets) < 3) { //--- Refuse rather than half-apply: a partially filled offset vector would silently bias two //--- classes against a third, which is far worse than running unadjusted. bLogitAdjust = false; return; } for(int i = 0; i < 3; i++) { //--- Guard against a non-finite offset reaching the softmax (a zero prior would give -inf and //--- turn every gradient into NaN). A class with no examples at all simply gets no push. double v = offsets[i]; if(!MathIsValidNumber(v)) v = 0.0; dLogitAdjust[i] = v; } bLogitAdjust = true; } CNet::~CNet(void) { if(CheckPointer(layers) != POINTER_INVALID) delete layers; if(CheckPointer(m_weightSnapshot) != POINTER_INVALID) delete m_weightSnapshot; // CArrayObj (FreeMode) deletes its per-neuron CArrayDouble elements if(CheckPointer(opencl) != POINTER_INVALID) { opencl.Shutdown(); delete opencl; } if(CheckPointer(directml) != POINTER_INVALID) delete directml; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "BufferDouble.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: COpenCLMy *OpenCL; CDirectMLMy *DirectML; CBufferDouble *Output; CBufferDouble *PrevOutput; CBufferDouble *Weights; CBufferDouble *DeltaWeights; CBufferDouble *Gradient; CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; //--- const double alpha; int t; //--- int m_myIndex; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); //--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only //--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists - //--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU). bool BackendBufferCreate(CBufferDouble *buf) { if(CheckPointer(buf) == POINTER_INVALID) return false; if(CheckPointer(OpenCL) != POINTER_INVALID) return buf.BufferCreate(OpenCL); if(CheckPointer(DirectML) != POINTER_INVALID) return buf.BufferCreate(DirectML); return true; // no backend: keep host m_data, skip device allocation } public: CNeuronBaseOCL(void); ~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- virtual int getOutputIndex(void) { return Output.GetIndex(); } virtual int getPrevOutIndex(void) { return PrevOutput.GetIndex(); } virtual int getGradientIndex(void) { return Gradient.GetIndex(); } virtual int getWeightsIndex(void) { return Weights.GetIndex(); } virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); } virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); } virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();} //--- virtual int getOutputVal(double &values[]) { return Output.GetData(values); } virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); } virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); } virtual int getGradient(double &values[]) { return Gradient.GetData(values); } //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same //--- buffer to propagate into the hidden layers. virtual bool setGradient(const double &values[]) { int count = ArraySize(values); for(int i = 0; i < count; i++) if(!Gradient.Update(i, values[i])) return false; return Gradient.BufferWrite(); } // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created, // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here. virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); } // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment // (see that method's declaration comment) - writes a full replacement weight array back to this // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element, // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment // then pushes it to the device - the shape callers use when blending two already-read-out arrays. virtual bool setWeights(double &values[]) { if(CheckPointer(Weights) == POINTER_INVALID) return false; if(!Weights.AssignArray(values)) return false; return Weights.BufferWrite(); } virtual int Neurons(void) { return Output.Total(); } virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); } //--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to //--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/ //--- getOutputVal() which route through BufferRead() (a device read that fails without a backend). double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; } double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; } int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; } //--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only //--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense //--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math. virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); //--- Host-side input write for the CPU inference path's layer 0, and host-side output read for //--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates. bool SetInputsCPU(CArrayDouble *inputVals); int GetOutputsCPU(CArrayDouble *values); //--- virtual bool feedForward(CObject *SourceObject); virtual bool calcHiddenGradients(CObject *TargetObject); virtual bool calcOutputGradients(CArrayDouble *Target); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CObject *SourceObject); virtual void SetOptimization(ENUM_OPTIMIZATION value) { optimization = value; } virtual ENUM_OPTIMIZATION Optimization(void) const { return optimization; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBaseOCL; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum), activation(TANH), optimization(SGD), t(1) { OpenCL = NULL; DirectML = NULL; Output = new CBufferDouble(); PrevOutput = new CBufferDouble(); Weights = new CBufferDouble(); DeltaWeights = new CBufferDouble(); Gradient = new CBufferDouble(); FirstMomentum = new CBufferDouble(); SecondMomentum = new CBufferDouble(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::~CNeuronBaseOCL(void) { if(CheckPointer(Output) != POINTER_INVALID) delete Output; if(CheckPointer(PrevOutput) != POINTER_INVALID) delete PrevOutput; if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; if(CheckPointer(Gradient) != POINTER_INVALID) delete Gradient; if(CheckPointer(FirstMomentum) != POINTER_INVALID) delete FirstMomentum; if(CheckPointer(SecondMomentum) != POINTER_INVALID) delete SecondMomentum; OpenCL = NULL; DirectML = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) return false; OpenCL = open_cl; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(OpenCL)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting // for the fact that they zero out roughly half their input distribution, whereas the // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and // was ~2x too conservative here. Applied to every layer through this one shared Init() // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through - // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting // extremes on every backend, so the imprecision there is not worth the much larger, riskier // change of threading activation awareness through every neuron subtype's Init() overload. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(OpenCL)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(OpenCL)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(OpenCL)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same | //| buffer layout, buffers just get created on the DML backend. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0) return false; DirectML = direct_ml; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(DirectML)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(DirectML)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init - see the matching OpenCL Init() overload above for the full rationale. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(DirectML)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(DirectML)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(DirectML)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ #include "NeuronOCLConvPool.mqh" #include "NeuronBatchNorm.mqh" //+------------------------------------------------------------------+ //| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from | //| scratch from the standard LSTM equations - NOT ported from the | //| NeuroNet_DNG reference (see the note above the LSTM kernels in | //| Network.cl for why). Single-timestep-truncated BPTT: gradient | //| does not flow back into h_prev/c_prev from an earlier step. | //| Adam-only - Init fails for any other optimization type. | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: int m_iInputs; CBufferDouble *WeightsLSTM; CBufferDouble *FirstMomentumLSTM; CBufferDouble *SecondMomentumLSTM; CBufferDouble *DeltaWeightsLSTM; CBufferDouble *WeightsGradient; CBufferDouble *Concatenated; CBufferDouble *ConcatenatedGradient; CBufferDouble *Memory; CBufferDouble *HiddenCache; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool SetInputs(int count); public: CNeuronLSTMOCL(void) : m_iInputs(-1) { WeightsLSTM = NULL; FirstMomentumLSTM = NULL; SecondMomentumLSTM = NULL; DeltaWeightsLSTM = NULL; WeightsGradient = NULL; Concatenated = NULL; ConcatenatedGradient = NULL; Memory = NULL; HiddenCache = NULL; } ~CNeuronLSTMOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTMOCL; } // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment. virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); } virtual bool setWeightsLSTM(double &values[]) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; if(!WeightsLSTM.AssignArray(values)) return false; return WeightsLSTM.BufferWrite(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::~CNeuronLSTMOCL(void) { if(CheckPointer(WeightsLSTM) != POINTER_INVALID) delete WeightsLSTM; if(CheckPointer(FirstMomentumLSTM) != POINTER_INVALID) delete FirstMomentumLSTM; if(CheckPointer(SecondMomentumLSTM) != POINTER_INVALID) delete SecondMomentumLSTM; if(CheckPointer(DeltaWeightsLSTM) != POINTER_INVALID) delete DeltaWeightsLSTM; if(CheckPointer(WeightsGradient) != POINTER_INVALID) delete WeightsGradient; if(CheckPointer(Concatenated) != POINTER_INVALID) delete Concatenated; if(CheckPointer(ConcatenatedGradient) != POINTER_INVALID) delete ConcatenatedGradient; if(CheckPointer(Memory) != POINTER_INVALID) delete Memory; if(CheckPointer(HiddenCache) != POINTER_INVALID) delete HiddenCache; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type)) return false; uint H = numNeurons; if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(OpenCL)) return false; if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(OpenCL)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(OpenCL)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(OpenCL)) return false; m_iInputs = -1; return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above. | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type)) return false; uint H = numNeurons; if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(DirectML)) return false; if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(DirectML)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(DirectML)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(DirectML)) return false; m_iInputs = -1; return true; } //+------------------------------------------------------------------+ //| Lazily sized on the first feedForward call, once the previous | //| layer's neuron count (the input size) is known. | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::SetInputs(int count) { m_iInputs = count; uint H = (uint)Neurons(); int total = (int)(4 * H * (H + count + 1)); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) { WeightsLSTM = new CBufferDouble(); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; } if(!WeightsLSTM.Reserve(total)) return false; // Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full // rationale; fan-in here is hidden units + input width (each gate reads both). double weighScale = 1.0 / MathSqrt((double)(H + count) + 1.0); for(int i = 0; i < total; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!WeightsLSTM.Add(weigh)) return false; } if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) { FirstMomentumLSTM = new CBufferDouble(); if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) return false; } if(!FirstMomentumLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) { SecondMomentumLSTM = new CBufferDouble(); if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) return false; } if(!SecondMomentumLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML)) return false; //--- // Momentum accumulator - only meaningfully used when optimization==SGD // (see updateInputWeights()), but allocated unconditionally regardless of // the configured optimizer, matching CNeuronBaseOCL::Init's pattern for // the dense-layer DeltaWeights/FirstMomentum/SecondMomentum trio above. if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) { DeltaWeightsLSTM = new CBufferDouble(); if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) return false; } if(!DeltaWeightsLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(WeightsGradient) == POINTER_INVALID) { WeightsGradient = new CBufferDouble(); if(CheckPointer(WeightsGradient) == POINTER_INVALID) return false; } if(!WeightsGradient.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(m_iInputs <= 0) { if(!SetInputs(NeuronOCL.Neurons())) return false; } else if(m_iInputs != NeuronOCL.Neurons()) return false; int H = Neurons(); int I = m_iInputs; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.LSTMGates(WeightsLSTM.GetIndex(), getOutputIndex(), NeuronOCL.getOutputIndex(), Concatenated.GetIndex(), H, I) || !DirectML.LSTMState(Concatenated.GetIndex(), Memory.GetIndex(), getOutputIndex(), HiddenCache.GetIndex(), getOutputIndex(), H)) { printf("Error of execution DirectML LSTM feedForward"); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset2[2] = {0, 0}; uint size2[2] = {(uint)H, 4}; OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_hidden_prev, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_inputs, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_concatenated, Concatenated.GetIndex()); OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_input_size, I); if(!OpenCL.Execute(def_k_LSTM_Gates, 2, offset2, size2)) { printf("Error of execution kernel LSTM_Gates: %d", GetLastError()); return false; } uint offset1[1] = {0}; uint size1[1] = {(uint)H}; OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_concatenated, Concatenated.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_memory, Memory.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_prev, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_cache, HiddenCache.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_output, getOutputIndex()); OpenCL.SetArgument(def_k_LSTM_State, def_k_lstms_hidden_size, H); if(!OpenCL.Execute(def_k_LSTM_State, 1, offset1, size1)) { printf("Error of execution kernel LSTM_State: %d", GetLastError()); return false; } //--- Output (== hidden_prev for the next timestep) stays GPU-resident; see the note in //--- CNeuronBaseOCL::feedForward(). return true; } //+------------------------------------------------------------------+ //| Pure-MQL5 double-precision mirror of Network.cl's LSTM_Gates + | //| LSTM_State kernels, host buffers only (CPU inference). Recurrent | //| state is carried EXACTLY as the kernels do: hidden_prev is this | //| neuron's Output, the cell state is Memory[0..H); both persist | //| across bars. Weight layout: 4 gates (forget,input,output,cand), | //| each H*(H+I+1), per unit [H recurrent | I input | 1 bias]. | //| NOTE: like every backend this advances state per call; over a | //| long backtest fp rounding vs the training backend can drift, but | //| within the validated per-step tolerance (see ValidateCpuInference)| //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID || CheckPointer(WeightsLSTM) == POINTER_INVALID || CheckPointer(Memory) == POINTER_INVALID) return false; int H = Neurons(); int I = m_iInputs; if(I <= 0 || NeuronOCL.Neurons() != I || H <= 0) return false; int per_gate = H * (H + I + 1); if(WeightsLSTM.Total() < 4 * per_gate || Memory.Total() < 2 * H || Output.Total() < H) return false; //--- Gates: read hidden_prev (this Output) and inputs (prev Output) BEFORE Output is overwritten by //--- the state step below - exactly the two-kernel ordering (Concatenated is a separate buffer there). double gates[]; if(ArrayResize(gates, 4 * H) != 4 * H) return false; for(int gate = 0; gate < 4; gate++) for(int id = 0; id < H; id++) { int shift = gate * per_gate + id * (H + I + 1); double sum = 0.0; for(int k = 0; k < H; k++) sum += Output.At(k) * WeightsLSTM.At(shift + k); // hidden_prev for(int k = 0; k < I; k++) sum += NeuronOCL.OutputHost(k) * WeightsLSTM.At(shift + H + k); // inputs sum += WeightsLSTM.At(shift + H + I); // bias gates[gate * H + id] = (gate < 3) ? (1.0 / (1.0 + exp(-sum))) : tanh(sum); } //--- State: c_t = f*c_prev + i*g ; h = o*tanh(c_t). memory[H+id] keeps c_prev (unused in inference). for(int id = 0; id < H; id++) { double f = gates[id]; double ii = gates[H + id]; double o = gates[2 * H + id]; double g = gates[3 * H + id]; double c_prev = Memory.At(id); double c_t = f * c_prev + ii * g; if(!Memory.Update(H + id, c_prev) || !Memory.Update(id, c_t)) return false; if(!Output.Update(id, o * tanh(c_t))) return false; } return true; } //+------------------------------------------------------------------+ //| Writes the gradient into NeuronOCL (the earlier/input-side layer) | //| - same inverted-call convention as CNeuronConvOCL::calcInputGradients.| //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || m_iInputs <= 0) return false; int H = Neurons(); int I = m_iInputs; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.LSTMGateGradient(getGradientIndex(), Memory.GetIndex(), Concatenated.GetIndex(), ConcatenatedGradient.GetIndex(), H) || !DirectML.LSTMWeightsGradient(ConcatenatedGradient.GetIndex(), HiddenCache.GetIndex(), NeuronOCL.getOutputIndex(), WeightsGradient.GetIndex(), H, I) || !DirectML.LSTMInputsGradient(ConcatenatedGradient.GetIndex(), WeightsLSTM.GetIndex(), NeuronOCL.getGradientIndex(), H, I)) { printf("Error of execution DirectML LSTM calcInputGradients"); return false; } double temp[]; return NeuronOCL.getGradient(temp) > 0; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset1[1] = {0}; uint size1[1] = {(uint)H}; OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_gradient, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_memory, Memory.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated, Concatenated.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgument(def_k_LSTM_GateGradient, def_k_lstmgg_hidden_size, H); if(!OpenCL.Execute(def_k_LSTM_GateGradient, 1, offset1, size1)) { printf("Error of execution kernel LSTM_GateGradient: %d", GetLastError()); return false; } uint sizeW[1] = {(uint)WeightsLSTM.Total()}; OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_cache, HiddenCache.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_inputs, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_input_size, I); if(!OpenCL.Execute(def_k_LSTM_WeightsGradient, 1, offset1, sizeW)) { printf("Error of execution kernel LSTM_WeightsGradient: %d", GetLastError()); return false; } uint sizeI[1] = {(uint)I}; OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_inputs_gradient, NeuronOCL.getGradientIndex()); OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_input_size, I); if(!OpenCL.Execute(def_k_LSTM_InputsGradient, 1, offset1, sizeI)) { printf("Error of execution kernel LSTM_InputsGradient: %d", GetLastError()); return false; } //--- NeuronOCL's Gradient stays GPU-resident; see the note in //--- CNeuronConvOCL::calcInputGradients(). return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; int total = WeightsLSTM.Total(); if(CheckPointer(DirectML) != POINTER_INVALID) { if(optimization == SGD) { if(!DirectML.LSTMUpdateWeightsMomentum(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), DeltaWeightsLSTM.GetIndex(), eta, alpha, total, 0)) { printf("Error of execution DirectML LSTM_UpdateWeightsMomentum"); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.LSTMUpdateWeightsAdam(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), FirstMomentumLSTM.GetIndex(), SecondMomentumLSTM.GetIndex(), lt, b1, b2, total)) { printf("Error of execution DirectML LSTM_UpdateWeightsAdam"); return false; } t++; } //--- WeightsLSTM stays DLL-resident; feedForward reads it via GetIndex() (same as the OpenCL //--- branch below). Save()/BlendWeightsFrom() BufferRead() on demand. return true; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset1[1] = {0}; uint size1[1] = {(uint)total}; if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_dw, DeltaWeightsLSTM.GetIndex()); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_momentum, (float)alpha); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_optimizer, 0); ResetLastError(); if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsMomentum, 1, offset1, size1)) { printf("Error of execution kernel LSTM_UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_m, FirstMomentumLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_v, SecondMomentumLSTM.GetIndex()); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_l, (float)lt); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b1, (float)b1); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b2, (float)b2); ResetLastError(); if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsAdam, 1, offset1, size1)) { printf("Error of execution kernel LSTM_UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- WeightsLSTM stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights(). return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) return false; if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < INT_VALUE) return false; if(m_iInputs <= 0) return true; if(CheckPointer(WeightsLSTM) == POINTER_INVALID || !WeightsLSTM.BufferRead() || !WeightsLSTM.Save(file_handle)) return false; if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID || !FirstMomentumLSTM.BufferRead() || !FirstMomentumLSTM.Save(file_handle)) return false; if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID || !SecondMomentumLSTM.BufferRead() || !SecondMomentumLSTM.Save(file_handle)) return false; if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID || !DeltaWeightsLSTM.BufferRead() || !DeltaWeightsLSTM.Save(file_handle)) return false; if(CheckPointer(Memory) == POINTER_INVALID || !Memory.BufferRead() || !Memory.Save(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) return false; m_iInputs = FileReadInteger(file_handle, INT_VALUE); uint H = (uint)Neurons(); //--- scratch buffers (not persisted) were sized for the placeholder Init() unit //--- count; resize them now that the real neuron count is known. if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } Concatenated.BufferFree(); if(!Concatenated.BufferInit(4 * H, 0)) return false; if(!BackendBufferCreate(Concatenated)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } ConcatenatedGradient.BufferFree(); if(!ConcatenatedGradient.BufferInit(4 * H, 0)) return false; if(!BackendBufferCreate(ConcatenatedGradient)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } HiddenCache.BufferFree(); if(!HiddenCache.BufferInit(H, 0)) return false; if(!BackendBufferCreate(HiddenCache)) return false; //--- if(m_iInputs <= 0) return true; int total = (int)(4 * H * (H + m_iInputs + 1)); //--- if(CheckPointer(WeightsLSTM) == POINTER_INVALID) { WeightsLSTM = new CBufferDouble(); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; } if(WeightsLSTM.GetIndex() >= 0) WeightsLSTM.BufferFree(); if(!WeightsLSTM.Load(file_handle)) return false; if(!BackendBufferCreate(WeightsLSTM)) return false; //--- if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) { FirstMomentumLSTM = new CBufferDouble(); if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) return false; } if(FirstMomentumLSTM.GetIndex() >= 0) FirstMomentumLSTM.BufferFree(); if(!FirstMomentumLSTM.Load(file_handle)) return false; if(!BackendBufferCreate(FirstMomentumLSTM)) return false; //--- if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) { SecondMomentumLSTM = new CBufferDouble(); if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) return false; } if(SecondMomentumLSTM.GetIndex() >= 0) SecondMomentumLSTM.BufferFree(); if(!SecondMomentumLSTM.Load(file_handle)) return false; if(!BackendBufferCreate(SecondMomentumLSTM)) return false; //--- if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) { DeltaWeightsLSTM = new CBufferDouble(); if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) return false; } if(DeltaWeightsLSTM.GetIndex() >= 0) DeltaWeightsLSTM.BufferFree(); if(!DeltaWeightsLSTM.Load(file_handle)) return false; if(!BackendBufferCreate(DeltaWeightsLSTM)) return false; //--- if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(Memory.GetIndex() >= 0) Memory.BufferFree(); if(!Memory.Load(file_handle)) return false; if(!BackendBufferCreate(Memory)) return false; //--- if(CheckPointer(WeightsGradient) == POINTER_INVALID) { WeightsGradient = new CBufferDouble(); if(CheckPointer(WeightsGradient) == POINTER_INVALID) return false; } if(!WeightsGradient.BufferInit(total, 0)) return false; if(!BackendBufferCreate(WeightsGradient)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return feedForward(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation))) { printf("Error of execution DirectML FeedForward"); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel FeedForward: %d", GetLastError()); return false; } //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex() //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData(). //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead. return true; } //+------------------------------------------------------------------+ //| Pure-MQL5 dense forward pass - exact double-precision mirror of | //| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, | //| which (Gizlyk convention) owns both the inputs (its Output) and | //| the weights connecting them to THIS layer (its Weights), laid out | //| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host | //| buffers only; used when no compute backend exists (CPU inference).| //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int inputs = NeuronOCL.Neurons(); int outCount = Output.Total(); int wTotal = NeuronOCL.WeightsCount(); if(wTotal < (inputs + 1) * outCount) return false; // weight buffer smaller than the dense layout requires - refuse rather than misread for(int i = 0; i < outCount; i++) { int shift = (inputs + 1) * i; double sum = 0.0; for(int k = 0; k < inputs; k++) sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k); sum += NeuronOCL.WeightHost(shift + inputs); // bias switch(activation) { case TANH: sum = tanh(sum); break; case SIGMOID: sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum)))); break; case PRELU: if(sum < 0.0) sum *= 0.01; break; // NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case. } if(!Output.Update(i, sum)) return false; } return true; } //+------------------------------------------------------------------+ //| Host-side input write for the CPU inference path's layer 0 (there | //| is no device buffer to write into, unlike CNet::feedForward's | //| BufferWrite branch). Copies inputVals into the Output host array. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals) { if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int total = MathMin(Output.Total(), inputVals.Total()); for(int i = 0; i < total; i++) if(!Output.Update(i, inputVals.At(i))) return false; return true; } //+------------------------------------------------------------------+ //| Host-side output read for getResults() on the CPU inference path | //| (no device BufferRead available). Returns the count copied. | //+------------------------------------------------------------------+ int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values) { if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return 0; values.Clear(); int n = Output.Total(); for(int i = 0; i < n; i++) if(!values.Add(Output.At(i))) return i; return n; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1)) { printf("Error of execution DirectML CalcHiddenGradient"); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons() + 1; OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError()); return false; } //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target) { if(CheckPointer(Target) == POINTER_INVALID) return false; int count = Target.Total(); for(int i = 0; i < count; i++) if(!Gradient.Update(i, Target.At(i))) return false; Gradient.BufferWrite(); //--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count)) { printf("Error of execution DirectML CalcOutputGradient"); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation)); ResetLastError(); if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclOutputGradient: %d", GetLastError()); return false; } //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which //--- BufferRead()s itself - see the note in feedForward() above. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { int inputs = NeuronOCL.Neurons(); int neurons = Neurons(); if(optimization == SGD) { if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons, 0)) { printf("Error of execution DirectML UpdateWeightsMomentum"); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(), inputs, lt, b1, b2, neurons)) { printf("Error of execution DirectML UpdateWeightsAdam"); return false; } t++; } //--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex() //--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand. return true; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons(); if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex())) return false; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons())) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2)) return false; uint rest = global_work_size[1] % 4; global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save() //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject) { if(CheckPointer(TargetObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; CNeuronConvOCL *tempConv = NULL; CNeuronLSTMOCL *tempLstm = NULL; CNeuronPoolOCL *tempPool = NULL; CNeuronBatchNormOCL *tempBN = NULL; switch(TargetObject.Type()) { case defNeuronBaseOCL: temp = TargetObject; return calcHiddenGradients(temp); break; case defNeuronBatchNormOCL: //--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step //--- and writes into this->Gradient. Routing it through the dense branch instead would run //--- CaclHiddenGradient against a weight matrix batch norm does not have. tempBN = TargetObject; return tempBN.calcInputGradients(GetPointer(this)); break; case defNeuronConvOCL: //--- Conv owns the backward step (calcInputGradients), called on itself with //--- "this" (the earlier layer) passed in so it writes into this->Gradient. tempConv = TargetObject; return tempConv.calcInputGradients(GetPointer(this)); break; case defNeuronLSTMOCL: tempLstm = TargetObject; return tempLstm.calcInputGradients(GetPointer(this)); break; case defNeuronPoolOCL: tempPool = TargetObject; return tempPool.calcInputGradients(GetPointer(this)); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return updateInputWeights(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE) return false; //--- if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle)) return false; if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle)) return false; if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle)) return false; //--- if(CheckPointer(Weights) == POINTER_INVALID) { FileWriteInteger(file_handle, 0); return true; } else FileWriteInteger(file_handle, 1); //--- if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle)) return false; if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle)) return false; if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; //--- activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(Output.GetIndex() >= 0) Output.BufferFree(); if(!Output.Load(file_handle)) return false; if(!BackendBufferCreate(Output)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(PrevOutput.GetIndex() >= 0) PrevOutput.BufferFree(); if(!PrevOutput.Load(file_handle)) return false; if(!BackendBufferCreate(PrevOutput)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(Gradient.GetIndex() >= 0) Gradient.BufferFree(); if(!Gradient.Load(file_handle)) return false; if(!BackendBufferCreate(Gradient)) return false; //--- if(FileReadInteger(file_handle) == 0) return true; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } if(Weights.GetIndex() >= 0) Weights.BufferFree(); if(!Weights.Load(file_handle)) return false; if(!BackendBufferCreate(Weights)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(DeltaWeights.GetIndex() >= 0) DeltaWeights.BufferFree(); if(!DeltaWeights.Load(file_handle)) return false; if(!BackendBufferCreate(DeltaWeights)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(FirstMomentum.GetIndex() >= 0) FirstMomentum.BufferFree(); if(!FirstMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(FirstMomentum)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(SecondMomentum.GetIndex() >= 0) SecondMomentum.BufferFree(); if(!SecondMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(SecondMomentum)) return false; } //--- return true; } //+------------------------------------------------------------------+