//+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include #include #include #include //--- kept local to this file (rather than Enumerations\InputEnums.mqh) since AI\Network.mqh is //--- included well before Variables\Inputs.mqh in Warrior_EA.mq5's include chain - this input has //--- to be self-contained here regardless. enum CPU_LOAD_PRESET { CPU_LOAD_10 = 10, // 10% of detected cores CPU_LOAD_20 = 20, // 20% of detected cores CPU_LOAD_30 = 30, // 30% of detected cores CPU_LOAD_40 = 40, // 40% of detected cores CPU_LOAD_50 = 50, // 50% of detected cores CPU_LOAD_60 = 60, // 60% of detected cores CPU_LOAD_70 = 70, // 70% of detected cores CPU_LOAD_80 = 80, // 80% of detected cores CPU_LOAD_90 = 90, // 90% of detected cores CPU_LOAD_MAX = 100 // Max - all detected cores }; //--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available, //--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Caps how //--- many of the auto-detected CPU cores WarriorCPU.dll's worker thread pool actually uses - has //--- no effect at all when a GPU tier (OpenCL or DirectML) is active, since neither one calls into //--- WarriorCPU.dll. //--- Applied directly, undivided, to every CNet's own WarriorCPU.dll worker pool (live Net, shadow //--- Net, and - under AIType=HYBRID - each of PAI/CONV/LSTM's own pair) even though several such //--- pools can be alive at once. An earlier version divided this by how many CNet instances were //--- alive (g_netPeerCount/EffectiveCpuLoadPercent()), reasoning that N pools each sized at 100% //--- would N-way oversubscribe the CPU - but MQL5 gives one chart's EA a single execution thread, //--- and every WarriorCPU.dll entry point (CPU_FeedForward/CPU_CalcOutputGradient/etc.) blocks that //--- thread until its own ParallelFor() completes, so only ONE pool is EVER actively computing at a //--- time - the rest sit idle (blocked on a condvar, ~0% CPU) regardless of how many exist. Dividing //--- was pure waste: it capped whichever pool happened to be running at a fraction of the cores the //--- user asked for, without preventing any contention that was never actually possible. input CPU_LOAD_PRESET TargetCPULoad = CPU_LOAD_MAX; // CPU worker thread cap when no GPU accel (% cores) //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //--- kept local to this file for the same include-order reason as CPU_LOAD_PRESET/TargetCPULoad //--- above - AI\Network.mqh is parsed well before Variables\Inputs.mqh. //--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference //--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's //--- abstract "0.001" mention, which the book's own worked examples don't actually use either. //--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005) //--- this project had separately tuned lr to before this input existed, so no behavior conflict. //--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak //--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to //--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this //--- reverts to the literature/book default. input double AdamLearningRate = 0.0003; // Adam learning rate (eta ceiling) - book default 3.0e-4 input double AdamBeta1 = 0.9; // Adam beta1 (1st-moment decay) - book default 0.9 input double AdamBeta2 = 0.999; // Adam beta2 (2nd-moment decay) - book default 0.999 //--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method //--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses //--- Adam's book-default rate as its starting point too. The book also states no numeric default for //--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses //--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for. input double SgdLearningRate = 0.0003; // SGD+momentum learning rate - book default 3.0e-4 input double SgdMomentum = 0.9; // SGD+momentum decay - reuses Adam's beta1 default #define lr AdamLearningRate #define b1 AdamBeta1 #define b2 AdamBeta2 #define momentum SgdMomentum double eta = lr; #define defConnect 0x7781 #define defArrayConnects 0x7782 #define defNeuronBase 0x7783 #define defNeuron 0x7784 #define defNeuronConv 0x7785 #define defNeuronPool 0x7786 #define defLayer 0x7787 #define defArrayLayer 0x7788 #define defNet 0x7789 #define defNeuronLSTM 0x7791 //--- #define defBufferDouble 0x7882 #define defNeuronBaseOCL 0x7883 #define defNeuronLSTMOCL 0x7884 #define defNeuronConvOCL 0x7885 #define defNeuronPoolOCL 0x7886 //--- #define def_k_FeedForward 0 #define def_k_ff_matrix_w 0 #define def_k_ff_matrix_i 1 #define def_k_ff_matrix_o 2 #define def_k_ff_inputs 3 #define def_k_ff_activation 4 //--- #define def_k_CaclOutputGradient 1 #define def_k_cog_matrix_t 0 #define def_k_cog_matrix_o 1 #define def_k_cog_matrix_ig 2 #define def_k_cog_activation 3 //--- #define def_k_CaclHiddenGradient 2 #define def_k_chg_matrix_w 0 #define def_k_chg_matrix_g 1 #define def_k_chg_matrix_o 2 #define def_k_chg_matrix_ig 3 #define def_k_chg_outputs 4 #define def_k_chg_activation 5 //--- #define def_k_UpdateWeightsMomentum 3 #define def_k_uwm_matrix_w 0 #define def_k_uwm_matrix_g 1 #define def_k_uwm_matrix_i 2 #define def_k_uwm_matrix_dw 3 #define def_k_uwm_inputs 4 #define def_k_uwm_learning_rates 5 #define def_k_uwm_momentum 6 //--- #define def_k_UpdateWeightsAdam 4 #define def_k_uwa_matrix_w 0 #define def_k_uwa_matrix_g 1 #define def_k_uwa_matrix_i 2 #define def_k_uwa_matrix_m 3 #define def_k_uwa_matrix_v 4 #define def_k_uwa_inputs 5 #define def_k_uwa_l 6 #define def_k_uwa_b1 7 #define def_k_uwa_b2 8 //--- #define def_k_FeedForwardProof 15 #define def_k_ffp_matrix_i 0 #define def_k_ffp_matrix_o 1 #define def_k_ffp_inputs 2 #define def_k_ffp_window 3 #define def_k_ffp_step 4 //--- #define def_k_CalcInputGradientProof 16 #define def_k_cigp_matrix_i 0 #define def_k_cigp_matrix_g 1 #define def_k_cigp_matrix_o 2 #define def_k_cigp_matrix_ig 3 #define def_k_cigp_outputs 4 #define def_k_cigp_window 5 #define def_k_cigp_step 6 //--- #define def_k_FeedForwardConv 5 #define def_k_ffc_matrix_w 0 #define def_k_ffc_matrix_i 1 #define def_k_ffc_matrix_o 2 #define def_k_ffc_inputs 3 #define def_k_ffc_step 4 #define def_k_ffc_window_in 5 #define def_k_ffc_window_out 6 #define def_k_ffc_activation 7 //--- #define def_k_CalcHiddenGradientConv 6 #define def_k_chgc_matrix_w 0 #define def_k_chgc_matrix_g 1 #define def_k_chgc_matrix_o 2 #define def_k_chgc_matrix_ig 3 #define def_k_chgc_outputs 4 #define def_k_chgc_step 5 #define def_k_chgc_window_in 6 #define def_k_chgc_window_out 7 #define def_k_chgc_activation 8 //--- #define def_k_UpdateWeightsConvMomentum 7 #define def_k_uwcm_matrix_w 0 #define def_k_uwcm_matrix_g 1 #define def_k_uwcm_matrix_i 2 #define def_k_uwcm_matrix_dw 3 #define def_k_uwcm_inputs 4 #define def_k_uwcm_learning_rates 5 #define def_k_uwcm_momentum 6 #define def_k_uwcm_window_in 7 #define def_k_uwcm_window_out 8 #define def_k_uwcm_step 9 //--- #define def_k_UpdateWeightsConvAdam 8 #define def_k_uwca_matrix_w 0 #define def_k_uwca_matrix_g 1 #define def_k_uwca_matrix_i 2 #define def_k_uwca_matrix_m 3 #define def_k_uwca_matrix_v 4 #define def_k_uwca_inputs 5 #define def_k_uwca_l 6 #define def_k_uwca_b1 7 #define def_k_uwca_b2 8 #define def_k_uwca_window_in 9 #define def_k_uwca_window_out 10 #define def_k_uwca_step 11 //--- // LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient // flows back into h_prev/c_prev from a prior step). Supports both Adam and // SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) - // see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch. // Derived from scratch from the standard LSTM equations - NOT ported from // the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites // the live weights buffer instead of writing to weights_gradient. #define def_k_LSTM_Gates 9 #define def_k_lstmg_matrix_w 0 #define def_k_lstmg_hidden_prev 1 #define def_k_lstmg_inputs 2 #define def_k_lstmg_concatenated 3 #define def_k_lstmg_hidden_size 4 #define def_k_lstmg_input_size 5 //--- #define def_k_LSTM_State 10 #define def_k_lstms_concatenated 0 #define def_k_lstms_memory 1 #define def_k_lstms_hidden_prev 2 #define def_k_lstms_hidden_cache 3 #define def_k_lstms_output 4 #define def_k_lstms_hidden_size 5 //--- #define def_k_LSTM_GateGradient 11 #define def_k_lstmgg_gradient 0 #define def_k_lstmgg_memory 1 #define def_k_lstmgg_concatenated 2 #define def_k_lstmgg_concatenated_gradient 3 #define def_k_lstmgg_hidden_size 4 //--- #define def_k_LSTM_WeightsGradient 12 #define def_k_lstmwg_concatenated_gradient 0 #define def_k_lstmwg_hidden_cache 1 #define def_k_lstmwg_inputs 2 #define def_k_lstmwg_weights_gradient 3 #define def_k_lstmwg_hidden_size 4 #define def_k_lstmwg_input_size 5 //--- #define def_k_LSTM_InputsGradient 13 #define def_k_lstmig_concatenated_gradient 0 #define def_k_lstmig_matrix_w 1 #define def_k_lstmig_inputs_gradient 2 #define def_k_lstmig_hidden_size 3 #define def_k_lstmig_input_size 4 //--- #define def_k_LSTM_UpdateWeightsAdam 14 #define def_k_lstmuwa_matrix_w 0 #define def_k_lstmuwa_weights_gradient 1 #define def_k_lstmuwa_matrix_m 2 #define def_k_lstmuwa_matrix_v 3 #define def_k_lstmuwa_l 4 #define def_k_lstmuwa_b1 5 #define def_k_lstmuwa_b2 6 //--- // SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see // AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body. #define def_k_LSTM_UpdateWeightsMomentum 17 #define def_k_lstmuwm_matrix_w 0 #define def_k_lstmuwm_weights_gradient 1 #define def_k_lstmuwm_matrix_dw 2 #define def_k_lstmuwm_learning_rates 3 #define def_k_lstmuwm_momentum 4 //--- // b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see // AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era // same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root // cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE // gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever // could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled // kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam // signatures), so they're safe to expose as ordinary inputs. // Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had // already gone unstable (real collapses were happening at weight magnitudes several orders of // magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh // engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually // reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below. #define MAX_WEIGHT 100.0 // Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in // DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see // WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright // +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of // training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi- // hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. // 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates, // ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's // sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a // weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the // discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the // calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the // spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt // dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay // strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins // argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth. #define WEIGHT_DECAY 0.001 // Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full // rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden // Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras), // most likely from 5x back-to-back oversampling replay building artificially correlated momentum. // Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the // post-update weight value and is far too loose (1e6) to prevent this. #define MAX_WEIGHT_DELTA 0.1 // Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see // SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its // activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that // neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated // unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in // Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine. #define MIN_ACTIVATION_DERIVATIVE 1.0e-4 // Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND // read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a // different distribution than it was trained on). The classification outputs are SIGMOID-bounded to // [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 - // the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training // can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three // outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the // ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal // modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays // (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the // loss, keeping gradient magnitudes (and thus eta tuning) unchanged. #define CLASS_LOGIT_SCALE 6.0 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "Network.cl" as string cl_program //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { NONE, TANH, SIGMOID, PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction }; //+------------------------------------------------------------------+ //| Translates ENUM_ACTIVATION to the "activation" int code every | //| native compute backend actually understands (Network.cl's | //| kernels, and the mirrored Activation()/inline switches in | //| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and | //| deliberately anything else (incl. NONE) falls through every one | //| of those switches unmatched, which is exactly linear passthrough -| //| there's no case 3 anywhere on the native side, so PRELU relies on | //| the FeedForwardConv-family kernels specifically, and NONE never | //| needs a case at all. This is NOT the same numbering as | //| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a | //| raw (int)activation cast at a kernel call site silently sends the | //| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native | //| SIGMOID). Only use this at actual kernel-dispatch call sites - | //| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw | //| ENUM_ACTIVATION value instead, and must keep using (int)activation | //| directly so saved topology files round-trip through Load() as-is. | //+------------------------------------------------------------------+ int NativeActivationCode(ENUM_ACTIVATION value) { switch(value) { case TANH: return 0; case SIGMOID: return 1; case PRELU: return 2; default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches } } //--- enum ENUM_OPTIMIZATION { SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras) ADAM // Adam (adaptive step, faster convergence, can overfit) }; //--- enum ENUM_BUFFERS { WEIGHTS, DELTA_WEIGHTS, OUTPUT, GRADIENT, FIRST_MOMENTUM, SECOND_MOMENTUM }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronPrimitives.mqh" class CLayer; //--- class CNeuronBase : public CObject { protected: double outputVal; double prevVal; uint m_myIndex; double gradient; CArrayCon *Connections; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; int t; //--- virtual bool feedForward(CLayer *prevLayer) { return false; } virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; } virtual bool updateInputWeights(CLayer *&prevLayer) { return false; } virtual double activationFunction(double x); virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); } virtual double TanhFunction(double x) { return tanh(x); } virtual CLayer *getOutputLayer(void) { return NULL; } public: CNeuronBase(void); ~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- static double alpha; //--- virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; } virtual double getOutputVal() { return outputVal; } virtual double getPrevVal() { return prevVal; } virtual void setGradient(double val) { gradient = val; } virtual double getGradient() { return gradient; } virtual CArrayCon *getConnections() { return Connections;} virtual double activationFunctionDerivative(double x); virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); } virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); } //--- virtual bool feedForward(CObject *&SourceObject); virtual bool calcHiddenGradients(CObject *&TargetObject); virtual bool updateInputWeights(CObject *&SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle) { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- virtual int Type(void) const { return defNeuronBase; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::alpha = momentum; // momentum //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::CNeuronBase(void) : outputVal(1), gradient(0), activation(TANH), t(1), optimization(SGD) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::~CNeuronBase(void) { if(CheckPointer(Connections) != POINTER_INVALID) delete Connections; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(Connections) == POINTER_INVALID) { Connections = new CArrayCon(); if(CheckPointer(Connections) == POINTER_INVALID) return false; } //--- if(Connections.Reserve(fmax(numOutputs, 1))) for(uint c = 0; c < numOutputs; c++) { if(!Connections.CreateElement(c)) return false; Connections.IncreaseTotal(); } //--- m_myIndex = myIndex; optimization = optimization_type; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "NeuronCPU.mqh" class COpenCLMy : public COpenCL { public: COpenCLMy(void) {}; ~COpenCLMy(void) {}; template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); }; #include "NeuronDirectML.mqh" class CLayer: public CArrayObj { private: uint iOutputs; int iFileHandle; int hWeights; int hDeltaWeights; int hOutput; int hGradient; COpenCLMy *OpenCL; CDirectMLMy *DirectML; public: CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL); ~CLayer(void) {}; //--- virtual bool CreateElement(int const index); virtual void IncreaseTotal() { m_data_total++; } virtual int Type(void) const { return defLayer; } virtual bool Load(const int file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::CreateElement(int index) { if(index >= m_data_max) return false; //--- bool result = false; CNeuronBase *temp = NULL; CNeuronPool *temp_p = NULL; CNeuronBaseOCL *temp_ocl = NULL; if(iFileHandle <= 0) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD)) return false; result = true; } else { int type = FileReadInteger(iFileHandle); switch(type) { case defNeuron: temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) result = false; result = temp.Init(iOutputs, index, ADAM); break; case defNeuronPool: temp_p = new CNeuronPool(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronConv: temp_p = new CNeuronConv(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronLSTM: temp_p = new CNeuronLSTM(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } break; case defNeuronBaseOCL: if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) return false; temp_ocl = new CNeuronBaseOCL(); if(CheckPointer(temp_ocl) == POINTER_INVALID) result = false; if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM) : temp_ocl.Init(iOutputs, index, DirectML, 1, ADAM)) { m_data[index] = temp_ocl; return true; } break; case defNeuronConvOCL: { if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) return false; //--- placeholder dims; the real window/step/units are restored by Load() right after CNeuronConvOCL *temp_conv = new CNeuronConvOCL(); if(CheckPointer(temp_conv) == POINTER_INVALID) result = false; if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM) : temp_conv.Init(iOutputs, index, DirectML, 1, 1, 1, 1, ADAM)) { m_data[index] = temp_conv; return true; } break; } case defNeuronPoolOCL: { if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) return false; CNeuronPoolOCL *temp_pool = new CNeuronPoolOCL(); if(CheckPointer(temp_pool) == POINTER_INVALID) result = false; if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_pool.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM) : temp_pool.Init(iOutputs, index, DirectML, 1, 1, 1, ADAM)) { m_data[index] = temp_pool; return true; } break; } case defNeuronLSTMOCL: { if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID) return false; CNeuronLSTMOCL *temp_lstm = new CNeuronLSTMOCL(); if(CheckPointer(temp_lstm) == POINTER_INVALID) result = false; if(CheckPointer(OpenCL) != POINTER_INVALID ? temp_lstm.Init(iOutputs, index, OpenCL, 1, ADAM) : temp_lstm.Init(iOutputs, index, DirectML, 1, ADAM)) { m_data[index] = temp_lstm; return true; } break; } default: result = false; break; } } if(result) m_data[index] = temp; //--- return (result); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "ArrayLayer.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPool : public CNeuronBase { protected: CLayer *OutputLayer; int iWindow; int iStep; virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); public: CNeuronPool(void) {}; ~CNeuronPool(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronPool; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronPool { protected: double param; //PReLU param virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual double activationFunction(double x); virtual bool updateInputWeights(CLayer *&prevLayer); public: CNeuronConv() : param(0.01) { }; ~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; virtual double activationFunctionDerivative(double x); virtual int Type(void) const { return defNeuronConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::feedForward(CObject *&SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = feedForward(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: temp_n = SourceObject; result = feedForward(temp_n.getOutputLayer()); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::updateInputWeights(CObject *&SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = updateInputWeights(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: temp_n = SourceObject; temp_l = temp_n.getOutputLayer(); result = updateInputWeights(temp_l); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::feedForward(CLayer *prevLayer) { bool result = false; //--- if(CheckPointer(prevLayer) == POINTER_INVALID) return result; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; CConnection *con; result = true; for(int i = 0; (i < total && result); i += iStep) { double sum = 0; for(int j = 0; (j < iWindow && result); j++) { temp = prevLayer.At(i + j); con = Connections.At(j); if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) return false; double val = temp.getOutputVal(); sum += val * con.weight; } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setOutputVal(activationFunction(sum)); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronConv::activationFunction(double x) { if(x >= 0) return x; return param * x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject) { bool result = false; //--- if(CheckPointer(TargetObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronPool *temp_n; switch(TargetObject.Type()) { case defLayer: temp_l = TargetObject; result = calcHiddenGradients(temp_l); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: switch(Type()) { case defNeuron: temp_n = TargetObject; result = temp_n.calcInputGradients(GetPointer(this), m_myIndex); break; case defNeuronLSTM: temp_n = TargetObject; temp_l = getOutputLayer(); if(!temp_n.calcInputGradients(temp_l)) { result = false; break; } result = calcHiddenGradients(temp_l); break; default: temp_l =getOutputLayer(); temp_n = TargetObject; result = temp_n.calcInputGradients(temp_l); break; } break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) return false; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal())); } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronConv::activationFunctionDerivative(double x) { if(x >= 0) return 1; return param; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- CConnection *con; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int n = 0; n < iWindow && !IsStopped(); n++) { con = Connections.At(n); if(CheckPointer(con) == POINTER_INVALID) continue; double delta = 0; int total_i = OutputLayer.Total(); CNeuron *prev, *out; for(int i = 0; i < total_i; i++) { prev = prevLayer.At(n * iStep + i); out = OutputLayer.At(total_i - i - 1); if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) continue; delta += prev.getOutputVal() * out.getGradient(); } if(optimization == SGD) con.weight += con.deltaWeight = (delta != 0 ? eta*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0); else { con.mt = b1 * con.mt + (1 - b1) * delta; con.vt = b2 * con.vt + (1 - b2) * delta * delta + 0.00000001; con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight)); // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why. con.weight += con.deltaWeight; } // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches // AI\Network.cl's UpdateWeightsConvMomentum/UpdateWeightsConvAdam MAX_WEIGHT clamp. con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight)); } if(optimization == ADAM) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { iWindow = window; iStep = step; if(!CNeuronBase::Init(window, myIndex, optimization_type)) return false; OutputLayer = new CLayer(numOutputs); if(CheckPointer(OutputLayer) == POINTER_INVALID) return false; if(OutputLayer.Reserve(units_count)) for(int i = 0; i < units_count; i++) { if(!OutputLayer.CreateElement(i)) return false; OutputLayer.IncreaseTotal(); } //--- if(Type() == defNeuronPool) { if(CheckPointer(Connections) != POINTER_INVALID) Connections.Clear(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronPool::~CNeuronPool(void) { delete OutputLayer; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) return false; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; for(int i = 0; i <= total; i += iStep) { double sum = 0; for(int j = 0; j < iWindow; j++) { temp = prevLayer.At(i + j); if(CheckPointer(temp) == POINTER_INVALID) continue; sum += temp.getOutputVal(); } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setOutputVal(sum / iWindow); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) return false; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || CheckPointer(prevLayer.At(0)) == POINTER_INVALID) return false; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronPool *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) return false; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) return false; } //--- CNeuronBase *prevNeuron, *outputNeuron; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; double prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (i - i % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevNeuron.Type() != defNeuron) { CNeuronPool *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; double prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (index - index % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronPool *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) return false; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) return false; } //--- CNeuronBase *prevNeuron, *outputNeuron; CConnection *con; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; double prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (i - i % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep; con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) return false; //--- if(prevNeuron.Type() != defNeuron) { CNeuronPool *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; CConnection *con; double prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; double stop = (index - index % iStep) / iStep + 1; for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep); con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Save(int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE) return false; //--- return Connections.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "LayerDescription.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNet { protected: void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0); bool InitOpenCL(void); bool InitDirectML(void); public: CNet(CArrayObj *Description); ~CNet(void); bool feedForward(CArrayDouble *inputVals); //--- sampleWeight scales this example's output-layer gradient before it propagates back through the //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same //--- example multiple times). void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0); void getResults(CArrayDouble *&resultVals) ; double getRecentAverageError() { return recentAverageError; } //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array //--- when there is nothing to persist/restore. bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]); bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[]); //--- ephemeral in-run weight checkpoints (agent-local scratch file, never FILE_COMMON): unlike //--- Save()/Load() these are NOT blocked in the tester/optimizer, since they exist only to let a //--- single Train() call snapshot/restore weights mid-run (stability-based early stopping) and //--- never persist a model across separate backtest or optimization passes. bool SaveCheckpoint(string file_name); bool LoadCheckpoint(string file_name); //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight + //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation. //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) - //--- true whenever the shadow was cloned from live via Save()/Load() and never independently //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers. bool BlendWeightsFrom(CNet &live, double tau); //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly //--- initialized and carry the real learning signal) of each output neuron's incoming weight block, //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more //--- non-majority classes than the true base rate warrants until backProp corrects it over many //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns //--- false (no-op) rather than corrupt anything if that assumption doesn't hold. bool SeedOutputLayerBias(const double &biasValues[]); //--- static double recentAverageSmoothingFactor; private: CArrayLayer *layers; COpenCLMy *opencl; CDirectMLMy *directml; double recentAverageError; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNet::CNet(CArrayObj *Description) { if(CheckPointer(Description) == POINTER_INVALID) return; //--- int total = Description.Total(); if(total <= 0) return; //--- layers = new CArrayLayer(); if(CheckPointer(layers) == POINTER_INVALID) return; //--- CLayer *temp; CLayerDescription *desc = NULL, *next = NULL, *prev = NULL; CNeuronBase *neuron = NULL; CNeuronPool *neuron_p = NULL; int output_count = 0; int temp_count = 0; //--- next = Description.At(1); if(CheckPointer(next) != POINTER_INVALID && (next.type == defNeuron || next.type == defNeuronBaseOCL || next.type == defNeuronConv || next.type == defNeuronConvOCL || next.type == defNeuronLSTM)) { //--- OpenCL first, DirectML/D3D12 next, plain CPU as the final fallback if(!InitOpenCL()) InitDirectML(); } //--- for(int i = 0; i < total; i++) { prev = desc; desc = Description.At(i); if((i + 1) < total) { next = Description.At(i + 1); if(CheckPointer(next) == POINTER_INVALID) return; } else next = NULL; int outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL) ? 0 : next.count); temp = new CLayer(outputs); int neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL ? 1 : 0)); if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { CNeuronBaseOCL *neuron_ocl = NULL; switch(desc.type) { case defNeuron: case defNeuronBaseOCL: neuron_ocl = new CNeuronBaseOCL(); if(CheckPointer(neuron_ocl) == POINTER_INVALID) { delete temp; return; } if(CheckPointer(opencl) != POINTER_INVALID ? !neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization) : !neuron_ocl.Init(outputs, 0, directml, desc.count, desc.optimization)) { delete temp; return; } neuron_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_ocl)) { delete neuron_ocl; delete temp; return; } neuron_ocl = NULL; break; case defNeuronConv: case defNeuronConvOCL: { CNeuronConvOCL *neuron_conv = new CNeuronConvOCL(); if(CheckPointer(neuron_conv) == POINTER_INVALID) { delete temp; return; } //--- number of sliding positions - same formula the CPU CNeuronConv path uses if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL) { int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count); temp_count = (prevCount - desc.window) % desc.step; output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } else { temp_count = (output_count - desc.window) % desc.step; output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } bool convInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, output_count, desc.optimization) : neuron_conv.Init(outputs, 0, directml, desc.window, desc.step, desc.count, output_count, desc.optimization)); if(!convInit) { delete neuron_conv; delete temp; return; } neuron_conv.SetActivationFunction(desc.activation); if(!temp.Add(neuron_conv)) { delete neuron_conv; delete temp; return; } neuron_conv = NULL; break; } case defNeuronPool: case defNeuronPoolOCL: { CNeuronPoolOCL *neuron_pool = new CNeuronPoolOCL(); if(CheckPointer(neuron_pool) == POINTER_INVALID) { delete temp; return; } //--- number of sliding positions - same formula the CPU CNeuronPool path uses if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL) { int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count); temp_count = (prevCount - desc.window) % desc.step; output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } else { temp_count = (output_count - desc.window) % desc.step; output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2); } bool poolInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_pool.Init(outputs, 0, opencl, desc.window, desc.step, output_count, desc.optimization) : neuron_pool.Init(outputs, 0, directml, desc.window, desc.step, output_count, desc.optimization)); if(!poolInit) { delete neuron_pool; delete temp; return; } if(!temp.Add(neuron_pool)) { delete neuron_pool; delete temp; return; } neuron_pool = NULL; break; } case defNeuronLSTM: case defNeuronLSTMOCL: { CNeuronLSTMOCL *neuron_lstm = new CNeuronLSTMOCL(); if(CheckPointer(neuron_lstm) == POINTER_INVALID) { delete temp; return; } bool lstmInit = (CheckPointer(opencl) != POINTER_INVALID ? neuron_lstm.Init(outputs, 0, opencl, desc.count, desc.optimization) : neuron_lstm.Init(outputs, 0, directml, desc.count, desc.optimization)); if(!lstmInit) { delete neuron_lstm; delete temp; return; } if(!temp.Add(neuron_lstm)) { delete neuron_lstm; delete temp; return; } neuron_lstm = NULL; break; } default: return; break; } } else for(int n = 0; n < neurons; n++) { switch(desc.type) { case defNeuron: neuron = new CNeuron(); if(CheckPointer(neuron) == POINTER_INVALID) { delete temp; delete layers; return; } neuron.Init(outputs, n, desc.optimization); neuron.SetActivationFunction(desc.activation); break; case defNeuronConv: neuron_p = new CNeuronConv(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronPool: neuron_p = new CNeuronPool(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronLSTM: neuron_p = new CNeuronLSTM(); if(CheckPointer(neuron_p) == POINTER_INVALID) { delete temp; delete layers; return; } output_count = (next != NULL ? next.window : desc.step); if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization)) neuron = neuron_p; break; } if(!temp.Add(neuron)) { delete temp; delete layers; return; } neuron = NULL; } if(!layers.Add(temp)) { delete temp; delete layers; return; } } //--- } //+------------------------------------------------------------------+ //| Tries to initialize OpenCL; on any failure (no GPU, driver | //| missing, kernel build error) frees it and leaves opencl==NULL | //| so the rest of CNet transparently runs its CPU code path. | //+------------------------------------------------------------------+ bool CNet::InitOpenCL(void) { if(CheckPointer(opencl) != POINTER_INVALID) return true; //--- opencl = new COpenCLMy(); if(CheckPointer(opencl) == POINTER_INVALID || !opencl.Initialize(cl_program, true)) { if(CheckPointer(opencl) != POINTER_INVALID) delete opencl; opencl = NULL; PrintFormat("%s: OpenCL unavailable, falling back to CPU", __FUNCTION__); return false; } //--- create kernels opencl.SetKernelsCount(18); opencl.KernelCreate(def_k_FeedForward, "FeedForward"); opencl.KernelCreate(def_k_CaclOutputGradient, "CaclOutputGradient"); opencl.KernelCreate(def_k_CaclHiddenGradient, "CaclHiddenGradient"); opencl.KernelCreate(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum"); opencl.KernelCreate(def_k_UpdateWeightsAdam, "UpdateWeightsAdam"); opencl.KernelCreate(def_k_FeedForwardConv, "FeedForwardConv"); opencl.KernelCreate(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv"); opencl.KernelCreate(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum"); opencl.KernelCreate(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam"); opencl.KernelCreate(def_k_LSTM_Gates, "LSTM_Gates"); opencl.KernelCreate(def_k_LSTM_State, "LSTM_State"); opencl.KernelCreate(def_k_LSTM_GateGradient, "LSTM_GateGradient"); opencl.KernelCreate(def_k_LSTM_WeightsGradient, "LSTM_WeightsGradient"); opencl.KernelCreate(def_k_LSTM_InputsGradient, "LSTM_InputsGradient"); opencl.KernelCreate(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam"); opencl.KernelCreate(def_k_LSTM_UpdateWeightsMomentum, "LSTM_UpdateWeightsMomentum"); opencl.KernelCreate(def_k_FeedForwardProof, "FeedForwardProof"); opencl.KernelCreate(def_k_CalcInputGradientProof, "CalcInputGradientProof"); return true; } //+------------------------------------------------------------------+ //| Second-tier GPU fallback: only tried when OpenCL init failed. | //| Requires DirectML\WarriorDML.dll in the terminal's Libraries | //| folder (build it with DirectML\build.bat); on any failure frees | //| itself and leaves directml==NULL so CNet falls through to CPU. | //+------------------------------------------------------------------+ bool CNet::InitDirectML(void) { if(CheckPointer(directml) != POINTER_INVALID) return true; //--- directml = new CDirectMLMy(); if(CheckPointer(directml) == POINTER_INVALID) return false; directml.SetCpuLoadPercent((int)TargetCPULoad); if(!directml.Initialize()) { int err = directml.LastError(); delete directml; directml = NULL; string reason; switch(err) { case 1: reason = "CreateDXGIFactory1 failed"; break; case 2: reason = "no DX12 hardware adapter found (feature level 11_0)"; break; case 3: reason = "compute command queue creation failed"; break; case 4: reason = "command allocator creation failed"; break; case 5: reason = "command list creation failed"; break; case 6: reason = "fence creation failed"; break; case 7: reason = "fence event creation failed"; break; case 8: reason = "HLSL kernel compile/PSO creation failed"; break; default: reason = "neither WarriorDML.dll nor WarriorCPU.dll loaded (check Libraries folder / \"Allow DLL imports\")"; } PrintFormat("%s: DirectML/D3D12 and CPU DLL both unavailable (%s), falling back to slow per-object CPU path", __FUNCTION__, reason); return false; } if(directml.Tier() == COMPUTE_TIER_CPU) PrintFormat("%s: DirectML/D3D12 unavailable, using multithreaded CPU DLL fallback (%d threads)", __FUNCTION__, directml.CpuThreadsUsed()); else PrintFormat("%s: DirectML/D3D12 GPU tier active", __FUNCTION__); return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CArrayDouble *inputVals) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1) return false; //--- CLayer *previous = NULL; CLayer *current = layers.At(0); int total = MathMin(current.Total(), inputVals.Total()); CNeuronBase *neuron = NULL; bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID); if(!gpuActive) { for(int i = 0; i < total; i++) { neuron = current.At(i); if(CheckPointer(neuron) == POINTER_INVALID) return false; neuron.setOutputVal(inputVals.At(i)); } } else { CNeuronBaseOCL *neuron_ocl = current.At(0); int total_data = inputVals.Total(); bool written; if(CheckPointer(opencl) != POINTER_INVALID) { //--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own //--- BufferWrite(), this call bypasses that class entirely (it writes straight into the //--- first layer's Output buffer by index), so the narrow-to-float has to happen here too. float array[]; if(ArrayResize(array, total_data) < 0) return false; for(int d = 0; d < total_data; d++) array[d] = (float)inputVals.At(d); written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data); } else { double array[]; if(ArrayResize(array, total_data) < 0) return false; for(int d = 0; d < total_data; d++) array[d] = inputVals.At(d); written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data); } if(!written) return false; } //--- CObject *temp = NULL; for(int l = 1; l < layers.Total(); l++) { previous = current; current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID) return false; //--- if(gpuActive) { CNeuronBaseOCL *current_ocl = current.At(0); if(!current_ocl.feedForward(previous.At(0))) return false; continue; } //--- total = current.Total(); if(current.At(0).Type() == defNeuron) total--; //--- for(int n = 0; n < total; n++) { neuron = current.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return false; if(previous.At(0).Type() == defNeuron) { temp = previous; if(!neuron.feedForward(temp)) return false; continue; } if(neuron.Type() == defNeuron) { if(n == 0) { CLayer *temp_l = new CLayer(total); if(CheckPointer(temp_l) == POINTER_INVALID) return false; CNeuronPool *Pool = NULL; for(int p = 0; p < previous.Total(); p++) { Pool = previous.At(p); if(CheckPointer(Pool) == POINTER_INVALID) return false; temp_l.AddArray(Pool.getOutputLayer()); } temp = temp_l; } if(!neuron.feedForward(temp)) return false; if(n == total - 1) { CLayer *temp_l = temp; temp_l.FreeMode(false); temp_l.Shutdown(); delete temp_l; } continue; } temp = previous.At(n); if(CheckPointer(temp) == POINTER_INVALID) return false; if(!neuron.feedForward(temp)) return false; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::backProp(CArrayDouble *targetVals, double sampleWeight) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID) return; if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { backPropOCL(targetVals, sampleWeight); return; } //--- CLayer *outputLayer = layers.At(layers.Total() - 1); if(CheckPointer(outputLayer) == POINTER_INVALID) return; //--- double error = 0.0; int total = outputLayer.Total() - 1; //--- 3-output classification case: true softmax + categorical-cross-entropy gradient //--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section //--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID //--- (bounded, avoids the historical logit-runaway collapse documented at //--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD //--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly, //--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/ //--- Neutral together during training: raising one class's softmax probability now structurally //--- lowers the other two's (via the shared normalizing sum), giving real competition instead of //--- three independent binary regressions that can all drift toward "predict Neutral" together - //--- root cause of the "overshoot to all-Neutral" convergence failure this replaces. bool useSoftmaxGrad = (total == 3); double smax[3]; if(useSoftmaxGrad) { double maxLogit = -DBL_MAX; for(int n = 0; n < 3; n++) { CNeuron *nrn = outputLayer.At(n); maxLogit = MathMax(maxLogit, CLASS_LOGIT_SCALE * nrn.getOutputVal()); } double sum = 0.0; for(int n = 0; n < 3; n++) { CNeuron *nrn = outputLayer.At(n); smax[n] = exp(CLASS_LOGIT_SCALE * nrn.getOutputVal() - maxLogit); sum += smax[n]; } for(int n = 0; n < 3; n++) smax[n] /= sum; } for(int n = 0; n < total && !IsStopped(); n++) { CNeuron *neuron = outputLayer.At(n); double target = targetVals.At(n); double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target); double delta = clampedTarget - neuron.getOutputVal(); error += delta * delta; if(useSoftmaxGrad) neuron.setGradient(clampedTarget - smax[n]); else neuron.calcOutputGradients(targetVals.At(n)); //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how //--- sampleWeight is derived) - scales the just-computed output gradient in place, before the //--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted //--- signal without needing its own separate weighting logic. if(sampleWeight != 1.0) neuron.setGradient(neuron.getGradient() * sampleWeight); } error /= total; error = sqrt(error); recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor; //--- CNeuronBase *neuron = NULL; CObject *temp = NULL; for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--) { CLayer *hiddenLayer = layers.At(layerNum); CLayer *nextLayer = layers.At(layerNum + 1); total = hiddenLayer.Total(); for(int n = 0; n < total && !IsStopped(); ++n) { neuron = hiddenLayer.At(n); if(nextLayer.At(0).Type() == defNeuron) { temp = nextLayer; neuron.calcHiddenGradients(temp); continue; } if(neuron.Type() == defNeuron) { double g = 0; for(int i = 0; i < nextLayer.Total(); i++) { temp = nextLayer.At(i); neuron.calcHiddenGradients(temp); g += neuron.getGradient(); } neuron.setGradient(g); continue; } temp = nextLayer.At(n); neuron.calcHiddenGradients(temp); } } //--- for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--) { CLayer *layer = layers.At(layerNum); CLayer *prevLayer = layers.At(layerNum - 1); total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0); int n_conv = 0; for(int n = 0; n < total && !IsStopped(); n++) { neuron = layer.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return; if(neuron.Type() == defNeuronPool) continue; switch(prevLayer.At(0).Type()) { case defNeuron: temp = prevLayer; neuron.updateInputWeights(temp); break; case defNeuronConv: case defNeuronPool: case defNeuronLSTM: if(neuron.Type() == defNeuron) { for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++) { temp = prevLayer.At(n_conv); neuron.updateInputWeights(temp); } } else { temp = prevLayer.At(n); neuron.updateInputWeights(temp); } break; default: temp = NULL; break; } } } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID || (CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID)) return; CLayer *currentLayer = layers.At(layers.Total() - 1); if(CheckPointer(currentLayer) == POINTER_INVALID) return; //--- double error = 0.0; int total = targetVals.Total(); double result[]; CNeuronBaseOCL *neuron = currentLayer.At(0); if(neuron.getOutputVal(result) < total) return; for(int n = 0; n < total && !IsStopped(); n++) { double target = targetVals.At(n); // Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever // target==0, so a one-hot classification target only ever counted the true class's own // error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric, // which is what CNet::backProp()'s CPU-fallback path computes for every output // unconditionally, and is what actually drives the dError<0.1 convergence gate in // ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp // / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was. double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n]; error += MathPow(delta, 2); } error /= total; error = sqrt(error); recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor; if(!neuron.calcOutputGradients(targetVals)) return; //--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true //--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5 //--- from the raw outputs already read back into result[] above - see the matching CNet::backProp() //--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared //--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted //--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely //--- on the MQL5 side, so none of the 3 compute backends need to change. if(total == 3) { double maxLogit = CLASS_LOGIT_SCALE * MathMax(result[0], MathMax(result[1], result[2])); double smax[3]; double sm = 0.0; for(int n = 0; n < 3; n++) { smax[n] = exp(CLASS_LOGIT_SCALE * result[n] - maxLogit); sm += smax[n]; } double gradOverwrite[3]; for(int n = 0; n < 3; n++) { smax[n] /= sm; double target = targetVals.At(n); double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target); gradOverwrite[n] = clampedTarget - smax[n]; } neuron.setGradient(gradOverwrite); } //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how //--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the //--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers //--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends. if(sampleWeight != 1.0) { double gradVals[]; int gradCount = neuron.getGradient(gradVals); if(gradCount > 0) { for(int g = 0; g < gradCount; g++) gradVals[g] *= sampleWeight; neuron.setGradient(gradVals); } } //--- Calc Hidden Gradients CObject *temp = NULL; total = layers.Total(); for(int layerNum = total - 2; layerNum > 0; layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); neuron = currentLayer.At(0); neuron.calcHiddenGradients(nextLayer.At(0)); } //--- CLayer *prevLayer = layers.At(total - 1); for(int layerNum = total - 1; layerNum > 0; layerNum--) { currentLayer = prevLayer; prevLayer = layers.At(layerNum - 1); neuron = currentLayer.At(0); neuron.updateInputWeights(prevLayer.At(0)); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::getResults(CArrayDouble *&resultVals) { if(CheckPointer(resultVals) == POINTER_INVALID) { resultVals = new CArrayDouble(); if(CheckPointer(resultVals) == POINTER_INVALID) return; } //--- resultVals.Clear(); if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0) return; //--- CLayer *output = layers.At(layers.Total() - 1); if(CheckPointer(output) == POINTER_INVALID) return; //--- if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID) { switch(output.At(0).Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: { CNeuronBaseOCL *temp = output.At(0); temp.getOutputVal(resultVals); return; } } } CNeuronBase *neuron = NULL; CLayer *temp = NULL; int total = output.Total(); if(output.At(0).Type() == defNeuron) total--; //--- for(int i = 0; i < total; i++) { neuron = output.At(i); if(CheckPointer(neuron) == POINTER_INVALID) continue; if(neuron.Type() == defNeuron) { resultVals.Add(neuron.getOutputVal()); continue; } CNeuronPool *n = neuron; temp = n.getOutputLayer(); for(int ii = 0; ii < temp.Total(); ii++) { neuron = temp.At(ii); if(CheckPointer(neuron) == POINTER_INVALID) continue; resultVals.Add(neuron.getOutputVal()); } } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]) { //--- only the shared FILE_COMMON production weights are protected from being overwritten by //--- backtest/optimization noise - a LOCAL (common=false) file is exactly what the tester's //--- per-agent cross-pass weight cache uses (see CExpertSignalAIBase::InitNeuralNetwork), and //--- must be allowed to write even inside the tester/optimizer. if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD))) return true; if(file_name == NULL) return false; //--- int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_WRITE); if(handle == INVALID_HANDLE) return false; //--- if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0 || FileWriteLong(handle, era) <= 0 || FileWriteInteger(handle, trainingComplete ? 1 : 0) <= 0) { FileClose(handle); return false; } //--- AutoTuneIndicators "winning" AD indicator params, same append-only pattern as the era/ //--- trainingComplete fields above: count-prefixed so older readers can still stop before this block int paramsCount = ArraySize(indicatorParams); if(FileWriteInteger(handle, paramsCount) <= 0) { FileClose(handle); return false; } for(int p = 0; p < paramsCount; p++) if(FileWriteDouble(handle, indicatorParams[p]) <= 0) { FileClose(handle); return false; } bool result = layers.Save(handle); FileFlush(handle); FileClose(handle); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[]) { //--- see the matching comment in Save() - only shared FILE_COMMON production weights are blocked if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD))) return false; //--- if(file_name == NULL) return false; //--- Print(file_name); int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ); if(handle == INVALID_HANDLE) return false; //--- error = FileReadDouble(handle); undefine = FileReadDouble(handle); forecast = FileReadDouble(handle); time = (datetime)FileReadLong(handle); era = FileReadLong(handle); // Older save files predate this field - FileReadInteger returns 0 past EOF, // which correctly defaults to "not complete" so a resumed run keeps training // instead of silently trusting an unfinished/unknown model as done. trainingComplete = (FileReadInteger(handle) != 0); // Older save files predate this block too - FileReadInteger returns 0 past EOF, which // correctly yields an empty indicatorParams (nothing to restore) instead of misreading weights. ArrayFree(indicatorParams); int paramsCount = FileReadInteger(handle); if(paramsCount > 0) { ArrayResize(indicatorParams, paramsCount); for(int p = 0; p < paramsCount; p++) indicatorParams[p] = FileReadDouble(handle); } //--- if(CheckPointer(layers) != POINTER_INVALID) layers.Clear(); else layers = new CArrayLayer(); int i = 0, num; //--- if(!InitOpenCL()) InitDirectML(); //--- check //--- read and check start marker - 0xFFFFFFFFFFFFFFFF long temp = FileReadLong(handle); if(temp == -1) { //--- read and check array type if(FileReadInteger(handle, INT_VALUE) != layers.Type()) { FileClose(handle); return(false); } } else { FileClose(handle); return(false); } //--- read array length num = FileReadInteger(handle, INT_VALUE); //--- read array if(num != 0) { for(i = 0; i < num; i++) { //--- create new element CLayer *Layer = new CLayer(0, handle, opencl, directml); if(!Layer.Load(handle)) break; if(!layers.Add(Layer)) break; } } FileClose(handle); //--- result return (layers.Total() == num); } //+------------------------------------------------------------------+ //| Ephemeral weight-only checkpoint, deliberately NOT gated by | //| MQL_TESTER/MQL_OPTIMIZATION - see class declaration comment. | //+------------------------------------------------------------------+ bool CNet::SaveCheckpoint(string file_name) { if(file_name == NULL || CheckPointer(layers) == POINTER_INVALID) return false; int handle = FileOpen(file_name, FILE_BIN | FILE_WRITE); if(handle == INVALID_HANDLE) return false; bool result = layers.Save(handle); FileFlush(handle); FileClose(handle); return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::LoadCheckpoint(string file_name) { if(file_name == NULL) return false; int handle = FileOpen(file_name, FILE_BIN | FILE_READ); if(handle == INVALID_HANDLE) return false; if(CheckPointer(layers) != POINTER_INVALID) layers.Clear(); else layers = new CArrayLayer(); int i = 0, num; long temp = FileReadLong(handle); if(temp == -1) { if(FileReadInteger(handle, INT_VALUE) != layers.Type()) { FileClose(handle); return(false); } } else { FileClose(handle); return(false); } num = FileReadInteger(handle, INT_VALUE); if(num != 0) { for(i = 0; i < num; i++) { CLayer *Layer = new CLayer(0, handle, opencl, directml); if(!Layer.Load(handle)) break; if(!layers.Add(Layer)) break; } } FileClose(handle); return (layers.Total() == num); } //+------------------------------------------------------------------+ //| See this method's declaration comment for the EMA shadow-weight | //| deployment rationale. | //+------------------------------------------------------------------+ bool CNet::BlendWeightsFrom(CNet &live, double tau) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID) return false; int layerTotal = MathMin(layers.Total(), live.layers.Total()); for(int l = 0; l < layerTotal; l++) { CLayer *shadowLayer = layers.At(l); CLayer *liveLayer = live.layers.At(l); if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID) continue; int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total()); for(int n = 0; n < neuronTotal; n++) { CNeuronBaseOCL *shadowNeuron = shadowLayer.At(n); CNeuronBaseOCL *liveNeuron = liveLayer.At(n); if(CheckPointer(shadowNeuron) == POINTER_INVALID || CheckPointer(liveNeuron) == POINTER_INVALID) continue; if(shadowNeuron.Type() != liveNeuron.Type()) continue; double shadowW[], liveW[]; switch(shadowNeuron.Type()) { case defNeuronBaseOCL: if(shadowNeuron.getWeights(shadowW) > 0 && liveNeuron.getWeights(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowNeuron.setWeights(shadowW); } break; case defNeuronConvOCL: { CNeuronConvOCL *shadowConv = shadowNeuron; CNeuronConvOCL *liveConv = liveNeuron; if(shadowConv.getWeightsConv(shadowW) > 0 && liveConv.getWeightsConv(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowConv.setWeightsConv(shadowW); } } break; case defNeuronLSTMOCL: { CNeuronLSTMOCL *shadowLstm = shadowNeuron; CNeuronLSTMOCL *liveLstm = liveNeuron; if(shadowLstm.getWeightsLSTM(shadowW) > 0 && liveLstm.getWeightsLSTM(liveW) > 0) { int wt = MathMin(ArraySize(shadowW), ArraySize(liveW)); for(int wi = 0; wi < wt; wi++) shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi]; shadowLstm.setWeightsLSTM(shadowW); } } break; } } } return true; } //+------------------------------------------------------------------+ //| See this method's declaration comment for the cold-start bias | //| rationale. The weight block that produces the output layer's | //| values is stored on the layer BEFORE it (see CNeuronBaseOCL:: | //| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE | //| neuron, laid out as (sourceNeurons+1) values per destination | //| neuron, the last of which is that neuron's bias term), so this | //| reaches one layer back from the output layer to edit it. | //+------------------------------------------------------------------+ bool CNet::SeedOutputLayerBias(const double &biasValues[]) { int outputs = ArraySize(biasValues); if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2) return false; CLayer *sourceLayer = layers.At(layers.Total() - 2); if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0) return false; CNeuronBaseOCL *sourceNeuron = sourceLayer.At(0); if(CheckPointer(sourceNeuron) == POINTER_INVALID || sourceNeuron.Type() != defNeuronBaseOCL) return false; int inputs = sourceNeuron.Neurons(); double weights[]; int count = sourceNeuron.getWeights(weights); if(count != (inputs + 1) * outputs) return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess for(int i = 0; i < outputs; i++) weights[(inputs + 1) * i + inputs] = biasValues[i]; return sourceNeuron.setWeights(weights); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Save(const int file_handle) { if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle)) return false; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPool::Load(const int file_handle) { if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle)) return false; iWindow = FileReadInteger(file_handle, INT_VALUE); iStep = FileReadInteger(file_handle, INT_VALUE); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Save(const int file_handle) { if(!CNeuronPool::Save(file_handle)) return false; if(FileWriteDouble(file_handle, param) < 8) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Load(const int file_handle) { if(!CNeuronPool::Load(file_handle)) return false; param = FileReadDouble(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronPool { protected: CLayer *ForgetGate; CLayer *InputGate; CLayer *OutputGate; CLayer *NewContent; CArrayDouble *Memory; CArrayDouble *PrevMemory; CArrayDouble *Input; CArrayDouble *InputGradient; //--- virtual bool feedForward(CLayer *prevLayer); virtual bool calcHiddenGradients(CLayer *&nextLayer); virtual bool updateInputWeights(CLayer *&prevLayer); virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data); virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type); virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence); public: CNeuronLSTM(void); ~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::CNeuronLSTM(void) { ForgetGate = new CLayer(); InputGate = new CLayer(); OutputGate = new CLayer(); NewContent = new CLayer(); Memory = new CArrayDouble(); PrevMemory = new CArrayDouble(); Input = new CArrayDouble(); InputGradient = new CArrayDouble(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::~CNeuronLSTM(void) { if(CheckPointer(ForgetGate) != POINTER_INVALID) delete ForgetGate; if(CheckPointer(InputGate) != POINTER_INVALID) delete InputGate; if(CheckPointer(OutputGate) != POINTER_INVALID) delete OutputGate; if(CheckPointer(NewContent) != POINTER_INVALID) delete NewContent; if(CheckPointer(Memory) != POINTER_INVALID) delete Memory; if(CheckPointer(PrevMemory) != POINTER_INVALID) delete PrevMemory; if(CheckPointer(Input) != POINTER_INVALID) delete Input; if(CheckPointer(InputGradient) != POINTER_INVALID) delete InputGradient; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { if(units_count <= 0) return false; //--- Init Layers if(!CNeuronPool::Init(numOutputs, myIndex, window, step, units_count, optimization_type)) return false; if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(InputGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type)) return false; if(!InitLayer(NewContent, units_count, window + units_count, optimization_type)) return false; if(!Memory.Reserve(units_count)) return false; if(!PrevMemory.Reserve(units_count)) return false; CNeuron *temp; for(int i = 0; i < units_count; i++) { if(!Memory.Add(0)) return false; if(!PrevMemory.Add(0)) return false; temp = OutputLayer.At(i); temp.setOutputVal(0); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(layer) == POINTER_INVALID) { layer = new CLayer(numOutputs); if(CheckPointer(layer) == POINTER_INVALID) return false; } else layer.Clear(); if(!layer.Reserve(numUnits)) return false; //--- CNeuron *temp; for(int i = 0; i < numUnits; i++) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) return false; if(!temp.Init(numOutputs + 1, i, optimization_type)) return false; if(!layer.Add(temp)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0) return false; CNeuronBase *temp; CConnection *temp_con; if(CheckPointer(Input) == POINTER_INVALID) { Input = new CArrayDouble(); if(CheckPointer(Input) == POINTER_INVALID) return false; } else Input.Clear(); //--- Concatenate input sequence int total = prevLayer.Total(); if(!Input.Reserve(total + OutputLayer.Total())) return false; for(int i = 0; i < total; i++) { temp = prevLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) return false; } total = OutputLayer.Total(); for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) return false; } int total_data = Input.Total(); //--- Calculated forget gate CArrayDouble *forget_gate = CalculateGate(ForgetGate, Input); if(CheckPointer(forget_gate) == POINTER_INVALID) return false; //--- Calculated input gate CArrayDouble *input_gate = CalculateGate(InputGate, Input); if(CheckPointer(input_gate) == POINTER_INVALID) return false; //--- Calculated output gate CArrayDouble *output_gate = CalculateGate(OutputGate, Input); if(CheckPointer(output_gate) == POINTER_INVALID) return false; //--- Calculated new content CArrayDouble *new_content = new CArrayDouble(); if(CheckPointer(new_content) == POINTER_INVALID) return false; total = NewContent.Total(); for(int i = 0; i < total; i++) { temp = NewContent.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; double val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) return false; val += temp_con.weight * Input.At(c); } val = TanhFunction(val); temp.setOutputVal(val); if(!new_content.Add(val)) return false; } //--- Calculated output sequences for(int i = 0; i < total; i++) { if(PrevMemory.Total() <= i) PrevMemory.Add(Memory.At(i)); else PrevMemory.Update(i, Memory.At(i)); double value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i); if(!Memory.Update(i, value)) return false; temp = OutputLayer.At(i); value = TanhFunction(value) * output_gate.At(i); temp.setOutputVal(value); } //--- delete forget_gate; delete input_gate; delete new_content; delete output_gate; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CArrayDouble *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayDouble *sequence) { CNeuronBase *temp; CConnection *temp_con; CArrayDouble *result = new CArrayDouble(); if(CheckPointer(gate) == POINTER_INVALID) return NULL; int total = gate.Total(); int total_data = sequence.Total(); for(int i = 0; i < total; i++) { temp = gate.At(i); if(CheckPointer(temp) == POINTER_INVALID) { delete result; return NULL; } double val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) { delete result; return NULL; } val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c)); } val = SigmoidFunction(val); temp.setOutputVal(val); if(!result.Add(val)) { delete result; return NULL; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(InputGradient) == POINTER_INVALID) { InputGradient = new CArrayDouble(); if(CheckPointer(InputGradient) == POINTER_INVALID) return false; } else InputGradient.Clear(); //--- int total = OutputLayer.Total(); CNeuron *temp; CArrayDouble *MemoryGradient = new CArrayDouble(); CNeuron *gate; CConnection *con; //--- if(nextLayer != OutputLayer) for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(temp.sumDOW(nextLayer)); } //--- Calculated memory and output gate gradients if(CheckPointer(MemoryGradient) == POINTER_INVALID) return false; if(!MemoryGradient.Reserve(total)) return false; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; double value = temp.getGradient() * gate.getOutputVal(); value = TanhFunctionDerivative(Memory.At(i)) * value; if(i >= MemoryGradient.Total()) { if(!MemoryGradient.Add(value)) return false; } else { value = MemoryGradient.At(i) + value; if(!MemoryGradient.Update(i, value)) return false; } gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0); temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0); } //--- Calculated input gradients int total_inp = temp.getConnections().Total(); for(int n = 0; n < total_inp; n++) { double value = 0; for(int i = 0; i < total; i++) { temp = ForgetGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = InputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = OutputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = NewContent.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; } if(InputGradient.Total() >= n) { if(!InputGradient.Add(value)) return false; } else if(!InputGradient.Update(n, value)) return false; } //--- Calculated gradients for prev. state int shift = total_inp - total; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) return false; temp.setGradient(InputGradient.At(shift + i)); } //--- Calculated memory and output gate gradients for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; double value = temp.getGradient() * gate.getPrevVal(); value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value; if(!MemoryGradient.Update(i, value)) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0)); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) return false; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0)); } //--- delete MemoryGradient; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID) return false; //--- if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input) || !updateInputWeights(NewContent, Input)) { return false; } if(optimization == ADAM) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayDouble *input_data) { if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID) return false; CNeuronBase *neuron; CConnection *con; int total_n = gate.Total(); int total_data = input_data.Total(); double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int n = 0; n < total_n; n++) { neuron = gate.At(n); if(CheckPointer(neuron) == POINTER_INVALID) return false; double g = neuron.getGradient(); double g2 = g * g; for(int i = 0; i < total_data; i++) { con = neuron.getConnections().At(i); if(CheckPointer(con) == POINTER_INVALID) return false; double data = input_data.At(i); if(optimization == SGD) con.weight += con.deltaWeight = (g != 0 && data != 0 ? eta * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight; else { con.mt = b1 * con.mt + (1 - b1) * g; con.vt = b2 * con.vt + (1 - b2) * g2 + 0.00000001; con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight)); // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why. con.weight += con.deltaWeight; } // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches // AI\Network.cl's LSTM_UpdateWeightsAdam MAX_WEIGHT clamp. con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight)); } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index) return false; //--- prevNeuron.setGradient(InputGradient.At(index)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) return false; //--- int total = prevLayer.Total(); if(total <= 0) return false; CNeuronBase *neuron; bool result = true; for(int i = 0; (i < total && result); i++) { neuron = prevLayer.At(i); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } result = calcInputGradients(neuron, i); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Save(const int file_handle) { if(!CNeuronPool::Save(file_handle)) return false; if(!ForgetGate.Save(file_handle)) return false; if(!InputGate.Save(file_handle)) return false; if(!OutputGate.Save(file_handle)) return false; if(!NewContent.Save(file_handle)) return false; if(!Memory.Save(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Load(const int file_handle) { if(!CNeuronPool::Load(file_handle)) return false; if(!ForgetGate.Load(file_handle)) return false; if(!InputGate.Load(file_handle)) return false; if(!OutputGate.Load(file_handle)) return false; if(!NewContent.Load(file_handle)) return false; if(!Memory.Load(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::activationFunction(double x) { switch(activation) { case NONE: return(x); break; case TANH: return TanhFunction(x); break; case SIGMOID: return SigmoidFunction(x); break; case PRELU: // fixed 0.01 slope, matches CNeuronConv::activationFunction's `param` default - // this case was previously missing here, so any generic Dense (defNeuron) layer // given PRELU silently fell through to the `return x` below (pure linear identity, // no nonlinearity at all) instead of leaky-ReLU. return(x >= 0 ? x : 0.01 * x); break; } //--- return x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ double CNeuronBase::activationFunctionDerivative(double x) { switch(activation) { case NONE: return(1); break; case TANH: return TanhFunctionDerivative(x); break; case SIGMOID: return SigmoidFunctionDerivative(x); break; case PRELU: // See activationFunction() above - was previously missing here too, defaulting // to a derivative of 1 (which happens to be right for x>=0, but wrong for x<0, // where it must be the 0.01 leak slope, not 1). return(x >= 0 ? 1.0 : 0.01); break; } //--- return 1; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = -1; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return -1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL, CDirectMLMy *directml = NULL) : hWeights(-1), hDeltaWeights(-1), hOutput(-1), hGradient(-1) { iOutputs = outputs; iFileHandle = handle; OpenCL = opencl; DirectML = directml; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::Load(const int file_handle) { iFileHandle = file_handle; if(!CArrayObj::Load(file_handle)) return false; if(CheckPointer(m_data[0]) == POINTER_INVALID) return false; //--- switch(m_data[0].Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: { CNeuronBaseOCL *temp = m_data[0]; iOutputs = temp.getConnections(); break; } default: { CNeuronBase *temp = m_data[0]; iOutputs = temp.getConnections().Total(); break; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNet::~CNet(void) { if(CheckPointer(layers) != POINTER_INVALID) delete layers; if(CheckPointer(opencl) != POINTER_INVALID) { opencl.Shutdown(); delete opencl; } if(CheckPointer(directml) != POINTER_INVALID) delete directml; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #include "BufferDouble.mqh" //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: COpenCLMy *OpenCL; CDirectMLMy *DirectML; CBufferDouble *Output; CBufferDouble *PrevOutput; CBufferDouble *Weights; CBufferDouble *DeltaWeights; CBufferDouble *Gradient; CBufferDouble *FirstMomentum; CBufferDouble *SecondMomentum; //--- const double alpha; int t; //--- int m_myIndex; ENUM_ACTIVATION activation; ENUM_OPTIMIZATION optimization; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronBaseOCL(void); ~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } //--- virtual int getOutputIndex(void) { return Output.GetIndex(); } virtual int getPrevOutIndex(void) { return PrevOutput.GetIndex(); } virtual int getGradientIndex(void) { return Gradient.GetIndex(); } virtual int getWeightsIndex(void) { return Weights.GetIndex(); } virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); } virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); } virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();} //--- virtual int getOutputVal(double &values[]) { return Output.GetData(values); } virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); } virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); } virtual int getGradient(double &values[]) { return Gradient.GetData(values); } //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same //--- buffer to propagate into the hidden layers. virtual bool setGradient(const double &values[]) { int count = ArraySize(values); for(int i = 0; i < count; i++) if(!Gradient.Update(i, values[i])) return false; return Gradient.BufferWrite(); } // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created, // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here. virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); } // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment // (see that method's declaration comment) - writes a full replacement weight array back to this // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element, // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment // then pushes it to the device - the shape callers use when blending two already-read-out arrays. virtual bool setWeights(double &values[]) { if(CheckPointer(Weights) == POINTER_INVALID) return false; if(!Weights.AssignArray(values)) return false; return Weights.BufferWrite(); } virtual int Neurons(void) { return Output.Total(); } virtual ENUM_ACTIVATION Activation(void) { return activation; } virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID ? Weights.Total() / (Gradient.Total()) : 0); } //--- virtual bool feedForward(CObject *SourceObject); virtual bool calcHiddenGradients(CObject *TargetObject); virtual bool calcOutputGradients(CArrayDouble *Target); virtual bool updateInputWeights(CObject *SourceObject); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBaseOCL; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum), activation(TANH), optimization(SGD), t(1) { OpenCL = NULL; DirectML = NULL; Output = new CBufferDouble(); PrevOutput = new CBufferDouble(); Weights = new CBufferDouble(); DeltaWeights = new CBufferDouble(); Gradient = new CBufferDouble(); FirstMomentum = new CBufferDouble(); SecondMomentum = new CBufferDouble(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::~CNeuronBaseOCL(void) { if(CheckPointer(Output) != POINTER_INVALID) delete Output; if(CheckPointer(PrevOutput) != POINTER_INVALID) delete PrevOutput; if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; if(CheckPointer(Gradient) != POINTER_INVALID) delete Gradient; if(CheckPointer(FirstMomentum) != POINTER_INVALID) delete FirstMomentum; if(CheckPointer(SecondMomentum) != POINTER_INVALID) delete SecondMomentum; OpenCL = NULL; DirectML = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) return false; OpenCL = open_cl; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(OpenCL)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting // for the fact that they zero out roughly half their input distribution, whereas the // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and // was ~2x too conservative here. Applied to every layer through this one shared Init() // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through - // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting // extremes on every backend, so the imprecision there is not worth the much larger, riskier // change of threading activation awareness through every neuron subtype's Init() overload. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(OpenCL)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(OpenCL)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(OpenCL)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same | //| buffer layout, buffers just get created on the DML backend. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0) return false; DirectML = direct_ml; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(DirectML)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(DirectML)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init - see the matching OpenCL Init() overload above for the full rationale. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(DirectML)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(DirectML)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(DirectML)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ #include "NeuronOCLConvPool.mqh" //+------------------------------------------------------------------+ //| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from | //| scratch from the standard LSTM equations - NOT ported from the | //| NeuroNet_DNG reference (see the note above the LSTM kernels in | //| Network.cl for why). Single-timestep-truncated BPTT: gradient | //| does not flow back into h_prev/c_prev from an earlier step. | //| Adam-only - Init fails for any other optimization type. | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: int m_iInputs; CBufferDouble *WeightsLSTM; CBufferDouble *FirstMomentumLSTM; CBufferDouble *SecondMomentumLSTM; CBufferDouble *DeltaWeightsLSTM; CBufferDouble *WeightsGradient; CBufferDouble *Concatenated; CBufferDouble *ConcatenatedGradient; CBufferDouble *Memory; CBufferDouble *HiddenCache; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool SetInputs(int count); public: CNeuronLSTMOCL(void) : m_iInputs(-1) { WeightsLSTM = NULL; FirstMomentumLSTM = NULL; SecondMomentumLSTM = NULL; DeltaWeightsLSTM = NULL; WeightsGradient = NULL; Concatenated = NULL; ConcatenatedGradient = NULL; Memory = NULL; HiddenCache = NULL; } ~CNeuronLSTMOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual int Type(void) const { return defNeuronLSTMOCL; } // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment. virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); } virtual bool setWeightsLSTM(double &values[]) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; if(!WeightsLSTM.AssignArray(values)) return false; return WeightsLSTM.BufferWrite(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::~CNeuronLSTMOCL(void) { if(CheckPointer(WeightsLSTM) != POINTER_INVALID) delete WeightsLSTM; if(CheckPointer(FirstMomentumLSTM) != POINTER_INVALID) delete FirstMomentumLSTM; if(CheckPointer(SecondMomentumLSTM) != POINTER_INVALID) delete SecondMomentumLSTM; if(CheckPointer(DeltaWeightsLSTM) != POINTER_INVALID) delete DeltaWeightsLSTM; if(CheckPointer(WeightsGradient) != POINTER_INVALID) delete WeightsGradient; if(CheckPointer(Concatenated) != POINTER_INVALID) delete Concatenated; if(CheckPointer(ConcatenatedGradient) != POINTER_INVALID) delete ConcatenatedGradient; if(CheckPointer(Memory) != POINTER_INVALID) delete Memory; if(CheckPointer(HiddenCache) != POINTER_INVALID) delete HiddenCache; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type)) return false; uint H = numNeurons; if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(OpenCL)) return false; if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(OpenCL)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(OpenCL)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(OpenCL)) return false; m_iInputs = -1; return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above. | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type)) return false; uint H = numNeurons; if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(DirectML)) return false; if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(DirectML)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(DirectML)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(DirectML)) return false; m_iInputs = -1; return true; } //+------------------------------------------------------------------+ //| Lazily sized on the first feedForward call, once the previous | //| layer's neuron count (the input size) is known. | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::SetInputs(int count) { m_iInputs = count; uint H = (uint)Neurons(); int total = (int)(4 * H * (H + count + 1)); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) { WeightsLSTM = new CBufferDouble(); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; } if(!WeightsLSTM.Reserve(total)) return false; // Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full // rationale; fan-in here is hidden units + input width (each gate reads both). double weighScale = 1.0 / MathSqrt((double)(H + count) + 1.0); for(int i = 0; i < total; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!WeightsLSTM.Add(weigh)) return false; } if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) { FirstMomentumLSTM = new CBufferDouble(); if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) return false; } if(!FirstMomentumLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) { SecondMomentumLSTM = new CBufferDouble(); if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) return false; } if(!SecondMomentumLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML)) return false; //--- // Momentum accumulator - only meaningfully used when optimization==SGD // (see updateInputWeights()), but allocated unconditionally regardless of // the configured optimizer, matching CNeuronBaseOCL::Init's pattern for // the dense-layer DeltaWeights/FirstMomentum/SecondMomentum trio above. if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) { DeltaWeightsLSTM = new CBufferDouble(); if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) return false; } if(!DeltaWeightsLSTM.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(WeightsGradient) == POINTER_INVALID) { WeightsGradient = new CBufferDouble(); if(CheckPointer(WeightsGradient) == POINTER_INVALID) return false; } if(!WeightsGradient.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(m_iInputs <= 0) { if(!SetInputs(NeuronOCL.Neurons())) return false; } else if(m_iInputs != NeuronOCL.Neurons()) return false; int H = Neurons(); int I = m_iInputs; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.LSTMGates(WeightsLSTM.GetIndex(), getOutputIndex(), NeuronOCL.getOutputIndex(), Concatenated.GetIndex(), H, I) || !DirectML.LSTMState(Concatenated.GetIndex(), Memory.GetIndex(), getOutputIndex(), HiddenCache.GetIndex(), getOutputIndex(), H)) { printf("Error of execution DirectML LSTM feedForward"); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset2[2] = {0, 0}; uint size2[2] = {(uint)H, 4}; OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_hidden_prev, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_inputs, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_concatenated, Concatenated.GetIndex()); OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_input_size, I); if(!OpenCL.Execute(def_k_LSTM_Gates, 2, offset2, size2)) { printf("Error of execution kernel LSTM_Gates: %d", GetLastError()); return false; } uint offset1[1] = {0}; uint size1[1] = {(uint)H}; OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_concatenated, Concatenated.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_memory, Memory.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_prev, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_cache, HiddenCache.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_output, getOutputIndex()); OpenCL.SetArgument(def_k_LSTM_State, def_k_lstms_hidden_size, H); if(!OpenCL.Execute(def_k_LSTM_State, 1, offset1, size1)) { printf("Error of execution kernel LSTM_State: %d", GetLastError()); return false; } //--- Output (== hidden_prev for the next timestep) stays GPU-resident; see the note in //--- CNeuronBaseOCL::feedForward(). return true; } //+------------------------------------------------------------------+ //| Writes the gradient into NeuronOCL (the earlier/input-side layer) | //| - same inverted-call convention as CNeuronConvOCL::calcInputGradients.| //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || m_iInputs <= 0) return false; int H = Neurons(); int I = m_iInputs; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.LSTMGateGradient(getGradientIndex(), Memory.GetIndex(), Concatenated.GetIndex(), ConcatenatedGradient.GetIndex(), H) || !DirectML.LSTMWeightsGradient(ConcatenatedGradient.GetIndex(), HiddenCache.GetIndex(), NeuronOCL.getOutputIndex(), WeightsGradient.GetIndex(), H, I) || !DirectML.LSTMInputsGradient(ConcatenatedGradient.GetIndex(), WeightsLSTM.GetIndex(), NeuronOCL.getGradientIndex(), H, I)) { printf("Error of execution DirectML LSTM calcInputGradients"); return false; } double temp[]; return NeuronOCL.getGradient(temp) > 0; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset1[1] = {0}; uint size1[1] = {(uint)H}; OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_gradient, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_memory, Memory.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated, Concatenated.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgument(def_k_LSTM_GateGradient, def_k_lstmgg_hidden_size, H); if(!OpenCL.Execute(def_k_LSTM_GateGradient, 1, offset1, size1)) { printf("Error of execution kernel LSTM_GateGradient: %d", GetLastError()); return false; } uint sizeW[1] = {(uint)WeightsLSTM.Total()}; OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_cache, HiddenCache.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_inputs, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_input_size, I); if(!OpenCL.Execute(def_k_LSTM_WeightsGradient, 1, offset1, sizeW)) { printf("Error of execution kernel LSTM_WeightsGradient: %d", GetLastError()); return false; } uint sizeI[1] = {(uint)I}; OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_concatenated_gradient, ConcatenatedGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_inputs_gradient, NeuronOCL.getGradientIndex()); OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_hidden_size, H); OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_input_size, I); if(!OpenCL.Execute(def_k_LSTM_InputsGradient, 1, offset1, sizeI)) { printf("Error of execution kernel LSTM_InputsGradient: %d", GetLastError()); return false; } //--- NeuronOCL's Gradient stays GPU-resident; see the note in //--- CNeuronConvOCL::calcInputGradients(). return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; int total = WeightsLSTM.Total(); if(CheckPointer(DirectML) != POINTER_INVALID) { if(optimization == SGD) { if(!DirectML.LSTMUpdateWeightsMomentum(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), DeltaWeightsLSTM.GetIndex(), eta, alpha, total)) { printf("Error of execution DirectML LSTM_UpdateWeightsMomentum"); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.LSTMUpdateWeightsAdam(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), FirstMomentumLSTM.GetIndex(), SecondMomentumLSTM.GetIndex(), lt, b1, b2, total)) { printf("Error of execution DirectML LSTM_UpdateWeightsAdam"); return false; } t++; } return WeightsLSTM.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint offset1[1] = {0}; uint size1[1] = {(uint)total}; if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_dw, DeltaWeightsLSTM.GetIndex()); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_momentum, (float)alpha); ResetLastError(); if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsMomentum, 1, offset1, size1)) { printf("Error of execution kernel LSTM_UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_w, WeightsLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_weights_gradient, WeightsGradient.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_m, FirstMomentumLSTM.GetIndex()); OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_v, SecondMomentumLSTM.GetIndex()); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_l, (float)lt); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b1, (float)b1); OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b2, (float)b2); ResetLastError(); if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsAdam, 1, offset1, size1)) { printf("Error of execution kernel LSTM_UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- WeightsLSTM stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights(). return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) return false; if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < INT_VALUE) return false; if(m_iInputs <= 0) return true; if(CheckPointer(WeightsLSTM) == POINTER_INVALID || !WeightsLSTM.BufferRead() || !WeightsLSTM.Save(file_handle)) return false; if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID || !FirstMomentumLSTM.BufferRead() || !FirstMomentumLSTM.Save(file_handle)) return false; if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID || !SecondMomentumLSTM.BufferRead() || !SecondMomentumLSTM.Save(file_handle)) return false; if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID || !DeltaWeightsLSTM.BufferRead() || !DeltaWeightsLSTM.Save(file_handle)) return false; if(CheckPointer(Memory) == POINTER_INVALID || !Memory.BufferRead() || !Memory.Save(file_handle)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) return false; m_iInputs = FileReadInteger(file_handle, INT_VALUE); uint H = (uint)Neurons(); //--- scratch buffers (not persisted) were sized for the placeholder Init() unit //--- count; resize them now that the real neuron count is known. if(CheckPointer(Concatenated) == POINTER_INVALID) { Concatenated = new CBufferDouble(); if(CheckPointer(Concatenated) == POINTER_INVALID) return false; } Concatenated.BufferFree(); if(!Concatenated.BufferInit(4 * H, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !Concatenated.BufferCreate(OpenCL) : !Concatenated.BufferCreate(DirectML)) return false; if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) { ConcatenatedGradient = new CBufferDouble(); if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID) return false; } ConcatenatedGradient.BufferFree(); if(!ConcatenatedGradient.BufferInit(4 * H, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !ConcatenatedGradient.BufferCreate(OpenCL) : !ConcatenatedGradient.BufferCreate(DirectML)) return false; if(CheckPointer(HiddenCache) == POINTER_INVALID) { HiddenCache = new CBufferDouble(); if(CheckPointer(HiddenCache) == POINTER_INVALID) return false; } HiddenCache.BufferFree(); if(!HiddenCache.BufferInit(H, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !HiddenCache.BufferCreate(OpenCL) : !HiddenCache.BufferCreate(DirectML)) return false; //--- if(m_iInputs <= 0) return true; int total = (int)(4 * H * (H + m_iInputs + 1)); //--- if(CheckPointer(WeightsLSTM) == POINTER_INVALID) { WeightsLSTM = new CBufferDouble(); if(CheckPointer(WeightsLSTM) == POINTER_INVALID) return false; } if(WeightsLSTM.GetIndex() >= 0) WeightsLSTM.BufferFree(); if(!WeightsLSTM.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) { FirstMomentumLSTM = new CBufferDouble(); if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID) return false; } if(FirstMomentumLSTM.GetIndex() >= 0) FirstMomentumLSTM.BufferFree(); if(!FirstMomentumLSTM.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) { SecondMomentumLSTM = new CBufferDouble(); if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID) return false; } if(SecondMomentumLSTM.GetIndex() >= 0) SecondMomentumLSTM.BufferFree(); if(!SecondMomentumLSTM.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) { DeltaWeightsLSTM = new CBufferDouble(); if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID) return false; } if(DeltaWeightsLSTM.GetIndex() >= 0) DeltaWeightsLSTM.BufferFree(); if(!DeltaWeightsLSTM.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Memory) == POINTER_INVALID) { Memory = new CBufferDouble(); if(CheckPointer(Memory) == POINTER_INVALID) return false; } if(Memory.GetIndex() >= 0) Memory.BufferFree(); if(!Memory.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !Memory.BufferCreate(OpenCL) : !Memory.BufferCreate(DirectML)) return false; //--- if(CheckPointer(WeightsGradient) == POINTER_INVALID) { WeightsGradient = new CBufferDouble(); if(CheckPointer(WeightsGradient) == POINTER_INVALID) return false; } if(!WeightsGradient.BufferInit(total, 0)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML)) return false; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: temp = SourceObject; return feedForward(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation))) { printf("Error of execution DirectML FeedForward"); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel FeedForward: %d", GetLastError()); return false; } //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex() //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData(). //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1)) { printf("Error of execution DirectML CalcHiddenGradient"); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons() + 1; OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError()); return false; } //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target) { if(CheckPointer(Target) == POINTER_INVALID) return false; int count = Target.Total(); for(int i = 0; i < count; i++) if(!Gradient.Update(i, Target.At(i))) return false; Gradient.BufferWrite(); //--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count)) { printf("Error of execution DirectML CalcOutputGradient"); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation)); ResetLastError(); if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclOutputGradient: %d", GetLastError()); return false; } //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which //--- BufferRead()s itself - see the note in feedForward() above. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { int inputs = NeuronOCL.Neurons(); int neurons = Neurons(); if(optimization == SGD) { if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons)) { printf("Error of execution DirectML UpdateWeightsMomentum"); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(), inputs, lt, b1, b2, neurons)) { printf("Error of execution DirectML UpdateWeightsAdam"); return false; } t++; } return NeuronOCL.Weights.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons(); if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex())) return false; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons())) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2)) return false; uint rest = global_work_size[1] % 4; global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save() //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject) { if(CheckPointer(TargetObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; CNeuronConvOCL *tempConv = NULL; CNeuronLSTMOCL *tempLstm = NULL; CNeuronPoolOCL *tempPool = NULL; switch(TargetObject.Type()) { case defNeuronBaseOCL: temp = TargetObject; return calcHiddenGradients(temp); break; case defNeuronConvOCL: //--- Conv owns the backward step (calcInputGradients), called on itself with //--- "this" (the earlier layer) passed in so it writes into this->Gradient. tempConv = TargetObject; return tempConv.calcInputGradients(GetPointer(this)); break; case defNeuronLSTMOCL: tempLstm = TargetObject; return tempLstm.calcInputGradients(GetPointer(this)); break; case defNeuronPoolOCL: tempPool = TargetObject; return tempPool.calcInputGradients(GetPointer(this)); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: temp = SourceObject; return updateInputWeights(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE) return false; //--- if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle)) return false; if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle)) return false; if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle)) return false; //--- if(CheckPointer(Weights) == POINTER_INVALID) { FileWriteInteger(file_handle, 0); return true; } else FileWriteInteger(file_handle, 1); //--- if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle)) return false; if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle)) return false; if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; //--- activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(Output.GetIndex() >= 0) Output.BufferFree(); if(!Output.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !Output.BufferCreate(OpenCL) : !Output.BufferCreate(DirectML)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(PrevOutput.GetIndex() >= 0) PrevOutput.BufferFree(); if(!PrevOutput.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !PrevOutput.BufferCreate(OpenCL) : !PrevOutput.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(Gradient.GetIndex() >= 0) Gradient.BufferFree(); if(!Gradient.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !Gradient.BufferCreate(OpenCL) : !Gradient.BufferCreate(DirectML)) return false; //--- if(FileReadInteger(file_handle) == 0) return true; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } if(Weights.GetIndex() >= 0) Weights.BufferFree(); if(!Weights.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !Weights.BufferCreate(OpenCL) : !Weights.BufferCreate(DirectML)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(DeltaWeights.GetIndex() >= 0) DeltaWeights.BufferFree(); if(!DeltaWeights.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeights.BufferCreate(OpenCL) : !DeltaWeights.BufferCreate(DirectML)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(FirstMomentum.GetIndex() >= 0) FirstMomentum.BufferFree(); if(!FirstMomentum.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentum.BufferCreate(OpenCL) : !FirstMomentum.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(SecondMomentum.GetIndex() >= 0) SecondMomentum.BufferFree(); if(!SecondMomentum.Load(file_handle)) return false; if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentum.BufferCreate(OpenCL) : !SecondMomentum.BufferCreate(DirectML)) return false; } //--- return true; } //+------------------------------------------------------------------+