//+------------------------------------------------------------------+ //| NeuronOCLBase.mqh | //| | //| CNeuronBaseOCL - the accelerated dense neuron (OpenCL / DirectML | //| / CPU-DLL tiers). | //| | //| Included from AI\Network.mqh AFTER every class declaration - | //| bodies only, no declarations. Relocation is behaviour-neutral by | //| construction: nothing here is reachable until Network.mqh ends. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AI_IMPL_NEURONOCLBASE_MQH #define WARRIOR_AI_IMPL_NEURONOCLBASE_MQH //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum), activation(TANH), optimization(SGD), t(1) { OpenCL = NULL; DirectML = NULL; Output = new CBufferDouble(); PrevOutput = new CBufferDouble(); Weights = new CBufferDouble(); DeltaWeights = new CBufferDouble(); Gradient = new CBufferDouble(); FirstMomentum = new CBufferDouble(); SecondMomentum = new CBufferDouble(); //--- Allocated lazily by EnsureGradAccum() only if a batched update ever runs - see its declaration. GradAccum = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::~CNeuronBaseOCL(void) { if(CheckPointer(Output) != POINTER_INVALID) delete Output; if(CheckPointer(PrevOutput) != POINTER_INVALID) delete PrevOutput; if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; if(CheckPointer(Gradient) != POINTER_INVALID) delete Gradient; if(CheckPointer(FirstMomentum) != POINTER_INVALID) delete FirstMomentum; if(CheckPointer(SecondMomentum) != POINTER_INVALID) delete SecondMomentum; if(CheckPointer(GradAccum) != POINTER_INVALID) delete GradAccum; OpenCL = NULL; DirectML = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) return false; OpenCL = open_cl; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(OpenCL)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting // for the fact that they zero out roughly half their input distribution, whereas the // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and // was ~2x too conservative here. Applied to every layer through this one shared Init() // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through - // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting // extremes on every backend, so the imprecision there is not worth the much larger, riskier // change of threading activation awareness through every neuron subtype's Init() overload. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(OpenCL)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(OpenCL)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(OpenCL)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same | //| buffer layout, buffers just get created on the DML backend. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0) return false; DirectML = direct_ml; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(DirectML)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(DirectML)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init - see the matching OpenCL Init() overload above for the full rationale. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(DirectML)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(DirectML)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(DirectML)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return feedForward(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation))) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " FeedForward failed, error " + IntegerToString(DirectML.LastError())); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel FeedForward: %d", GetLastError()); return false; } //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex() //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData(). //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead. return true; } //+------------------------------------------------------------------+ //| Pure-MQL5 dense forward pass - exact double-precision mirror of | //| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, | //| which (Gizlyk convention) owns both the inputs (its Output) and | //| the weights connecting them to THIS layer (its Weights), laid out | //| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host | //| buffers only; used when no compute backend exists (CPU inference).| //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int inputs = NeuronOCL.Neurons(); int outCount = Output.Total(); int wTotal = NeuronOCL.WeightsCount(); if(wTotal < (inputs + 1) * outCount) return false; // weight buffer smaller than the dense layout requires - refuse rather than misread for(int i = 0; i < outCount; i++) { int shift = (inputs + 1) * i; double sum = 0.0; for(int k = 0; k < inputs; k++) sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k); sum += NeuronOCL.WeightHost(shift + inputs); // bias switch(activation) { case TANH: sum = tanh(sum); break; case SIGMOID: sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum)))); break; case PRELU: if(sum < 0.0) sum *= 0.01; break; // NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case. } if(!Output.Update(i, sum)) return false; } return true; } //+------------------------------------------------------------------+ //| Host-side input write for the CPU inference path's layer 0 (there | //| is no device buffer to write into, unlike CNet::feedForward's | //| BufferWrite branch). Copies inputVals into the Output host array. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals) { if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int total = MathMin(Output.Total(), inputVals.Total()); for(int i = 0; i < total; i++) if(!Output.Update(i, inputVals.At(i))) return false; return true; } //+------------------------------------------------------------------+ //| Host-side output read for getResults() on the CPU inference path | //| (no device BufferRead available). Returns the count copied. | //+------------------------------------------------------------------+ int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values) { if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return 0; values.Clear(); int n = Output.Total(); for(int i = 0; i < n; i++) if(!values.Add(Output.At(i))) return i; return n; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { //--- count is exactly Neurons() - biases receive no backprop gradient, and the DLL needs the //--- true neuron count to derive the weight-row stride (inputs + 1). The old Neurons() + 1 //--- "bias row" only ever produced out-of-bounds reads (see the 2026-08-11 transpose fix in //--- the kernels: Network.cl CaclHiddenGradient, WarriorCPU/WarriorDML CalcHiddenGradient). if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons())) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcHiddenGradient failed, error " + IntegerToString(DirectML.LastError())); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; //--- Exactly Neurons(): the kernel derives the weight-row stride from get_global_size(0), and //--- biases receive no backprop gradient (the old +1 work-item only ever read past matrix_o). global_work_size[0] = Neurons(); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError()); return false; } //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target) { if(CheckPointer(Target) == POINTER_INVALID) return false; int count = Target.Total(); for(int i = 0; i < count; i++) if(!Gradient.Update(i, Target.At(i))) return false; Gradient.BufferWrite(); //--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcOutputGradient failed, error " + IntegerToString(DirectML.LastError())); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation)); ResetLastError(); if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclOutputGradient: %d", GetLastError()); return false; } //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which //--- BufferRead()s itself - see the note in feedForward() above. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { int inputs = NeuronOCL.Neurons(); int neurons = Neurons(); if(optimization == SGD) { if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons, 0)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsMomentum failed, error " + IntegerToString(DirectML.LastError())); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(), inputs, lt, b1, b2, neurons)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsAdam failed, error " + IntegerToString(DirectML.LastError())); return false; } t++; } //--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex() //--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand. return true; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons(); if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex())) return false; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons())) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2)) return false; //--- Dim 1 covers the full (inputs + 1)-wide weight row in groups of 4, so the bias column is //--- dispatched too. Sizing this on Neurons() (without the +1) left the bias group unreachable //--- whenever inputs % 4 == 0 - see the kernel's 2026-08-11 comment. global_work_size[1] = NeuronOCL.Neurons() + 1; uint rest = global_work_size[1] % 4; global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save() //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself. return true; } //+------------------------------------------------------------------+ //| MINI-BATCH ACCUMULATE (dense) - see the declaration comment. | //| Same dispatch shape as updateInputWeights above, but it only ADDS | //| this sample's outer product into GradAccum; no weight is touched | //| and no optimizer state advances until ApplyAccumulatedGradients. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; //--- The weight block being accumulated for lives on the SOURCE neuron, exactly as //--- updateInputWeights' matrix_w does - so the accumulator is sized and owned there too. if(!NeuronOCL.EnsureGradAccum(NeuronOCL.Weights)) return false; int inputs = NeuronOCL.Neurons(); int neurons = Neurons(); if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.AccumulateWeightGrad(NeuronOCL.GradAccum.GetIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), inputs, neurons)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " AccumulateWeightGrad failed, error " + IntegerToString(DirectML.LastError())); return false; } return true; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = neurons; global_work_size[1] = inputs + 1; // the last slot is the bias, exactly as the Adam kernel treats it if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_acc, NeuronOCL.GradAccum.GetIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_g, getGradientIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_i, NeuronOCL.getOutputIndex())) return false; if(!OpenCL.SetArgument(def_k_AccumulateWeightGrad, def_k_awg_inputs, inputs)) return false; ResetLastError(); if(!OpenCL.Execute(def_k_AccumulateWeightGrad, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel AccumulateWeightGrad: %d", GetLastError()); return false; } return true; } //+------------------------------------------------------------------+ //| Type-dispatching wrapper, mirroring updateInputWeights(CObject*). | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::accumulateInputWeightGrads(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- Every *OCL neuron type derives from CNeuronBaseOCL and stores its incoming dense matrix in //--- Weights, so one branch covers them all. Conv and LSTM own EXTRA weight blocks and override //--- this method; the legacy scalar hierarchy never reaches here (CNet refuses to batch without a //--- backend - see CNet::BatchSize). switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronBatchNormOCL: case defNeuronConvOCL: case defNeuronPoolOCL: case defNeuronLSTMOCL: { CNeuronBaseOCL *temp = SourceObject; return accumulateInputWeightGrads(temp); } } return false; } //+------------------------------------------------------------------+ //| Zero this neuron's accumulator - see the declaration comment. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::BeginGradAccum(void) { if(CheckPointer(GradAccum) == POINTER_INVALID || GradAccum.Total() <= 0) return true; // nothing accumulated here yet; the first accumulate allocates it zeroed return ZeroOptimizerBuffer(GradAccum); } //+------------------------------------------------------------------+ //| DEVICE-SIDE mini-batch apply (OpenCL only). One dispatch, nothing | //| crosses the bus. | //| | //| This exists because the host-side version below moved EIGHT full | //| weight-matrix transfers per batch per weight block - read w, acc, | //| m, v; write m, v, w, acc - each a blocking sync. At batch size 8 | //| that is roughly one whole weight matrix over the bus PER SAMPLE, | //| and it is why an RX 580 lost to a CPU thread pool on this | //| workload. Market builds cannot import a DLL, so OpenCL is the | //| tier paying clients run: this path is the product's speed. | //| | //| Returns false (rather than reporting) if anything is missing, so | //| the caller can fall back to the host implementation - which stays | //| the reference for the DLL and pure-MQL5 tiers and must be kept | //| line-for-line identical to the kernels. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale, int total) { if(CheckPointer(OpenCL) == POINTER_INVALID || total <= 0) return false; uint offset[1] = {0}; uint size[1]; size[0] = (uint)total; if(optimization == ADAM) { if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID || m.Total() < total || v.Total() < total || m.GetIndex() < 0 || v.GetIndex() < 0) return false; //--- Same bias-corrected step the host path and the unbatched kernel compute. t is NOT advanced //--- here - ApplyAccumulatedGradients owns that, once per batch across every block it holds. double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_w, w.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_acc, acc.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_m, m.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_v, v.GetIndex()) || !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_scale, (float)scale) || !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_l, (float)lt) || !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b1, (float)b1) || !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b2, (float)b2)) return false; ResetLastError(); return OpenCL.Execute(def_k_ApplyAccumAdam, 1, offset, size); } if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total || dw.GetIndex() < 0) return false; if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_w, w.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_acc, acc.GetIndex()) || !OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_dw, dw.GetIndex()) || !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_scale, (float)scale) || !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_lr, (float)eta) || !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_momentum, (float)alpha)) return false; ResetLastError(); return OpenCL.Execute(def_k_ApplyAccumMomentum, 1, offset, size); } //+------------------------------------------------------------------+ //| MINI-BATCH APPLY (dense) - ONE optimizer step on the batch MEAN. | //| | //| OpenCL takes the device path above. The host implementation below | //| remains the one shared by DirectML, the CPU DLL and the pure-MQL5 | //| tier, and is the REFERENCE the kernels are transcribed from: it | //| is a line-for-line copy of Network.cl's UpdateWeightsAdam / | //| UpdateWeightsMomentum, including both clamps and the decoupled | //| weight decay, so that batch size 1 reproduces them exactly. Edit | //| the two together or the tiers silently disagree. | //| | //| One deliberate difference remains between the two: this runs in | //| fp64 while the kernel runs in fp32. That is not a regression, it | //| is the OpenCL tier becoming SELF-consistent - its device buffers | //| are already fp32 (see CBufferDouble's m_data_f) and its unbatched | //| optimizer already ran in fp32, so the batched path was the odd | //| one out. The DLL tiers keep fp64 end to end. | //| | //| `scale` is 1/batchCount - the MEAN, not the sum. Note Adam is | //| very nearly invariant to a global gradient rescale (mt/sqrt(vt) | //| cancels it), so this matters mainly for SGD and for keeping the | //| decoupled decay term correctly proportioned. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m, CBufferDouble *v, CBufferDouble *dw, double scale) { if(CheckPointer(acc) == POINTER_INVALID || acc.Total() <= 0 || CheckPointer(w) == POINTER_INVALID || w.Total() <= 0) return true; // no weight block here (e.g. an output-layer neuron) - nothing to apply int total = MathMin(acc.Total(), w.Total()); //--- DEVICE FAST PATH. Latched off on first failure rather than retried every batch: a kernel that //--- did not build will not build later, and the host path below is a correct answer, so the right //--- behaviour is one warning and full speed on the fallback - not a failed dispatch per batch. if(g_applyAccumKernelUsable && CheckPointer(OpenCL) != POINTER_INVALID && w.GetIndex() >= 0 && acc.GetIndex() >= 0) { if(ApplyAccumOnDevice(w, acc, m, v, dw, scale, total)) return true; g_applyAccumKernelUsable = false; PrintFormat("%s: device-side mini-batch apply failed (error %d) - falling back to the host step " "for the rest of this run. Training stays correct; each batch now costs a full " "weight-matrix round trip, so expect it to be substantially slower.", __FUNCTION__, GetLastError()); } //--- Pull the device-side copies into the host mirrors. On a host-only net (no backend) there is no //--- device to read FROM and the host mirror is already the truth, so the round-trip is skipped //--- rather than treated as a failure. bool haveDevice = (w.GetIndex() >= 0); if(haveDevice && (!w.BufferRead() || !acc.BufferRead())) return false; if(optimization == ADAM) { if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID || m.Total() < total || v.Total() < total) return false; if(haveDevice && (!m.BufferRead() || !v.BufferRead())) return false; //--- Bias correction reads t but does NOT advance it here - the caller advances once per batch //--- after every block it owns has been stepped, because t counts optimizer STEPS and a batch is //--- one step no matter how many weight blocks the neuron carries. double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int i = 0; i < total; i++) { double grad = acc.At(i) * scale; double wv = w.At(i); double mt = b1 * m.At(i) + (1 - b1) * grad; //--- v is STORED square-rooted, so square it back before the recursion. See Network.cl's //--- UpdateWeightsAdam for why feeding the stored sqrt straight in is not Adam at all. double vt = sqrt(b2 * v.At(i) * v.At(i) + (1 - b2) * grad * grad); double delta = lt * mt / (vt > 0 ? vt : lt * 10) - lt * WEIGHT_DECAY * wv; delta = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, delta)); if(!w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, wv + delta))) || !m.Update(i, mt) || !v.Update(i, vt)) return false; } if(haveDevice && (!m.BufferWrite() || !v.BufferWrite())) return false; } else { if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total) return false; if(haveDevice && !dw.BufferRead()) return false; for(int i = 0; i < total; i++) { double grad = acc.At(i) * scale; double delta = eta * grad + alpha * dw.At(i); if(!dw.Update(i, delta) || !w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, w.At(i) + delta)))) return false; } if(haveDevice && !dw.BufferWrite()) return false; } //--- Clear the accumulator for the next batch and push the new weights back to the device. for(int i = 0; i < acc.Total(); i++) if(!acc.Update(i, 0.0)) return false; if(haveDevice && (!w.BufferWrite() || !acc.BufferWrite())) return false; return true; } //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ApplyAccumulatedGradients(double scale) { if(!ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale)) return false; if(optimization == ADAM) t++; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject) { if(CheckPointer(TargetObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; CNeuronConvOCL *tempConv = NULL; CNeuronLSTMOCL *tempLstm = NULL; CNeuronPoolOCL *tempPool = NULL; CNeuronBatchNormOCL *tempBN = NULL; switch(TargetObject.Type()) { case defNeuronBaseOCL: temp = TargetObject; return calcHiddenGradients(temp); break; case defNeuronBatchNormOCL: //--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step //--- and writes into this->Gradient. Routing it through the dense branch instead would run //--- CaclHiddenGradient against a weight matrix batch norm does not have. tempBN = TargetObject; return tempBN.calcInputGradients(GetPointer(this)); break; case defNeuronConvOCL: //--- Conv owns the backward step (calcInputGradients), called on itself with //--- "this" (the earlier layer) passed in so it writes into this->Gradient. tempConv = TargetObject; return tempConv.calcInputGradients(GetPointer(this)); break; case defNeuronLSTMOCL: tempLstm = TargetObject; return tempLstm.calcInputGradients(GetPointer(this)); break; case defNeuronPoolOCL: tempPool = TargetObject; return tempPool.calcInputGradients(GetPointer(this)); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return updateInputWeights(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE) return false; //--- if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle)) return false; if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle)) return false; if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle)) return false; //--- if(CheckPointer(Weights) == POINTER_INVALID) { FileWriteInteger(file_handle, 0); return true; } else FileWriteInteger(file_handle, 1); //--- if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle)) return false; if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle)) return false; if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; //--- activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(Output.GetIndex() >= 0) Output.BufferFree(); if(!Output.Load(file_handle)) return false; if(!BackendBufferCreate(Output)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(PrevOutput.GetIndex() >= 0) PrevOutput.BufferFree(); if(!PrevOutput.Load(file_handle)) return false; if(!BackendBufferCreate(PrevOutput)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(Gradient.GetIndex() >= 0) Gradient.BufferFree(); if(!Gradient.Load(file_handle)) return false; if(!BackendBufferCreate(Gradient)) return false; //--- if(FileReadInteger(file_handle) == 0) return true; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } if(Weights.GetIndex() >= 0) Weights.BufferFree(); if(!Weights.Load(file_handle)) return false; if(!BackendBufferCreate(Weights)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(DeltaWeights.GetIndex() >= 0) DeltaWeights.BufferFree(); if(!DeltaWeights.Load(file_handle)) return false; if(!BackendBufferCreate(DeltaWeights)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(FirstMomentum.GetIndex() >= 0) FirstMomentum.BufferFree(); if(!FirstMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(FirstMomentum)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(SecondMomentum.GetIndex() >= 0) SecondMomentum.BufferFree(); if(!SecondMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(SecondMomentum)) return false; } //--- return true; } #endif