Warrior_EA/AI/Impl/NeuronOCLBase.mqh

//+------------------------------------------------------------------+
//| NeuronOCLBase.mqh                                                 |
//|                                                                   |
//| CNeuronBaseOCL - the accelerated dense neuron (OpenCL / CPU-DLL   |
//| tiers).                                                           |
//|                                                                   |
//| Included from AI\Network.mqh AFTER every class declaration -      |
//| bodies only, no declarations. Relocation is behaviour-neutral by  |
//| construction: nothing here is reachable until Network.mqh ends.   |
//+------------------------------------------------------------------+
#ifndef WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
#define WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
#include "..\..\System\Random.mqh"
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
CNeuronBaseOCL::CNeuronBaseOCL(void)   :  alpha(SgdMomentum),
   activation(TANH),
   optimization(SGD),
   t(1)
  {
   OpenCL = NULL;
   ComputeDll = NULL;
   Output = new CBufferDouble();
   PrevOutput = new CBufferDouble();
   Weights = new CBufferDouble();
   DeltaWeights = new CBufferDouble();
   Gradient = new CBufferDouble();
   FirstMomentum = new CBufferDouble();
   SecondMomentum = new CBufferDouble();
   //--- Allocated lazily by EnsureGradAccum() only if a batched update ever runs - see its declaration.
   GradAccum = NULL;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
CNeuronBaseOCL::~CNeuronBaseOCL(void)
  {
   if(CheckPointer(Output) != POINTER_INVALID)
      delete Output;
   if(CheckPointer(PrevOutput) != POINTER_INVALID)
      delete PrevOutput;
   if(CheckPointer(Weights) != POINTER_INVALID)
      delete Weights;
   if(CheckPointer(DeltaWeights) != POINTER_INVALID)
      delete DeltaWeights;
   if(CheckPointer(Gradient) != POINTER_INVALID)
      delete Gradient;
   if(CheckPointer(FirstMomentum) != POINTER_INVALID)
      delete FirstMomentum;
   if(CheckPointer(SecondMomentum) != POINTER_INVALID)
      delete SecondMomentum;
   if(CheckPointer(GradAccum) != POINTER_INVALID)
      delete GradAccum;
   OpenCL = NULL;
   ComputeDll = NULL;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
      return false;
   OpenCL = open_cl;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(OpenCL))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
      // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
      // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
      // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
      // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
      // for the fact that they zero out roughly half their input distribution, whereas the
      // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
      // was ~2x too conservative here. Applied to every layer through this one shared Init()
      // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
      // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
      // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
      // extremes on every backend, so the imprecision there is not worth the much larger, riskier
      // change of threading activation awareness through every neuron subtype's Init() overload.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = WarriorRandSymmetric() * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(OpenCL))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(OpenCL))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(OpenCL))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(OpenCL))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| CPU-DLL tier equivalent of Init(COpenCLMy*) above - same          |
//| buffer layout, buffers just get created on the CPU-DLL backend.   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(compute_dll) == POINTER_INVALID || numNeurons <= 0)
      return false;
   ComputeDll = compute_dll;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(ComputeDll))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(ComputeDll))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(ComputeDll))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = WarriorRandSymmetric() * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(ComputeDll))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(ComputeDll))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(ComputeDll))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(ComputeDll))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
      case defNeuronBatchNormOCL:
         temp = SourceObject;
         return feedForward(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(ComputeDll) != POINTER_INVALID)
     {
      if(!ComputeDll.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
                                NeuronOCL.Neurons(), NativeActivationCode(activation)))
        {
         Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " FeedForward failed, error " + IntegerToString(ComputeDll.LastError()));
         return false;
        }
      return Output.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Output.Total();
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel FeedForward: %d", GetLastError());
      return false;
     }
   //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
   //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
   //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
   //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
   //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
   return true;
  }
//+------------------------------------------------------------------+
//| Pure-MQL5 dense forward pass - exact double-precision mirror of   |
//| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, |
//| which (Gizlyk convention) owns both the inputs (its Output) and   |
//| the weights connecting them to THIS layer (its Weights), laid out |
//| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host     |
//| buffers only; used when no compute backend exists (CPU inference).|
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return false;
   int inputs   = NeuronOCL.Neurons();
   int outCount = Output.Total();
   int wTotal   = NeuronOCL.WeightsCount();
   if(wTotal < (inputs + 1) * outCount)
      return false; // weight buffer smaller than the dense layout requires - refuse rather than misread
   for(int i = 0; i < outCount; i++)
     {
      int shift = (inputs + 1) * i;
      double sum = 0.0;
      for(int k = 0; k < inputs; k++)
         sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k);
      sum += NeuronOCL.WeightHost(shift + inputs); // bias
      switch(activation)
        {
         case TANH:
            sum = tanh(sum);
            break;
         case SIGMOID:
            sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum))));
            break;
         case PRELU:
            if(sum < 0.0)
               sum *= 0.01;
            break;
         // NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case.
        }
      if(!Output.Update(i, sum))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Host-side input write for the CPU inference path's layer 0 (there |
//| is no device buffer to write into, unlike CNet::feedForward's     |
//| BufferWrite branch). Copies inputVals into the Output host array. |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals)
  {
   if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return false;
   int total = MathMin(Output.Total(), inputVals.Total());
   for(int i = 0; i < total; i++)
      if(!Output.Update(i, inputVals.At(i)))
         return false;
   return true;
  }
//+------------------------------------------------------------------+
//| Host-side output read for getResults() on the CPU inference path  |
//| (no device BufferRead available). Returns the count copied.       |
//+------------------------------------------------------------------+
int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values)
  {
   if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return 0;
   values.Clear();
   int n = Output.Total();
   for(int i = 0; i < n; i++)
      if(!values.Add(Output.At(i)))
         return i;
   return n;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(ComputeDll) != POINTER_INVALID)
     {
      //--- count is exactly Neurons() - biases receive no backprop gradient, and the DLL needs the
      //--- true neuron count to derive the weight-row stride (inputs + 1). The old Neurons() + 1
      //--- "bias row" only ever produced out-of-bounds reads (see the 2026-08-11 transpose fix in
      //--- the kernels: Network.cl CaclHiddenGradient, WarriorCPU.cpp CalcHiddenGradient).
      if(!ComputeDll.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
                                       NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons()))
        {
         Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " CalcHiddenGradient failed, error " + IntegerToString(ComputeDll.LastError()));
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   //--- Exactly Neurons(): the kernel derives the weight-row stride from get_global_size(0), and
   //--- biases receive no backprop gradient (the old +1 work-item only ever read past matrix_o).
   global_work_size[0] = Neurons();
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
      return false;
     }
   //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
   //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
  {
   if(CheckPointer(Target) == POINTER_INVALID)
      return false;
   int count = Target.Total();
   for(int i = 0; i < count; i++)
      if(!Gradient.Update(i, Target.At(i)))
         return false;
   Gradient.BufferWrite();
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
   if(CheckPointer(ComputeDll) != POINTER_INVALID)
     {
      if(!ComputeDll.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
        {
         Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " CalcOutputGradient failed, error " + IntegerToString(ComputeDll.LastError()));
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = count;
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
   ResetLastError();
   if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
      return false;
     }
   //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
   //--- BufferRead()s itself - see the note in feedForward() above.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(ComputeDll) != POINTER_INVALID)
     {
      int inputs = NeuronOCL.Neurons();
      int neurons = Neurons();
      if(optimization == SGD)
        {
         if(!ComputeDll.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                             NeuronOCL.getDeltaWeightsIndex(), inputs, g_eta, alpha, neurons,
                                             0))
           {
            Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " UpdateWeightsMomentum failed, error " + IntegerToString(ComputeDll.LastError()));
            return false;
           }
        }
      else
        {
         double lt = g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
         if(!ComputeDll.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                         NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
                                         inputs, lt, AdamBeta1, AdamBeta2, neurons))
           {
            Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " UpdateWeightsAdam failed, error " + IntegerToString(ComputeDll.LastError()));
            return false;
           }
         t++;
        }
      //--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex()
      //--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand.
      return true;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Neurons();
   global_work_size[1] = NeuronOCL.Neurons();
   if(optimization == SGD)
     {
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)g_eta);
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
         return false;
        }
     }
   else
     {
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
         return false;
      double lt = g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)AdamBeta1))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)AdamBeta2))
         return false;
      //--- Dim 1 covers the full (inputs + 1)-wide weight row in groups of 4, so the bias column is
      //--- dispatched too. Sizing this on Neurons() (without the +1) left the bias group unreachable
      //--- whenever inputs % 4 == 0 - see the kernel's 2026-08-11 comment.
      global_work_size[1] = NeuronOCL.Neurons() + 1;
      uint rest = global_work_size[1] % 4;
      global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
         return false;
        }
      t++;
     }
   //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
   //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
   return true;
  }
//+------------------------------------------------------------------+
//| MINI-BATCH ACCUMULATE (dense) - see the declaration comment.      |
//| Same dispatch shape as updateInputWeights above, but it only ADDS |
//| this sample's outer product into GradAccum; no weight is touched  |
//| and no optimizer state advances until ApplyAccumulatedGradients.  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   //--- The weight block being accumulated for lives on the SOURCE neuron, exactly as
   //--- updateInputWeights' matrix_w does - so the accumulator is sized and owned there too.
   if(!NeuronOCL.EnsureGradAccum(NeuronOCL.Weights))
      return false;
   int inputs = NeuronOCL.Neurons();
   int neurons = Neurons();
   if(CheckPointer(ComputeDll) != POINTER_INVALID)
     {
      if(!ComputeDll.AccumulateWeightGrad(NeuronOCL.GradAccum.GetIndex(), getGradientIndex(),
                                        NeuronOCL.getOutputIndex(), inputs, neurons))
        {
         Print(__FUNCTION__ + ": " + ComputeDll.BackendName() + " AccumulateWeightGrad failed, error " + IntegerToString(ComputeDll.LastError()));
         return false;
        }
      return true;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = neurons;
   global_work_size[1] = inputs + 1;   // the last slot is the bias, exactly as the Adam kernel treats it
   if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_acc, NeuronOCL.GradAccum.GetIndex()))
      return false;
   if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_g, getGradientIndex()))
      return false;
   if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_i, NeuronOCL.getOutputIndex()))
      return false;
   if(!OpenCL.SetArgument(def_k_AccumulateWeightGrad, def_k_awg_inputs, inputs))
      return false;
   ResetLastError();
   if(!OpenCL.Execute(def_k_AccumulateWeightGrad, 2, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel AccumulateWeightGrad: %d", GetLastError());
      return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Type-dispatching wrapper, mirroring updateInputWeights(CObject*). |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::accumulateInputWeightGrads(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
   //--- Every *OCL neuron type derives from CNeuronBaseOCL and stores its incoming dense matrix in
   //--- Weights, so one branch covers them all. Conv and LSTM own EXTRA weight blocks and override
   //--- this method; the legacy scalar hierarchy never reaches here (CNet refuses to batch without a
   //--- backend - see CNet::BatchSize).
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronBatchNormOCL:
      case defNeuronConvOCL:
      case defNeuronPoolOCL:
      case defNeuronLSTMOCL:
        {
         CNeuronBaseOCL *temp = SourceObject;
         return accumulateInputWeightGrads(temp);
        }
     }
   return false;
  }
//+------------------------------------------------------------------+
//| Zero this neuron's accumulator - see the declaration comment.     |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::BeginGradAccum(void)
  {
   if(CheckPointer(GradAccum) == POINTER_INVALID || GradAccum.Total() <= 0)
      return true;  // nothing accumulated here yet; the first accumulate allocates it zeroed
   return ZeroOptimizerBuffer(GradAccum);
  }
//+------------------------------------------------------------------+
//| DEVICE-SIDE mini-batch apply (OpenCL only). One dispatch,        |
//| nothing crosses the bus.                                         |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
                                        CBufferDouble *v, CBufferDouble *dw, double scale, int total)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID || total <= 0)
      return false;
   uint offset[1] = {0};
   uint size[1];
   size[0] = (uint)total;
   if(optimization == ADAM)
     {
      if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID ||
         m.Total() < total || v.Total() < total || m.GetIndex() < 0 || v.GetIndex() < 0)
         return false;
      //--- Same bias-corrected step the host path and the unbatched kernel compute. t is NOT advanced
      //--- here - ApplyAccumulatedGradients owns that, once per batch across every block it holds.
      double lt = g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
      if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_w, w.GetIndex()) ||
         !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_acc, acc.GetIndex()) ||
         !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_m, m.GetIndex()) ||
         !OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_v, v.GetIndex()) ||
         !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_scale, (float)scale) ||
         !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_l, (float)lt) ||
         !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b1, (float)AdamBeta1) ||
         !OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b2, (float)AdamBeta2))
         return false;
      ResetLastError();
      return OpenCL.Execute(def_k_ApplyAccumAdam, 1, offset, size);
     }
   if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total || dw.GetIndex() < 0)
      return false;
   if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_w, w.GetIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_acc, acc.GetIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_dw, dw.GetIndex()) ||
      !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_scale, (float)scale) ||
      !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_lr, (float)g_eta) ||
      !OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_momentum, (float)alpha))
      return false;
   ResetLastError();
   return OpenCL.Execute(def_k_ApplyAccumMomentum, 1, offset, size);
  }
//+------------------------------------------------------------------+
//| MINI-BATCH APPLY (dense) - ONE optimizer step on the batch MEAN. |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
                                       CBufferDouble *v, CBufferDouble *dw, double scale)
  {
   if(CheckPointer(acc) == POINTER_INVALID || acc.Total() <= 0 ||
      CheckPointer(w) == POINTER_INVALID || w.Total() <= 0)
      return true;   // no weight block here (e.g. an output-layer neuron) - nothing to apply
   int total = MathMin(acc.Total(), w.Total());
   //--- DEVICE FAST PATH. Latched off on first failure rather than retried every batch: a kernel that
   //--- did not build will not build later, and the host path below is a correct answer, so the right
   //--- behaviour is one warning and full speed on the fallback - not a failed dispatch per batch.
   if(g_applyAccumKernelUsable && CheckPointer(OpenCL) != POINTER_INVALID && w.GetIndex() >= 0 &&
      acc.GetIndex() >= 0)
     {
      if(ApplyAccumOnDevice(w, acc, m, v, dw, scale, total))
         return true;
      g_applyAccumKernelUsable = false;
      PrintFormat("%s: device-side mini-batch apply failed (error %d) - falling back to the host step "
                  "for the rest of this run. Training stays correct; each batch now costs a full "
                  "weight-matrix round trip, so expect it to be substantially slower.",
                  __FUNCTION__, GetLastError());
     }
   //--- Pull the device-side copies into the host mirrors. On a host-only net (no backend) there is no
   //--- device to read FROM and the host mirror is already the truth, so the round-trip is skipped
   //--- rather than treated as a failure.
   bool haveDevice = (w.GetIndex() >= 0);
   if(haveDevice && (!w.BufferRead() || !acc.BufferRead()))
      return false;
   if(optimization == ADAM)
     {
      if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID ||
         m.Total() < total || v.Total() < total)
         return false;
      if(haveDevice && (!m.BufferRead() || !v.BufferRead()))
         return false;
      //--- Bias correction reads t but does NOT advance it here - the caller advances once per batch
      //--- after every block it owns has been stepped, because t counts optimizer STEPS and a batch is
      //--- one step no matter how many weight blocks the neuron carries.
      double lt = g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
      for(int i = 0; i < total; i++)
        {
         double grad = acc.At(i) * scale;
         double wv = w.At(i);
         double mt = AdamBeta1 * m.At(i) + (1 - AdamBeta1) * grad;
         //--- v is STORED square-rooted, so square it back before the recursion. See Network.cl's
         //--- UpdateWeightsAdam for why feeding the stored sqrt straight in is not Adam at all.
         double vt = sqrt(AdamBeta2 * v.At(i) * v.At(i) + (1 - AdamBeta2) * grad * grad);
         double delta = lt * mt / (vt > 0 ? vt : lt * 10) - lt * WEIGHT_DECAY * wv;
         delta = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, delta));
         if(!w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, wv + delta))) ||
            !m.Update(i, mt) || !v.Update(i, vt))
            return false;
        }
      if(haveDevice && (!m.BufferWrite() || !v.BufferWrite()))
         return false;
     }
   else
     {
      if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total)
         return false;
      if(haveDevice && !dw.BufferRead())
         return false;
      for(int i = 0; i < total; i++)
        {
         double grad = acc.At(i) * scale;
         double delta = g_eta * grad + alpha * dw.At(i);
         if(!dw.Update(i, delta) ||
            !w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, w.At(i) + delta))))
            return false;
        }
      if(haveDevice && !dw.BufferWrite())
         return false;
     }
   //--- Clear the accumulator for the next batch and push the new weights back to the device.
   for(int i = 0; i < acc.Total(); i++)
      if(!acc.Update(i, 0.0))
         return false;
   if(haveDevice && (!w.BufferWrite() || !acc.BufferWrite()))
      return false;
   return true;
  }
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::ApplyAccumulatedGradients(double scale)
  {
   if(!ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale))
      return false;
   if(optimization == ADAM)
      t++;
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
  {
   if(CheckPointer(TargetObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   CNeuronConvOCL *tempConv = NULL;
   CNeuronLSTMOCL *tempLstm = NULL;
   CNeuronPoolOCL *tempPool = NULL;
   CNeuronBatchNormOCL *tempBN = NULL;
   switch(TargetObject.Type())
     {
      case defNeuronBaseOCL:
         temp = TargetObject;
         return calcHiddenGradients(temp);
         break;
      case defNeuronBatchNormOCL:
         //--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step
         //--- and writes into this->Gradient. Routing it through the dense branch instead would run
         //--- CaclHiddenGradient against a weight matrix batch norm does not have.
         tempBN = TargetObject;
         return tempBN.calcInputGradients(GetPointer(this));
         break;
      case defNeuronConvOCL:
         //--- Conv owns the backward step (calcInputGradients), called on itself with
         //--- "this" (the earlier layer) passed in so it writes into this->Gradient.
         tempConv = TargetObject;
         return tempConv.calcInputGradients(GetPointer(this));
         break;
      case defNeuronLSTMOCL:
         tempLstm = TargetObject;
         return tempLstm.calcInputGradients(GetPointer(this));
         break;
      case defNeuronPoolOCL:
         tempPool = TargetObject;
         return tempPool.calcInputGradients(GetPointer(this));
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
      case defNeuronBatchNormOCL:
         temp = SourceObject;
         return updateInputWeights(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Save(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
   if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
      return false;
   if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
      return false;
   if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
      return false;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      FileWriteInteger(file_handle, 0);
      return true;
     }
   else
      FileWriteInteger(file_handle, 1);
//---
   if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
      return false;
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
         return false;
      if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                   |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Load(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
//---
   activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
   optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
   t = FileReadInteger(file_handle, INT_VALUE);
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(Output.GetIndex() >= 0)
      Output.BufferFree();
   if(!Output.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Output))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(PrevOutput.GetIndex() >= 0)
      PrevOutput.BufferFree();
   if(!PrevOutput.Load(file_handle))
      return false;
   if(!BackendBufferCreate(PrevOutput))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(Gradient.GetIndex() >= 0)
      Gradient.BufferFree();
   if(!Gradient.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Gradient))
      return false;
//---
   if(FileReadInteger(file_handle) == 0)
      return true;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      Weights = new CBufferDouble();
      if(CheckPointer(Weights) == POINTER_INVALID)
         return false;
     }
   if(Weights.GetIndex() >= 0)
      Weights.BufferFree();
   if(!Weights.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Weights))
      return false;
//---
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID)
        {
         DeltaWeights = new CBufferDouble();
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
            return false;
        }
      if(DeltaWeights.GetIndex() >= 0)
         DeltaWeights.BufferFree();
      if(!DeltaWeights.Load(file_handle))
         return false;
      if(!BackendBufferCreate(DeltaWeights))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID)
        {
         FirstMomentum = new CBufferDouble();
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
            return false;
        }
      if(FirstMomentum.GetIndex() >= 0)
         FirstMomentum.BufferFree();
      if(!FirstMomentum.Load(file_handle))
         return false;
      if(!BackendBufferCreate(FirstMomentum))
         return false;
      //---
      if(CheckPointer(SecondMomentum) == POINTER_INVALID)
        {
         SecondMomentum = new CBufferDouble();
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
            return false;
        }
      if(SecondMomentum.GetIndex() >= 0)
         SecondMomentum.BufferFree();
      if(!SecondMomentum.Load(file_handle))
         return false;
      if(!BackendBufferCreate(SecondMomentum))
         return false;
     }
//---
   return true;
  }
#endif