Warrior_EA/AI/Impl/NeuronBatchNorm.mqh

//+------------------------------------------------------------------+
//| NeuronBatchNorm.mqh                                               |
//|                                                                   |
//| CNeuronBatchNormOCL bodies - batch normalization (Ioffe & Szegedy |
//| 2015).                                                            |
//|                                                                   |
//| Included from AI\Network.mqh AFTER every class declaration -      |
//| bodies only, no declarations. Relocation is behaviour-neutral by  |
//| construction: nothing here is reachable until Network.mqh ends.   |
//+------------------------------------------------------------------+
#ifndef WARRIOR_AI_IMPL_NEURONBATCHNORM_MQH
#define WARRIOR_AI_IMPL_NEURONBATCHNORM_MQH
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBatchNormOCL::CNeuronBatchNormOCL(void) : iBatchSize(1), bStatsFrozen(false), iSamplesSeen(0),
   m_fwdInputCached(false), m_gradCached(false), m_bnDeviceAuthoritative(false),
   m_bnCheckedFwd(false), m_bnCheckedGrad(false), m_bnCheckedAccum(false), m_bnCheckedApply(false)
  {
   BatchOptions = NULL;
   m_bnAcc = NULL;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBatchNormOCL::~CNeuronBatchNormOCL(void)
  {
   if(CheckPointer(BatchOptions) != POINTER_INVALID)
      delete BatchOptions;
   if(CheckPointer(m_bnAcc) != POINTER_INVALID)
      delete m_bnAcc;
  }
//+------------------------------------------------------------------+
//| gamma=1 / beta=0 / zeroed statistics for every unit.              |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::InitOptions(int neurons)
  {
   if(neurons <= 0)
      return false;
   if(CheckPointer(BatchOptions) == POINTER_INVALID)
     {
      BatchOptions = new CBufferDouble();
      if(CheckPointer(BatchOptions) == POINTER_INVALID)
         return false;
     }
   BatchOptions.Clear();
   if(!BatchOptions.Reserve(neurons * BN_OPT_STRIDE))
      return false;
   for(int n = 0; n < neurons; n++)
      for(int s = 0; s < BN_OPT_STRIDE; s++)
         if(!BatchOptions.Add(s == BN_OPT_GAMMA ? 1.0 : 0.0))
            return false;
   iSamplesSeen = 0;
   //--- Full host overwrite: if a device copy exists, push it so the two cannot disagree. The host is
   //--- authoritative after a re-init by definition.
   if(BatchOptions.GetIndex() >= 0)
     {
      if(!BatchOptions.BufferWrite())
         return false;
      m_bnDeviceAuthoritative = false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type))
      return false;
   //--- Identity forward transform. The activation belongs to the layer AFTER this one; normalizing
   //--- and then squashing in the same step would defeat the point (Ioffe & Szegedy place the
   //--- normalization immediately BEFORE the non-linearity, not around it).
   activation = NONE;
   iBatchSize = (int)batchSize;
   return InitOptions((int)numNeurons);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, compute_dll, numNeurons, optimization_type))
      return false;
   activation = NONE;
   iBatchSize = (int)batchSize;
   return InitOptions((int)numNeurons);
  }
//+------------------------------------------------------------------+
//| The forward transform, host-side. Mirrors the NeuroNet_DNG        |
//| BatchFeedForward kernel, plus the bias-corrected warm-up window.  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::NormalizeHost(const double &inputs[], int count)
  {
   if(CheckPointer(Output) == POINTER_INVALID || CheckPointer(BatchOptions) == POINTER_INVALID)
      return false;
   if(count > Output.Total())
      count = Output.Total();
   if(BatchOptions.Total() < count * BN_OPT_STRIDE)
      return false;
   //--- A window of 1 makes mean==x and variance==0 for every sample, i.e. a constant-zero output.
   //--- Treat it as "normalization off" and pass the input through untouched rather than emit zeros.
   if(iBatchSize <= 1)
     {
      for(int i = 0; i < count; i++)
         if(!Output.Update(i, inputs[i]))
            return false;
      return true;
     }
   if(!bStatsFrozen && iSamplesSeen < iBatchSize)
      iSamplesSeen++;
   //--- Effective window: ramps 1,2,3... up to iBatchSize, so the first samples produce an honest
   //--- running mean instead of one biased toward the zero initializer.
   double w = (double)MathMax(1, iSamplesSeen);
   for(int i = 0; i < count; i++)
     {
      int shift = i * BN_OPT_STRIDE;
      double x = inputs[i];
      //--- The running mean/variance below are PERSISTENT: they live in BatchOptions, are carried
      //--- into the .nnw by getWeightsBN, and every later sample normalizes against them. That
      //--- makes them a LATCH.
      if(!MathIsValidNumber(x))
         x = 0.0;
      x = MathMax(-BN_MAX_INPUT, MathMin(BN_MAX_INPUT, x));
      double mean = BatchOptions.At(shift + BN_OPT_MEAN);
      double variance = BatchOptions.At(shift + BN_OPT_VAR);
      //--- Self-heal statistics that were already poisoned before this guard existed. Without it an
      //--- affected .nnw stays dead across restarts, because Load faithfully restores the NaN.
      if(!MathIsValidNumber(mean))
         mean = x;
      if(!MathIsValidNumber(variance) || variance < 0.0)
         variance = 0.0;
      if(!bStatsFrozen)
        {
         mean = (mean * (w - 1.0) + x) / w;
         variance = (variance * (w - 1.0) + (x - mean) * (x - mean)) / w;
        }
      double delta = x - mean;
      double sd = MathMax(MathSqrt(variance + BN_EPSILON), BN_MIN_STD);
      //--- Bounded - see BN_MAX_NX. The clamped value is what gets CACHED below, which is what makes
      //--- the backward pass able to see that it bound (HiddenGradHost reads BN_OPT_NX).
      double nx = MathMax(-BN_MAX_NX, MathMin(BN_MAX_NX, delta / sd));
      //--- gamma/beta cannot go non-finite going forward (updateInputWeights validates and clamps every
      //--- step), but a model SAVED before that guard existed can carry NaN in here through Load. Same
      //--- self-heal as the statistics above: fall back to the identity transform for that unit.
      double gamma = BatchOptions.At(shift + BN_OPT_GAMMA);
      double beta = BatchOptions.At(shift + BN_OPT_BETA);
      if(!MathIsValidNumber(gamma))
         gamma = 1.0;
      if(!MathIsValidNumber(beta))
         beta = 0.0;
      double y = gamma * nx + beta;
      //--- nx is still cached when frozen: it costs nothing and keeps the buffer consistent with the
      //--- output just produced. There is no backward pass while frozen, so nothing reads it.
      if(!bStatsFrozen &&
         (!BatchOptions.Update(shift + BN_OPT_MEAN, mean) ||
          !BatchOptions.Update(shift + BN_OPT_VAR, variance)))
         return false;
      if(!BatchOptions.Update(shift + BN_OPT_NX, nx))
         return false;
      if(!Output.Update(i, y))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Forward. On OpenCL this is a KERNEL since 2026-08-09 (verified    |
//| against NormalizeHost on first use, see SelfCheckBnForward); on   |
//| every other backend it pulls the previous layer's output to the   |
//| host, runs the transform, and pushes its output back.             |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   //--- A new sample starts here, so last sample's gradient is no longer valid to reuse. Disarmed
   //--- BEFORE anything can fail below, so an aborted forward pass cannot leave a consumer holding a
   //--- cache that looks current - see the declaration comment.
   m_gradCached = false;
   m_fwdInputCached = false;
   //--- DEVICE PATH: nothing crosses the bus at all. The self-check runs the first time through and
   //--- proves the kernel against NormalizeHost before the device is trusted with the persisted
   //--- statistics; after that it is one dispatch per sample.
   if(BnDeviceEligible() && NeuronOCL.getOutputIndex() >= 0 && getOutputIndex() >= 0 &&
      EnsureBnDeviceBuffers())
     {
      if(!m_bnCheckedFwd)
         return SelfCheckBnForward(NeuronOCL);
      //--- Ramp bookkeeping stays host-side with iSamplesSeen; committed only on success so a failed
      //--- dispatch that falls through to NormalizeHost (which increments itself) cannot double-count.
      int seen = iSamplesSeen;
      if(!bStatsFrozen && seen < iBatchSize)
         seen++;
      if(DispatchBnForward(NeuronOCL, (double)MathMax(1, seen)))
        {
         iSamplesSeen = seen;
         return true;
        }
      LatchBnKernelsOff("BatchNormForward dispatch failed (error " +
                        IntegerToString(GetLastError()) + ")");
     }
   //--- HOST PATH (every non-OpenCL tier, and OpenCL after a latch). If the device had been
   //--- authoritative, pull its state first - NormalizeHost below must advance the REAL statistics,
   //--- not a stale mirror.
   EnsureHostAuthoritative();
   double inputs[];
   int count = NeuronOCL.getOutputVal(inputs);
   if(count <= 0)
      return false;
   if(!NormalizeHost(inputs, count))
      return false;
   //--- Armed for calcInputGradients, which needs exactly these values and would otherwise read the
   //--- same buffer back a second time this sample.
   if(ArrayCopy(m_fwdInputCache, inputs, 0, 0, count) == count)
      m_fwdInputCached = true;
   //--- Unlike the dense/conv kernels this layer's Output does NOT stay device-resident by itself -
   //--- it was just written host-side, so it has to be pushed before the next layer's kernel reads
   //--- it through getOutputIndex().
   return Output.BufferWrite();
  }
//+------------------------------------------------------------------+
//| Pure-MQL5 inference path (no backend at all): the previous        |
//| layer's values live only in its host mirror, so read them there.  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   int count = NeuronOCL.Neurons();
   if(count <= 0)
      return false;
   double inputs[];
   if(ArrayResize(inputs, count) != count)
      return false;
   for(int i = 0; i < count; i++)
      inputs[i] = NeuronOCL.OutputHost(i);
   //--- No BufferWrite: there is no device buffer on this path, and Output's host mirror is what
   //--- the next layer's feedForwardCPU (and GetOutputsCPU) read.
   return NormalizeHost(inputs, count);
  }
//+------------------------------------------------------------------+
//| DEVICE PATH plumbing - see the m_bnAcc declaration comment for    |
//| the design. "Device" means EITHER backend since 2026-08-25: the   |
//| OpenCL kernels where a device exists, the WarriorCPU.dll mirrors  |
//| on the CPU tier. Before that the DLL tier had no BN kernels at    |
//| all, so every sample crossed the bus twice per BN layer and       |
//| normalized in interpreted MQL5 - the one stage of the network     |
//| that still ran host-side per sample.                              |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::BnDeviceEligible(void)
  {
   return g_bnKernelUsable && iBatchSize > 1 &&
          (CheckPointer(OpenCL) != POINTER_INVALID || CheckPointer(ComputeDll) != POINTER_INVALID) &&
          CheckPointer(BatchOptions) != POINTER_INVALID;
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::EnsureBnDeviceBuffers(void)
  {
   //--- BufferCreate pushes the current host contents, so a freshly-created device copy is exactly
   //--- the state the host was authoritative over - the invariant every self-check relies on.
   bool haveOcl = (CheckPointer(OpenCL) != POINTER_INVALID);
   if(BatchOptions.GetIndex() < 0 &&
      !(haveOcl ? BatchOptions.BufferCreate(OpenCL) : BatchOptions.BufferCreate(ComputeDll)))
      return false;
   if(CheckPointer(m_bnAcc) == POINTER_INVALID)
     {
      m_bnAcc = new CBufferDouble();
      if(CheckPointer(m_bnAcc) == POINTER_INVALID)
         return false;
      if(!m_bnAcc.BufferInit(Neurons() * 2, 0.0))
         return false;
     }
   if(m_bnAcc.GetIndex() < 0 &&
      !(haveOcl ? m_bnAcc.BufferCreate(OpenCL) : m_bnAcc.BufferCreate(ComputeDll)))
      return false;
   return true;
  }
//+------------------------------------------------------------------+
void CNeuronBatchNormOCL::SyncOptionsToHost(void)
  {
   //--- Read-only pull for checkpoints/saves taken mid-training; the device REMAINS authoritative.
   //--- Guarded by the flag, because an unguarded BufferRead on a host-authoritative layer would
   //--- clobber good host state with whatever stale bytes the device still holds.
   if(m_bnDeviceAuthoritative && CheckPointer(BatchOptions) != POINTER_INVALID &&
      BatchOptions.GetIndex() >= 0)
      BatchOptions.BufferRead();
  }
//+------------------------------------------------------------------+
void CNeuronBatchNormOCL::EnsureHostAuthoritative(void)
  {
   if(!m_bnDeviceAuthoritative)
      return;
   if(CheckPointer(BatchOptions) != POINTER_INVALID && BatchOptions.GetIndex() >= 0)
      BatchOptions.BufferRead();
   //--- Drain the device-side batch accumulator into the host arrays, so a handover in the MIDDLE of
   //--- a batch keeps the samples the kernels already accumulated - without this, latching off after
   //--- sample 3 of 8 would silently drop three samples' gradients from the batch.
   if(CheckPointer(m_bnAcc) != POINTER_INVALID && m_bnAcc.GetIndex() >= 0 && m_bnAcc.BufferRead())
     {
      int units = Neurons();
      if(ArraySize(m_accGamma) != units || ArraySize(m_accBeta) != units)
        {
         ArrayResize(m_accGamma, units);
         ArrayResize(m_accBeta, units);
         ArrayInitialize(m_accGamma, 0.0);
         ArrayInitialize(m_accBeta, 0.0);
        }
      for(int i = 0; i < units && 2 * i + 1 < m_bnAcc.Total(); i++)
        {
         m_accGamma[i] += m_bnAcc.At(2 * i);
         m_accBeta[i]  += m_bnAcc.At(2 * i + 1);
        }
      ZeroOptimizerBuffer(m_bnAcc);
     }
   m_bnDeviceAuthoritative = false;
  }
//+------------------------------------------------------------------+
void CNeuronBatchNormOCL::LatchBnKernelsOff(const string reason)
  {
   if(g_bnKernelUsable)
     {
      g_bnKernelUsable = false;
      Print(__FUNCTION__ + ": BATCH-NORM KERNELS DISABLED for the rest of this run - " + reason +
            ". Every batch-norm layer falls back to the host implementation: results stay correct "
            "(the host path is the reference the kernels were transcribed from), each layer just pays "
            "its device round-trips again. State was resynced from the good copy before anything "
            "could persist.");
     }
   EnsureHostAuthoritative();
  }
//+------------------------------------------------------------------+
double CNeuronBatchNormOCL::BnDiffScore(double ref, double got)
  {
   bool fRef = MathIsValidNumber(ref), fGot = MathIsValidNumber(got);
   if(!fRef || !fGot)
      return (fRef == fGot) ? 0.0 : DBL_MAX; // both poisoned the same way is agreement
   return MathAbs(got - ref) / (1.0e-3 * MathMax(1.0, MathAbs(ref)));
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::DispatchBnForward(CNeuronBaseOCL *NeuronOCL, double w)
  {
   //--- CPU-DLL tier: same kernel semantics, double precision (matches the host reference EXACTLY,
   //--- unlike the float OpenCL kernels the self-check tolerance was sized for).
   if(CheckPointer(OpenCL) == POINTER_INVALID)
     {
      if(CheckPointer(ComputeDll) == POINTER_INVALID)
         return false;
      if(!ComputeDll.BatchNormForward(NeuronOCL.getOutputIndex(), getOutputIndex(),
                                      BatchOptions.GetIndex(), w, bStatsFrozen ? 1 : 0, Neurons()))
         return false;
      m_bnDeviceAuthoritative = true;
      return true;
     }
   uint offset[1] = {0};
   uint size[1];
   size[0] = (uint)Neurons();
   if(!OpenCL.SetArgumentBuffer(def_k_BatchNormForward, def_k_bnf_matrix_i, NeuronOCL.getOutputIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormForward, def_k_bnf_matrix_o, getOutputIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormForward, def_k_bnf_options, BatchOptions.GetIndex()) ||
      !OpenCL.SetArgument(def_k_BatchNormForward, def_k_bnf_w, (float)w) ||
      !OpenCL.SetArgument(def_k_BatchNormForward, def_k_bnf_frozen, bStatsFrozen ? 1 : 0))
      return false;
   ResetLastError();
   if(!OpenCL.Execute(def_k_BatchNormForward, 1, offset, size))
      return false;
   //--- The kernel just wrote the running statistics on the device: it is now the authority.
   m_bnDeviceAuthoritative = true;
   return true;
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::DispatchBnHiddenGrad(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID)
     {
      if(CheckPointer(ComputeDll) == POINTER_INVALID)
         return false;
      return ComputeDll.BatchNormHiddenGrad(getGradientIndex(), NeuronOCL.getOutputIndex(),
                                            NeuronOCL.getGradientIndex(), BatchOptions.GetIndex(),
                                            NativeActivationCode(NeuronOCL.Activation()), Neurons());
     }
   uint offset[1] = {0};
   uint size[1];
   size[0] = (uint)Neurons();
   if(!OpenCL.SetArgumentBuffer(def_k_BatchNormHiddenGrad, def_k_bnh_matrix_g, getGradientIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormHiddenGrad, def_k_bnh_prev_o, NeuronOCL.getOutputIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormHiddenGrad, def_k_bnh_prev_g, NeuronOCL.getGradientIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormHiddenGrad, def_k_bnh_options, BatchOptions.GetIndex()) ||
      !OpenCL.SetArgument(def_k_BatchNormHiddenGrad, def_k_bnh_activation, NativeActivationCode(NeuronOCL.Activation())))
      return false;
   ResetLastError();
   return OpenCL.Execute(def_k_BatchNormHiddenGrad, 1, offset, size);
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::DispatchBnAccum(void)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID)
     {
      if(CheckPointer(ComputeDll) == POINTER_INVALID)
         return false;
      return ComputeDll.BatchNormAccumGammaBeta(getGradientIndex(), BatchOptions.GetIndex(),
                                                m_bnAcc.GetIndex(), Neurons());
     }
   uint offset[1] = {0};
   uint size[1];
   size[0] = (uint)Neurons();
   if(!OpenCL.SetArgumentBuffer(def_k_BatchNormAccumGammaBeta, def_k_bna_matrix_g, getGradientIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormAccumGammaBeta, def_k_bna_options, BatchOptions.GetIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormAccumGammaBeta, def_k_bna_acc, m_bnAcc.GetIndex()))
      return false;
   ResetLastError();
   return OpenCL.Execute(def_k_BatchNormAccumGammaBeta, 1, offset, size);
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::DispatchBnApply(double scale, double lt)
  {
   if(CheckPointer(OpenCL) == POINTER_INVALID)
     {
      if(CheckPointer(ComputeDll) == POINTER_INVALID)
         return false;
      if(!ComputeDll.BatchNormApplyGammaBeta(BatchOptions.GetIndex(), m_bnAcc.GetIndex(),
                                             scale, lt, AdamBeta1, AdamBeta2, g_eta, alpha,
                                             (optimization == SGD) ? 0 : 1, Neurons()))
         return false;
      m_bnDeviceAuthoritative = true;
      return true;
     }
   uint offset[1] = {0};
   uint size[1];
   size[0] = (uint)Neurons();
   if(!OpenCL.SetArgumentBuffer(def_k_BatchNormApplyGammaBeta, def_k_bnp_options, BatchOptions.GetIndex()) ||
      !OpenCL.SetArgumentBuffer(def_k_BatchNormApplyGammaBeta, def_k_bnp_acc, m_bnAcc.GetIndex()) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_scale, (float)scale) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_lt, (float)lt) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_b1, (float)AdamBeta1) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_b2, (float)AdamBeta2) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_lr, (float)g_eta) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_momentum, (float)alpha) ||
      !OpenCL.SetArgument(def_k_BatchNormApplyGammaBeta, def_k_bnp_optimizer, (optimization == SGD) ? 0 : 1))
      return false;
   ResetLastError();
   if(!OpenCL.Execute(def_k_BatchNormApplyGammaBeta, 1, offset, size))
      return false;
   m_bnDeviceAuthoritative = true;
   return true;
  }
//+------------------------------------------------------------------+
//| SELF-CHECK: forward. Runs the HOST transform first (on the same   |
//| pre-state the device holds), dispatches the kernel, then compares |
//| output + statistics elementwise. Pass -> the device is verified   |
//| and becomes authoritative. Fail -> the host result is restored to |
//| both copies and the kernels latch off. Cost: one extra input read |
//| and one output+options read, ONCE per layer per process.          |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::SelfCheckBnForward(CNeuronBaseOCL *NeuronOCL)
  {
   int units = Neurons();
   double inputs[];
   int count = NeuronOCL.getOutputVal(inputs);
   if(count <= 0)
      return false;
   //--- Host reference. NormalizeHost mutates the host mirrors (Output, BatchOptions, iSamplesSeen);
   //--- the device still holds the untouched pre-state, which is exactly what the kernel must see.
   if(!NormalizeHost(inputs, count))
      return false;
   int oTotal = BatchOptions.Total();
   double refY[], refOpt[];
   ArrayResize(refY, units);
   ArrayResize(refOpt, oTotal);
   for(int i = 0; i < units; i++)
      refY[i] = Output.At(i);
   for(int i = 0; i < oTotal; i++)
      refOpt[i] = BatchOptions.At(i);
   //--- Kernel on the same sample. NormalizeHost already advanced iSamplesSeen, so the ramped window
   //--- it used is exactly MathMax(1, iSamplesSeen) now.
   bool ok = DispatchBnForward(NeuronOCL, (double)MathMax(1, iSamplesSeen));
   if(ok)
      ok = Output.BufferRead() && BatchOptions.BufferRead();
   double worst = 0.0;
   int worstAt = -1;
   if(ok)
     {
      for(int i = 0; i < units; i++)
        {
         double s = BnDiffScore(refY[i], Output.At(i));
         if(s > worst) { worst = s; worstAt = i; }
        }
      for(int i = 0; i < oTotal; i++)
        {
         double s = BnDiffScore(refOpt[i], BatchOptions.At(i));
         if(s > worst) { worst = s; worstAt = units + i; }
        }
     }
   if(ok && worst <= 1.0)
     {
      m_bnCheckedFwd = true;
      m_bnDeviceAuthoritative = true;
      Print(__FUNCTION__ + StringFormat(": BatchNormForward kernel VERIFIED against the host math on "
                                        "%d units (worst normalized diff %.2e) - this layer's forward "
                                        "pass now runs device-side.", units, worst));
      return true;
     }
   //--- Kernel wrong or unreachable: the host result is the answer. Put it back in both copies.
   for(int i = 0; i < units; i++)
      Output.Update(i, refY[i]);
   for(int i = 0; i < oTotal; i++)
      BatchOptions.Update(i, refOpt[i]);
   if(BatchOptions.GetIndex() >= 0)
      BatchOptions.BufferWrite();
   m_bnDeviceAuthoritative = false;
   LatchBnKernelsOff(ok ? StringFormat("BatchNormForward disagrees with the host math (worst "
                                       "normalized diff %.2e at element %d)", worst, worstAt)
                        : "BatchNormForward dispatch/readback failed (error " +
                          IntegerToString(GetLastError()) + ")");
   return Output.BufferWrite();
  }
//+------------------------------------------------------------------+
//| SELF-CHECK: backward. Same pattern; the reference is              |
//| HiddenGradHost, which IS the host path's own loop.                |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::SelfCheckBnHiddenGrad(CNeuronBaseOCL *NeuronOCL)
  {
   int units = Neurons();
   SyncOptionsToHost(); // the host reference must see the statistics the kernels have been updating
   double grad[];
   if(getGradient(grad) < units)
      return false;
   double prevOut[];
   if(NeuronOCL.getOutputVal(prevOut) < units)
      return false;
   double ig[];
   if(ArrayResize(ig, units) != units)
      return false;
   HiddenGradHost(grad, prevOut, NeuronOCL.Activation(), ig, units);
   bool ok = DispatchBnHiddenGrad(NeuronOCL);
   double got[];
   if(ok)
      ok = (NeuronOCL.getGradient(got) >= units);
   double worst = 0.0;
   int worstAt = -1;
   if(ok)
      for(int i = 0; i < units; i++)
        {
         double s = BnDiffScore(ig[i], got[i]);
         if(s > worst) { worst = s; worstAt = i; }
        }
   if(ok && worst <= 1.0)
     {
      m_bnCheckedGrad = true;
      Print(__FUNCTION__ + StringFormat(": BatchNormHiddenGrad kernel VERIFIED against the host math "
                                        "on %d units (worst normalized diff %.2e).", units, worst));
      return true;
     }
   LatchBnKernelsOff(ok ? StringFormat("BatchNormHiddenGrad disagrees with the host math (worst "
                                       "normalized diff %.2e at unit %d)", worst, worstAt)
                        : "BatchNormHiddenGrad dispatch/readback failed (error " +
                          IntegerToString(GetLastError()) + ")");
   //--- The host result is the answer either way; setGradient writes host and device copies.
   return NeuronOCL.setGradient(ig);
  }
//+------------------------------------------------------------------+
//| SELF-CHECK: gamma/beta accumulate. First use ever, so the device  |
//| accumulator holds the zeros it was created with.                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::SelfCheckBnAccum(void)
  {
   int units = Neurons();
   SyncOptionsToHost(); // NX is device-fresh in kernel mode
   double grad[];
   if(getGradient(grad) < units)
      return false;
   double preAcc[];
   if(m_bnAcc.GetIndex() >= 0 && !m_bnAcc.BufferRead())
     {
      //--- Cannot even read the accumulator: latch (which drains whatever earlier samples the kernels
      //--- put there) before reporting the failure, so the batch is not silently truncated.
      LatchBnKernelsOff("BatchNormAccumGammaBeta pre-check read failed (error " +
                        IntegerToString(GetLastError()) + ")");
      return false;
     }
   ArrayResize(preAcc, m_bnAcc.Total());
   for(int i = 0; i < m_bnAcc.Total(); i++)
      preAcc[i] = m_bnAcc.At(i);
   bool ok = DispatchBnAccum();
   if(ok)
      ok = m_bnAcc.BufferRead();
   double worst = 0.0;
   int worstAt = -1;
   if(ok)
      for(int i = 0; i < units; i++)
        {
         double expG = preAcc[2 * i] + grad[i] * BatchOptions.At(i * BN_OPT_STRIDE + BN_OPT_NX);
         double expB = preAcc[2 * i + 1] + grad[i];
         double s = MathMax(BnDiffScore(expG, m_bnAcc.At(2 * i)), BnDiffScore(expB, m_bnAcc.At(2 * i + 1)));
         if(s > worst) { worst = s; worstAt = i; }
        }
   if(ok && worst <= 1.0)
     {
      m_bnCheckedAccum = true;
      Print(__FUNCTION__ + StringFormat(": BatchNormAccumGammaBeta kernel VERIFIED against the host "
                                        "math on %d units (worst normalized diff %.2e).", units, worst));
      return true;
     }
   //--- Restore the truth (pre-state plus this sample's host-computed contribution) into the device
   //--- accumulator BEFORE latching, so the drain inside the latch hands the host arrays exactly the
   //--- right sums.
   for(int i = 0; i < units; i++)
     {
      m_bnAcc.Update(2 * i, preAcc[2 * i] + grad[i] * BatchOptions.At(i * BN_OPT_STRIDE + BN_OPT_NX));
      m_bnAcc.Update(2 * i + 1, preAcc[2 * i + 1] + grad[i]);
     }
   if(m_bnAcc.GetIndex() >= 0)
      m_bnAcc.BufferWrite();
   LatchBnKernelsOff(ok ? StringFormat("BatchNormAccumGammaBeta disagrees with the host math (worst "
                                       "normalized diff %.2e at unit %d)", worst, worstAt)
                        : "BatchNormAccumGammaBeta dispatch/readback failed (error " +
                          IntegerToString(GetLastError()) + ")");
   return true;
  }
//+------------------------------------------------------------------+
//| SELF-CHECK: gamma/beta apply. The host reference is StepGammaBeta |
//| itself, run on the synced host mirror - literally the code the    |
//| host path executes, so the comparison cannot drift from it.       |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::SelfCheckBnApply(double scale, double lt)
  {
   int units = Neurons();
   SyncOptionsToHost();
   if(m_bnAcc.GetIndex() >= 0 && !m_bnAcc.BufferRead())
     {
      //--- Same reasoning as the accumulate pre-check: latch drains the device sums into the host
      //--- arrays, so the caller's host fallback steps the REAL batch rather than an empty one.
      LatchBnKernelsOff("BatchNormApplyGammaBeta pre-check read failed (error " +
                        IntegerToString(GetLastError()) + ")");
      return false;
     }
   //--- Host reference: run the real host step on the host mirror (currently the pre-state).
   double accG[], accB[];
   ArrayResize(accG, units);
   ArrayResize(accB, units);
   for(int i = 0; i < units; i++)
     {
      accG[i] = m_bnAcc.At(2 * i);
      accB[i] = m_bnAcc.At(2 * i + 1);
      if(!StepGammaBeta(i * BN_OPT_STRIDE, accG[i] * scale, accB[i] * scale, lt))
         return false;
     }
   int oTotal = BatchOptions.Total();
   double refOpt[];
   ArrayResize(refOpt, oTotal);
   for(int i = 0; i < oTotal; i++)
      refOpt[i] = BatchOptions.At(i);
   bool ok = DispatchBnApply(scale, lt);
   if(ok)
      ok = BatchOptions.BufferRead() && m_bnAcc.BufferRead();
   double worst = 0.0;
   int worstAt = -1;
   if(ok)
     {
      for(int i = 0; i < oTotal; i++)
        {
         double s = BnDiffScore(refOpt[i], BatchOptions.At(i));
         if(s > worst) { worst = s; worstAt = i; }
        }
      //--- and the kernel must have zeroed the accumulator
      for(int i = 0; i < m_bnAcc.Total(); i++)
         if(MathAbs(m_bnAcc.At(i)) > 1.0e-12)
           { worst = DBL_MAX; worstAt = oTotal + i; break; }
     }
   if(ok && worst <= 1.0)
     {
      m_bnCheckedApply = true;
      m_bnDeviceAuthoritative = true;
      Print(__FUNCTION__ + StringFormat(": BatchNormApplyGammaBeta kernel VERIFIED against the host "
                                        "math on %d units (worst normalized diff %.2e).", units, worst));
      return true;
     }
   //--- The host step already produced the correct post-state in the host mirror - push it, zero the
   //--- accumulator everywhere, and latch.
   for(int i = 0; i < oTotal; i++)
      BatchOptions.Update(i, refOpt[i]);
   if(BatchOptions.GetIndex() >= 0)
      BatchOptions.BufferWrite();
   ZeroOptimizerBuffer(m_bnAcc);
   m_bnDeviceAuthoritative = false;
   LatchBnKernelsOff(ok ? StringFormat("BatchNormApplyGammaBeta disagrees with the host math (worst "
                                       "normalized diff %.2e at slot %d)", worst, worstAt)
                        : "BatchNormApplyGammaBeta dispatch/readback failed (error " +
                          IntegerToString(GetLastError()) + ")");
   return true;
  }
//+------------------------------------------------------------------+
//| Backward. Called by the layer BELOW this one (inverted-call      |
//| convention, same as Conv/Pool/LSTM): consumes this layer's own   |
//| Gradient (dL/dy, already filled by calcHiddenGradients against   |
//| the layer above) and writes dL/dx into NeuronOCL's Gradient.     |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(BatchOptions) == POINTER_INVALID)
      return false;
   int units = Neurons();
   int inputsCount = NeuronOCL.Neurons();
   if(units <= 0 || inputsCount != units)
      return false; // batch norm is elementwise - a size mismatch means the topology is wrong
   //--- DEVICE PATH: the gradient, the previous output and the statistics are all already on the
   //--- device; the result lands in the previous layer's device gradient where its own backward
   //--- kernels read it. Zero transfers.
   if(BnDeviceEligible() && NeuronOCL.getOutputIndex() >= 0 && NeuronOCL.getGradientIndex() >= 0 &&
      getGradientIndex() >= 0 && EnsureBnDeviceBuffers())
     {
      if(!m_bnCheckedGrad)
         return SelfCheckBnHiddenGrad(NeuronOCL);
      if(DispatchBnHiddenGrad(NeuronOCL))
         return true;
      LatchBnKernelsOff("BatchNormHiddenGrad dispatch failed (error " +
                        IntegerToString(GetLastError()) + ")");
     }
   //--- HOST PATH.
   EnsureHostAuthoritative();
   double grad[];
   int count = getGradient(grad);
   //--- Gradient is allocated with one slot more than Neurons() (see CNeuronBaseOCL::Init); only the
   //--- real units carry a value.
   if(count < units)
      return false;
   //--- Armed for the weight-update pass, which reads this same Gradient again - see the declaration.
   if(ArrayCopy(m_gradCache, grad, 0, 0, count) == count)
      m_gradCached = true;
   //--- The previous layer's Output, from this sample's forward pass rather than a second read of the
   //--- same device buffer. Falls back to reading whenever the cache is not armed or does not match.
   double prevOut[];
   if(m_fwdInputCached && ArraySize(m_fwdInputCache) >= inputsCount)
     {
      if(ArrayCopy(prevOut, m_fwdInputCache, 0, 0, inputsCount) != inputsCount)
         return false;
     }
   else
      if(NeuronOCL.getOutputVal(prevOut) < inputsCount)
         return false;
   double ig[];
   if(ArrayResize(ig, inputsCount) != inputsCount)
      return false;
   HiddenGradHost(grad, prevOut, NeuronOCL.Activation(), ig, inputsCount);
   return NeuronOCL.setGradient(ig);
  }
//+------------------------------------------------------------------+
//| The elementwise backward math, factored out of calcInputGradients |
//| (2026-08-09) so the kernel self-check compares against LITERALLY  |
//| the code the host path runs, not a second copy that could drift.  |
//+------------------------------------------------------------------+
void CNeuronBatchNormOCL::HiddenGradHost(const double &grad[], const double &prevOut[],
                                         ENUM_ACTIVATION act, double &ig[], int n)
  {
   for(int i = 0; i < n; i++)
     {
      int shift = i * BN_OPT_STRIDE;
      double g = grad[i];
      if(iBatchSize > 1)
        {
         //--- THE CLAMP'S OWN DERIVATIVE. If the forward pass bound this unit at +-BN_MAX_NX then
         //--- the output stopped depending on the input there, so d(nx)/dx is 0 and no gradient may
         //--- pass.
         if(MathAbs(BatchOptions.At(shift + BN_OPT_NX)) >= BN_MAX_NX)
            g = 0.0;
         else
           {
            double sd = MathMax(MathSqrt(BatchOptions.At(shift + BN_OPT_VAR) + BN_EPSILON), BN_MIN_STD);
            g = g * BatchOptions.At(shift + BN_OPT_GAMMA) / sd;
           }
        }
      //--- Then the previous layer's own activation derivative, byte-for-byte the same treatment
      //--- Network.cl's CaclHiddenGradient applies - including the clamp-to-range "implied target"
      //--- reformulation - so that from the previous layer's point of view a batch-norm layer is
      //--- indistinguishable from any other. NONE falls through unscaled.
      double out = prevOut[i];
      switch(act)
        {
         case TANH:
            g = MathMax(-1.0, MathMin(1.0, g + out)) - out;
            g = g * MathMax(MIN_ACTIVATION_DERIVATIVE, 1.0 - out * out);
            break;
         case SIGMOID:
            g = MathMax(0.0, MathMin(1.0, g + out)) - out;
            g = g * MathMax(MIN_ACTIVATION_DERIVATIVE, out * (1.0 - out));
            break;
         case PRELU:
            g = g * (out >= 0 ? 1.0 : 0.01);
            break;
         default:
            break;
        }
      ig[i] = g;
     }
  }
//+------------------------------------------------------------------+
//| gamma/beta update. NeuronOCL is the previous layer and is        |
//| deliberately unused: batch norm consumes its output elementwise, |
//| so there is no incoming weight matrix to update (the CNet        |
//| constructor gives the previous layer 0 outgoing weights when     |
//| this layer follows it).                                          |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(BatchOptions) == POINTER_INVALID)
      return false;
   if(iBatchSize <= 1)
      return true; // normalization off - gamma/beta are not in the graph
   int units = Neurons();
   //--- This per-sample path stays HOST-side even in kernel mode, deliberately: it only runs when
   //--- the net's train batch is 1 (online learning on a deployed model - once per confirmed bar,
   //--- not per training sample), so kernelizing it buys nothing, and the host step is the
   //--- reference implementation.
   EnsureHostAuthoritative();
   //--- Gradient from calcInputGradients' read this same sample, not a second device round trip.
   double grad[];
   int count = m_gradCached ? ArrayCopy(grad, m_gradCache) : getGradient(grad);
   if(count < units || units <= 0)
      return false;
   //--- Same bias-corrected step size the dense Adam path computes, so gamma/beta move on the same
   //--- schedule as the weights around them.
   double lt = (optimization == SGD) ? 0.0 : g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
   for(int i = 0; i < units; i++)
     {
      int shift = i * BN_OPT_STRIDE;
      double g = grad[i];
      //--- dL/dgamma = dL/dy * nx  |  dL/dbeta = dL/dy
      double gGamma = g * BatchOptions.At(shift + BN_OPT_NX);
      double gBeta  = g;
      if(!StepGammaBeta(shift, gGamma, gBeta, lt))
         return false;
     }
   if(optimization != SGD && t < INT_MAX)
      t++;
   //--- Keep the device copy coherent so a later kernel forward reads the stepped gamma/beta.
   if(BatchOptions.GetIndex() >= 0 && !BatchOptions.BufferWrite())
      return false;
   return true;
  }
//+------------------------------------------------------------------+
//| ONE unit's gamma/beta optimizer step, factored out of             |
//| updateInputWeights so the mini-batch path                         |
//| (ApplyAccumulatedGradients) takes the identical step on the batch |
//| mean instead of carrying a second copy of this arithmetic.        |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::StepGammaBeta(int shift, double gGamma, double gBeta, double lt)
  {
     {
      double gamma  = BatchOptions.At(shift + BN_OPT_GAMMA);
      double beta   = BatchOptions.At(shift + BN_OPT_BETA);
      //--- Matches NormalizeHost's self-heal, and this is the copy that makes it STICK: the clamped
      //--- write at the end of this loop persists the repaired value, whereas the forward pass only
      //--- substitutes one locally. Without this a model that loaded a NaN gamma would normalize
      //--- correctly but never train that unit's scale again, since NaN + anything stays NaN.
      if(!MathIsValidNumber(gamma))
         gamma = 1.0;
      if(!MathIsValidNumber(beta))
         beta = 0.0;
      double dGamma = 0.0, dBeta = 0.0;
      if(optimization == SGD)
        {
         dGamma = g_eta * gGamma + alpha * BatchOptions.At(shift + BN_OPT_MG);
         dBeta  = g_eta * gBeta  + alpha * BatchOptions.At(shift + BN_OPT_MB);
         if(!BatchOptions.Update(shift + BN_OPT_MG, dGamma) ||
            !BatchOptions.Update(shift + BN_OPT_MB, dBeta))
            return false;
        }
      else
        {
         //--- Second momentum is stored ALREADY square-rooted so it can be used as the denominator
         //--- directly, the same convention this engine's UpdateWeightsAdam kernels use
         //--- (matrix_v[wi] = sqrt(...)).
         double mg = AdamBeta1 * BatchOptions.At(shift + BN_OPT_MG) + (1 - AdamBeta1) * gGamma;
         double mb = AdamBeta1 * BatchOptions.At(shift + BN_OPT_MB) + (1 - AdamBeta1) * gBeta;
         double vg = sqrt(AdamBeta2 * pow(BatchOptions.At(shift + BN_OPT_VG), 2) + (1 - AdamBeta2) * gGamma * gGamma);
         double vb = sqrt(AdamBeta2 * pow(BatchOptions.At(shift + BN_OPT_VB), 2) + (1 - AdamBeta2) * gBeta * gBeta);
         dGamma = lt * mg / (vg > 0 ? vg : lt * 10);
         dBeta  = lt * mb / (vb > 0 ? vb : lt * 10);
         if(!BatchOptions.Update(shift + BN_OPT_MG, mg) ||
            !BatchOptions.Update(shift + BN_OPT_MB, mb) ||
            !BatchOptions.Update(shift + BN_OPT_VG, vg) ||
            !BatchOptions.Update(shift + BN_OPT_VB, vb))
            return false;
        }
      dGamma = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, dGamma));
      dBeta  = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, dBeta));
      //--- `return true` and not `continue`: this is one unit's step now that the loop lives in the
      //--- caller, and a non-finite delta means SKIP this unit, never fail the layer.
      if(!MathIsValidNumber(dGamma) || !MathIsValidNumber(dBeta))
         return true;
      if(!BatchOptions.Update(shift + BN_OPT_GAMMA, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, gamma + dGamma))) ||
         !BatchOptions.Update(shift + BN_OPT_BETA,  MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, beta + dBeta))))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| MINI-BATCH (2026-08-09 audit, F4). gamma/beta are host-side      |
//| parameters, so their accumulation is a plain host sum - no       |
//| kernel and no extra device buffer.                               |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL)
  {
   //--- The outgoing dense matrix is accumulated by the layer above, exactly as for a plain dense
   //--- neuron; this adds only the normalization parameters' own gradients.
   if(CheckPointer(BatchOptions) == POINTER_INVALID || iBatchSize <= 1)
      return true;   // normalization off - gamma/beta are not in the graph
   int units = Neurons();
   //--- DEVICE PATH: the gradient and NX are both device-fresh, so the accumulation happens where
   //--- they already live. One dispatch, no transfers.
   if(BnDeviceEligible() && getGradientIndex() >= 0 && EnsureBnDeviceBuffers())
     {
      if(!m_bnCheckedAccum)
         return SelfCheckBnAccum();
      if(DispatchBnAccum())
         return true;
      LatchBnKernelsOff("BatchNormAccumGammaBeta dispatch failed (error " +
                        IntegerToString(GetLastError()) + ")");
      //--- The latch drained whatever the kernels had accumulated this batch into m_accGamma/
      //--- m_accBeta; the host code below adds THIS sample on top, so nothing is lost or doubled.
     }
   EnsureHostAuthoritative();
   //--- Gradient from calcInputGradients' read this same sample - see the declaration comment. This is
   //--- the batched twin of updateInputWeights above and takes the value from the same place.
   double grad[];
   int count = m_gradCached ? ArrayCopy(grad, m_gradCache) : getGradient(grad);
   if(count < units || units <= 0)
      return false;
   if(ArraySize(m_accGamma) != units || ArraySize(m_accBeta) != units)
     {
      ArrayResize(m_accGamma, units);
      ArrayResize(m_accBeta, units);
      ArrayInitialize(m_accGamma, 0.0);
      ArrayInitialize(m_accBeta, 0.0);
     }
   for(int i = 0; i < units; i++)
     {
      double g = grad[i];
      m_accGamma[i] += g * BatchOptions.At(i * BN_OPT_STRIDE + BN_OPT_NX);
      m_accBeta[i]  += g;
     }
   return true;
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::BeginGradAccum(void)
  {
   bool ok = CNeuronBaseOCL::BeginGradAccum();
   ArrayInitialize(m_accGamma, 0.0);
   ArrayInitialize(m_accBeta, 0.0);
   //--- Belt and braces for the device accumulator: the apply kernel zeroes it itself, but a batch
   //--- abandoned mid-way (era boundary, restore) must not leak its partial sums into the next one.
   if(CheckPointer(m_bnAcc) != POINTER_INVALID && m_bnAcc.GetIndex() >= 0)
      ok = ZeroOptimizerBuffer(m_bnAcc) && ok;
   return ok;
  }
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::ApplyAccumulatedGradients(double scale)
  {
   //--- Outgoing dense matrix first, through the shared block optimizer.
   bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale);
   if(CheckPointer(BatchOptions) != POINTER_INVALID && iBatchSize > 1)
     {
      double lt = (optimization == SGD) ? 0.0 : g_eta * sqrt(1 - pow(AdamBeta2, t)) / (1 - pow(AdamBeta1, t));
      //--- DEVICE PATH: the batch was accumulated by the kernels, so it is stepped by the kernel too.
      //--- A dispatch failure latches (which drains the device sums into m_accGamma/m_accBeta) and
      //--- drops to the host loop below, so the batch is stepped exactly once either way.
      bool stepped = false;
      if(BnDeviceEligible() && CheckPointer(m_bnAcc) != POINTER_INVALID && m_bnAcc.GetIndex() >= 0)
        {
         if(!m_bnCheckedApply)
            stepped = SelfCheckBnApply(scale, lt);
         else
            if(DispatchBnApply(scale, lt))
               stepped = true;
            else
               LatchBnKernelsOff("BatchNormApplyGammaBeta dispatch failed (error " +
                                 IntegerToString(GetLastError()) + ")");
        }
      if(!stepped)
        {
         EnsureHostAuthoritative();
         int units = MathMin(Neurons(), MathMin(ArraySize(m_accGamma), ArraySize(m_accBeta)));
         for(int i = 0; i < units; i++)
            if(!StepGammaBeta(i * BN_OPT_STRIDE, m_accGamma[i] * scale, m_accBeta[i] * scale, lt))
              {
               ok = false;
               break;
              }
         ArrayInitialize(m_accGamma, 0.0);
         ArrayInitialize(m_accBeta, 0.0);
         //--- Keep a created-but-idle device copy coherent with the host step.
         if(BatchOptions.GetIndex() >= 0)
            ok = BatchOptions.BufferWrite() && ok;
        }
     }
   //--- One step, so t advances once - matching every other layer's batched apply.
   if(optimization != SGD && t < INT_MAX)
      t++;
   return ok;
  }
//+------------------------------------------------------------------+
//| Outgoing dense weight matrix followed by the whole BatchOptions   |
//| block, as one flat array - see the declaration comment.           |
//+------------------------------------------------------------------+
int CNeuronBatchNormOCL::getWeightsBN(double &values[])
  {
   //--- Checkpoints and the health report call this mid-training; in kernel mode the statistics live
   //--- on the device, so pull them first. Read-only - the device stays authoritative.
   SyncOptionsToHost();
   double w[];
   int wCount = getWeights(w);
   if(wCount < 0)
      wCount = 0;
   int oCount = (CheckPointer(BatchOptions) == POINTER_INVALID) ? 0 : BatchOptions.Total();
   if(ArrayResize(values, wCount + oCount) != wCount + oCount)
      return 0;
   for(int i = 0; i < wCount; i++)
      values[i] = w[i];
   for(int i = 0; i < oCount; i++)
      values[wCount + i] = BatchOptions.At(i);
   return wCount + oCount;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::setWeightsBN(double &values[])
  {
   int total = ArraySize(values);
   int oCount = (CheckPointer(BatchOptions) == POINTER_INVALID) ? 0 : BatchOptions.Total();
   int wCount = total - oCount;
   if(wCount < 0)
      return false; // snapshot predates this layer's parameters - refuse rather than half-restore
   if(wCount > 0)
     {
      double w[];
      if(ArrayResize(w, wCount) != wCount)
         return false;
      for(int i = 0; i < wCount; i++)
         w[i] = values[i];
      if(!setWeights(w))
         return false;
     }
   for(int i = 0; i < oCount; i++)
      if(!BatchOptions.Update(i, values[wCount + i]))
         return false;
   //--- Full host overwrite of every slot: push it so a kernel-mode net keeps computing on the
   //--- RESTORED statistics rather than the diverged ones the device still holds. After this the two
   //--- copies are identical, so whichever side was authoritative remains consistent.
   if(BatchOptions.GetIndex() >= 0 && !BatchOptions.BufferWrite())
      return false;
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::Save(const int file_handle)
  {
   //--- The statistics ARE the layer's inference behaviour; in kernel mode their current values live
   //--- on the device, and a .nnw written from the stale host mirror would be a different model.
   SyncOptionsToHost();
   if(!CNeuronBaseOCL::Save(file_handle))
      return false;
   if(FileWriteInteger(file_handle, iBatchSize, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, iSamplesSeen, INT_VALUE) < INT_VALUE)
      return false;
   if(CheckPointer(BatchOptions) == POINTER_INVALID)
      return false;
   return BatchOptions.Save(file_handle);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBatchNormOCL::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      return false;
   iBatchSize = FileReadInteger(file_handle, INT_VALUE);
   iSamplesSeen = FileReadInteger(file_handle, INT_VALUE);
   if(CheckPointer(BatchOptions) == POINTER_INVALID)
     {
      BatchOptions = new CBufferDouble();
      if(CheckPointer(BatchOptions) == POINTER_INVALID)
         return false;
     }
   if(!BatchOptions.Load(file_handle))
      return false;
   //--- Loaded state is a full host overwrite - push it if a device copy already exists (a reload into
   //--- a live net), and the host is authoritative until the first kernel forward.
   if(BatchOptions.GetIndex() >= 0 && !BatchOptions.BufferWrite())
      return false;
   m_bnDeviceAuthoritative = false;
   //--- A model whose statistics block does not match its own width is not usable for inference -
   //--- fail loudly here rather than index past the end on the first forward pass.
   if(BatchOptions.Total() != Neurons() * BN_OPT_STRIDE)
     {
      Print(__FUNCTION__ + ": batch-norm parameter block is " + IntegerToString(BatchOptions.Total()) +
            " values but this layer has " + IntegerToString(Neurons()) + " units (expected " +
            IntegerToString(Neurons() * BN_OPT_STRIDE) + ") - file does not match the topology");
      return false;
     }
   return true;
  }
#endif // WARRIOR_AI_IMPL_NEURONBATCHNORM_MQH
//+------------------------------------------------------------------+