Warrior_EA/AI/Network.mqh

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include <Arrays\ArrayDouble.mqh>
#include <Arrays\ArrayInt.mqh>
#include <Arrays\ArrayObj.mqh>
#include <OpenCL\OpenCL.mqh>
#include "..\System\AtomicFile.mqh"
//--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available,
//--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Sizes
//--- WarriorCPU.dll's worker thread pool - no effect at all when a GPU tier (OpenCL or DirectML) is
//--- active, since neither one calls into WarriorCPU.dll.
//---
//--- A FIXED SMALL THREAD COUNT PER NETWORK, not a share of the machine. This replaced a TargetCPULoad
//--- input divided by the live chart count, which was wrong twice over:
//---  - The count is a SNAPSHOT taken when each net's pool is built, and charts are attached one at a
//---    time. Measured 2026-07-29 with five charts: they took 10/6/5/4/4% of the same budget, because
//---    the first chart only ever saw itself and the last saw all five. So the earliest chart got
//---    several times the threads of the latest - which silently skews any cross-topology comparison
//---    run on those charts, the exact thing the setting existed to make fair.
//---  - Nothing rebalances when a chart is added or removed, and rebalancing would mean tearing down a
//---    DLL context underneath a running trainer.
//--- Neither problem exists once the answer stops depending on how many charts are running.
//---
//--- Why 2 threads is not a compromise: since the topology became data-derived the widest dense layer
//--- is 64 units (ExpertSignalAIBase.mqh's ComputeFirstLayerWidth), so each ParallelFor has almost
//--- nothing to split and per-dispatch overhead dominates. The measurements agree - an MLP era cost
//--- ~66s at a wildly oversubscribed 12 threads and ~80s at 1 thread, a 20% spread across a 12x
//--- difference in thread count. Two per net also lands six concurrent charts exactly on a 12-core box.
//---
//--- Removing the input costs nothing on the product side: a Market build has no DLL tier at all (MQL5
//--- Market rule IV strips the #import blocks - see AI\NeuronDirectML.mqh), so it was already compiled
//--- out to a constant there and no buyer could ever reach it.
#define CPU_THREADS_PER_NETWORK 2
//+------------------------------------------------------------------+
//| The percentage CDirectMLMy needs in order to land on              |
//| CPU_THREADS_PER_NETWORK workers, given the detected core count.   |
//| (WarriorCPU.dll takes a percentage of cores, not a thread count.) |
//+------------------------------------------------------------------+
int EffectiveCpuLoadPercent()
  {
   int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES);
   //--- Unknown core count: assume a small machine rather than a large one. Guessing high here would
   //--- reintroduce exactly the oversubscription this function exists to prevent.
   if(cores <= 0)
      cores = 4;
   //--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the
   //--- DLL, so this is the one value that needs no arithmetic.
   if(cores <= CPU_THREADS_PER_NETWORK)
      return 100;
   //--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread
   //--- short of the target is a worse error than landing one over.
   int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores);
   return (int)MathMax(1, MathMin(100, pct));
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
//--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference
//--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's
//--- abstract "0.001" mention, which the book's own worked examples don't actually use either.
//--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005)
//--- this project had separately tuned lr to before this input existed, so no behavior conflict.
//--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak
//--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to
//--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this
//--- reverts to the literature/book default.
input double AdamLearningRate = 0.0003; // Adam learning rate
input double AdamBeta1         = 0.9;   // Adam beta1
input double AdamBeta2         = 0.999; // Adam beta2
//--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method
//--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses
//--- Adam's book-default rate as its starting point too. The book also states no numeric default for
//--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses
//--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for.
input double SgdLearningRate   = 0.0003; // SGD learning rate
input double SgdMomentum       = 0.9;    // SGD momentum
#define  lr AdamLearningRate
#define  b1 AdamBeta1
#define  b2 AdamBeta2
#define  momentum SgdMomentum
double eta = lr;
#define defConnect         0x7781
#define defArrayConnects   0x7782
#define defNeuronBase      0x7783
#define defNeuron          0x7784
#define defNeuronConv      0x7785
#define defNeuronPool     0x7786
#define defLayer           0x7787
#define defArrayLayer      0x7788
#define defNet             0x7789
#define defNeuronLSTM      0x7791
//--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like
//--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps
//--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh.
#define defNeuronBatchNorm 0x7792
//---
#define defBufferDouble    0x7882
#define defNeuronBaseOCL   0x7883
#define defNeuronLSTMOCL   0x7884
#define defNeuronConvOCL   0x7885
#define defNeuronPoolOCL   0x7886
#define defNeuronBatchNormOCL 0x7887
//---
#define def_k_FeedForward     0
#define def_k_ff_matrix_w     0
#define def_k_ff_matrix_i     1
#define def_k_ff_matrix_o     2
#define def_k_ff_inputs       3
#define def_k_ff_activation   4
//---
#define def_k_CaclOutputGradient 1
#define def_k_cog_matrix_t       0
#define def_k_cog_matrix_o       1
#define def_k_cog_matrix_ig      2
#define def_k_cog_activation     3
//---
#define def_k_CaclHiddenGradient 2
#define def_k_chg_matrix_w       0
#define def_k_chg_matrix_g       1
#define def_k_chg_matrix_o       2
#define def_k_chg_matrix_ig      3
#define def_k_chg_outputs        4
#define def_k_chg_activation     5
//---
#define def_k_UpdateWeightsMomentum      3
#define def_k_uwm_matrix_w        0
#define def_k_uwm_matrix_g        1
#define def_k_uwm_matrix_i        2
#define def_k_uwm_matrix_dw       3
#define def_k_uwm_inputs          4
#define def_k_uwm_learning_rates  5
#define def_k_uwm_momentum        6
#define def_k_uwm_optimizer       7
//---
#define def_k_UpdateWeightsAdam   4
#define def_k_uwa_matrix_w        0
#define def_k_uwa_matrix_g        1
#define def_k_uwa_matrix_i        2
#define def_k_uwa_matrix_m        3
#define def_k_uwa_matrix_v        4
#define def_k_uwa_inputs          5
#define def_k_uwa_l               6
#define def_k_uwa_b1              7
#define def_k_uwa_b2              8
//---
#define def_k_FeedForwardProof    15
#define def_k_ffp_matrix_i        0
#define def_k_ffp_matrix_o        1
#define def_k_ffp_inputs          2
#define def_k_ffp_window          3
#define def_k_ffp_step            4
//---
#define def_k_CalcInputGradientProof 16
#define def_k_cigp_matrix_i       0
#define def_k_cigp_matrix_g       1
#define def_k_cigp_matrix_o       2
#define def_k_cigp_matrix_ig      3
#define def_k_cigp_outputs        4
#define def_k_cigp_window         5
#define def_k_cigp_step           6
//---
#define def_k_FeedForwardConv     5
#define def_k_ffc_matrix_w        0
#define def_k_ffc_matrix_i        1
#define def_k_ffc_matrix_o        2
#define def_k_ffc_inputs          3
#define def_k_ffc_step            4
#define def_k_ffc_window_in       5
#define def_k_ffc_window_out      6
#define def_k_ffc_activation      7
//---
#define def_k_CalcHiddenGradientConv 6
#define def_k_chgc_matrix_w       0
#define def_k_chgc_matrix_g       1
#define def_k_chgc_matrix_o       2
#define def_k_chgc_matrix_ig      3
#define def_k_chgc_outputs        4
#define def_k_chgc_step           5
#define def_k_chgc_window_in      6
#define def_k_chgc_window_out     7
#define def_k_chgc_activation     8
//---
#define def_k_UpdateWeightsConvMomentum 7
#define def_k_uwcm_matrix_w       0
#define def_k_uwcm_matrix_g       1
#define def_k_uwcm_matrix_i       2
#define def_k_uwcm_matrix_dw      3
#define def_k_uwcm_inputs         4
#define def_k_uwcm_learning_rates 5
#define def_k_uwcm_momentum       6
#define def_k_uwcm_window_in      7
#define def_k_uwcm_window_out     8
#define def_k_uwcm_step           9
#define def_k_uwcm_optimizer      10
//---
#define def_k_UpdateWeightsConvAdam 8
#define def_k_uwca_matrix_w       0
#define def_k_uwca_matrix_g       1
#define def_k_uwca_matrix_i       2
#define def_k_uwca_matrix_m       3
#define def_k_uwca_matrix_v       4
#define def_k_uwca_inputs         5
#define def_k_uwca_l              6
#define def_k_uwca_b1             7
#define def_k_uwca_b2             8
#define def_k_uwca_window_in      9
#define def_k_uwca_window_out     10
#define def_k_uwca_step           11
//---
// LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient
// flows back into h_prev/c_prev from a prior step). Supports both Adam and
// SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) -
// see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch.
// Derived from scratch from the standard LSTM equations - NOT ported from
// the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites
// the live weights buffer instead of writing to weights_gradient.
#define def_k_LSTM_Gates          9
#define def_k_lstmg_matrix_w      0
#define def_k_lstmg_hidden_prev   1
#define def_k_lstmg_inputs        2
#define def_k_lstmg_concatenated  3
#define def_k_lstmg_hidden_size   4
#define def_k_lstmg_input_size    5
//---
#define def_k_LSTM_State          10
#define def_k_lstms_concatenated  0
#define def_k_lstms_memory        1
#define def_k_lstms_hidden_prev   2
#define def_k_lstms_hidden_cache  3
#define def_k_lstms_output        4
#define def_k_lstms_hidden_size   5
//---
#define def_k_LSTM_GateGradient   11
#define def_k_lstmgg_gradient     0
#define def_k_lstmgg_memory       1
#define def_k_lstmgg_concatenated 2
#define def_k_lstmgg_concatenated_gradient 3
#define def_k_lstmgg_hidden_size  4
//---
#define def_k_LSTM_WeightsGradient 12
#define def_k_lstmwg_concatenated_gradient 0
#define def_k_lstmwg_hidden_cache 1
#define def_k_lstmwg_inputs       2
#define def_k_lstmwg_weights_gradient 3
#define def_k_lstmwg_hidden_size  4
#define def_k_lstmwg_input_size   5
//---
#define def_k_LSTM_InputsGradient 13
#define def_k_lstmig_concatenated_gradient 0
#define def_k_lstmig_matrix_w     1
#define def_k_lstmig_inputs_gradient 2
#define def_k_lstmig_hidden_size  3
#define def_k_lstmig_input_size   4
//---
#define def_k_LSTM_UpdateWeightsAdam 14
#define def_k_lstmuwa_matrix_w    0
#define def_k_lstmuwa_weights_gradient 1
#define def_k_lstmuwa_matrix_m    2
#define def_k_lstmuwa_matrix_v    3
#define def_k_lstmuwa_l           4
#define def_k_lstmuwa_b1          5
#define def_k_lstmuwa_b2          6
//---
// SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see
// AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body.
#define def_k_LSTM_UpdateWeightsMomentum 17
#define def_k_lstmuwm_matrix_w    0
#define def_k_lstmuwm_weights_gradient 1
#define def_k_lstmuwm_matrix_dw   2
#define def_k_lstmuwm_learning_rates 3
#define def_k_lstmuwm_momentum    4
#define def_k_lstmuwm_optimizer   5
//---
// b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see
// AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era
// same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root
// cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE
// gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever
// could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled
// kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam
// signatures), so they're safe to expose as ordinary inputs.
// Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had
// already gone unstable (real collapses were happening at weight magnitudes several orders of
// magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh
// engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually
// reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below.
#define MAX_WEIGHT                100.0
// Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in
// DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see
// WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright
// +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of
// training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi-
// hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles.
// 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates,
// ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's
// sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a
// weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the
// discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the
// calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the
// spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt
// dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay
// strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins
// argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth.
#define WEIGHT_DECAY              0.001
// Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full
// rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden
// Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras),
// most likely from 5x back-to-back oversampling replay building artificially correlated momentum.
// Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the
// post-update weight value and is far too loose (1e6) to prevent this.
#define MAX_WEIGHT_DELTA          0.1
// Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see
// SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its
// activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that
// neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated
// unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in
// Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine.
// 2026-07-27: Increased from 1.0e-4 to 1.0e-3 — must stay in sync with AI\Network.cl's matching
// constant. See that file's comment for the full rationale (fp32 OpenCL saturation floor fix).
#define MIN_ACTIVATION_DERIVATIVE 1.0e-3
// Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND
// read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a
// different distribution than it was trained on). The classification outputs are SIGMOID-bounded to
// [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 -
// the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training
// can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three
// outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the
// ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal
// modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays
// (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the
// loss, keeping gradient magnitudes (and thus eta tuning) unchanged.
#define CLASS_LOGIT_SCALE         6.0

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#resource "Network.cl" as string cl_program
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
enum ENUM_ACTIVATION
  {
   NONE,
   TANH,
   SIGMOID,
   PRELU     // fixed param=0.01, matches CNeuronConv's CPU activationFunction
  };
//+------------------------------------------------------------------+
//| Translates ENUM_ACTIVATION to the "activation" int code every     |
//| native compute backend actually understands (Network.cl's         |
//| kernels, and the mirrored Activation()/inline switches in         |
//| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and  |
//| deliberately anything else (incl. NONE) falls through every one   |
//| of those switches unmatched, which is exactly linear passthrough -|
//| there's no case 3 anywhere on the native side, so PRELU relies on |
//| the FeedForwardConv-family kernels specifically, and NONE never   |
//| needs a case at all. This is NOT the same numbering as            |
//| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a    |
//| raw (int)activation cast at a kernel call site silently sends the |
//| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native     |
//| SIGMOID). Only use this at actual kernel-dispatch call sites -     |
//| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw         |
//| ENUM_ACTIVATION value instead, and must keep using (int)activation |
//| directly so saved topology files round-trip through Load() as-is. |
//+------------------------------------------------------------------+
int NativeActivationCode(ENUM_ACTIVATION value)
  {
   switch(value)
     {
      case TANH:    return 0;
      case SIGMOID: return 1;
      case PRELU:   return 2;
      default:      return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches
     }
  }
//+------------------------------------------------------------------+
//| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the |
//| load-time architecture repair (CNet::EnforceOutputActivation) so  |
//| the log names the stale value it found instead of printing a bare |
//| integer nobody can decode months later.                           |
//+------------------------------------------------------------------+
string ActivationName(ENUM_ACTIVATION value)
  {
   switch(value)
     {
      case NONE:    return "NONE";
      case TANH:    return "TANH";
      case SIGMOID: return "SIGMOID";
      case PRELU:   return "PRELU";
     }
   return "UNKNOWN(" + IntegerToString((int)value) + ")";
  }
//---
//--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\
//--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead of
//--- this AI header - without a duplicate-definition error. Whichever file is parsed first defines it;
//--- the other's copy is skipped. Keep the two definitions in sync.
#ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED
#define WARRIOR_ENUM_OPTIMIZATION_DEFINED
//--- 2026-07-28: a third DFA entry was removed. It was never Direct Feedback Alignment: its feedback
//--- signal multiplied dL/dw by a DETERMINISTIC sign pattern (connectionIndex % 2), which makes half of
//--- every weight tensor perform gradient ASCENT permanently - it diverges by construction, with no
//--- hyperparameter able to rescue it. Real DFA (Nokland 2016) works because a FIXED RANDOM matrix gives
//--- a consistent feedback direction the forward weights can align to; an index-parity sign flip has no
//--- such alignment property. Its backward pass was also structurally incompatible with the OpenCL/
//--- DirectML neuron model this project actually runs on (one CNeuronBaseOCL object holds a whole layer
//--- in a device buffer, so the per-neuron host-scalar loops it used saw layer.Total()==1 and updated
//--- nothing). SGD/ADAM keep their ordinal values 0/1 - m_optimizationAlgo feeds the weights-filename
//--- fingerprint, so these must never be renumbered.
enum ENUM_OPTIMIZATION
  {
   SGD,  // SGD + Momentum (heavy-ball, simpler, needs more eras)
   ADAM  // Adam (adaptive step, faster convergence, can overfit)
  };
#endif
//---
enum ENUM_BUFFERS
  {
   WEIGHTS,
   DELTA_WEIGHTS,
   OUTPUT,
   GRADIENT,
   FIRST_MOMENTUM,
   SECOND_MOMENTUM
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "NeuronPrimitives.mqh"
class CLayer;
//---
class CNeuronBase    :  public CObject
  {
protected:
   double            outputVal;
   double            prevVal;
   uint              m_myIndex;
   double            gradient;
   CArrayCon         *Connections;
   ENUM_ACTIVATION   activation;
   ENUM_OPTIMIZATION optimization;
   int               t;
   //---
   virtual bool      feedForward(CLayer *prevLayer)               {  return false;     }
   virtual bool      calcHiddenGradients(CLayer *&nextLayer)     {  return false;     }
   virtual double    activationFunction(double x);
   virtual double    SigmoidFunction(double x)                    {  return MathPow(1 + exp(-x), -1); }
   virtual double    TanhFunction(double x)                       {  return tanh(x);   }
   virtual CLayer    *getOutputLayer(void)                        {  return NULL;      }
public:
                     CNeuronBase(void);
                    ~CNeuronBase(void);
   virtual bool      Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }
   //--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back
   //--- what a Load() restored from disk without caring which neuron model the net was built from.
   virtual ENUM_ACTIVATION Activation(void)                       {  return activation;  }
   virtual void      SetOptimization(ENUM_OPTIMIZATION value) {  optimization = value; }
   virtual ENUM_OPTIMIZATION Optimization(void) const { return optimization; }
   //---
   static double            alpha;
   //---
   virtual void      setOutputVal(double val)                     {  prevVal = outputVal;   outputVal = val;    }
   virtual double    getOutputVal()                               {  return outputVal; }
   virtual double    getPrevVal()                                 {  return prevVal;   }
   virtual void      setGradient(double val)                      {  gradient = val;     }
   virtual double    getGradient()                                {  return gradient;  }
   virtual CArrayCon *getConnections()                            {  return Connections;}
   virtual double    activationFunctionDerivative(double x);
   virtual double    SigmoidFunctionDerivative(double x)          {  return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x));   }
   virtual double    TanhFunctionDerivative(double x)             {  return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x));  }
   //---
   virtual bool      feedForward(CObject *&SourceObject);
   virtual bool      calcHiddenGradients(CObject *&TargetObject);
   virtual bool      updateInputWeights(CLayer *prevLayer)       {  return false;     }
   virtual bool      updateInputWeights(CObject *SourceObject);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle)
     {
      activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
      optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      return(Connections.Load(file_handle));
     }
   //---
   virtual int       Type(void)        const                      {  return defNeuronBase;                  }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::alpha = momentum;     // momentum
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBase::CNeuronBase(void)  :
   outputVal(1),
   gradient(0),
   activation(TANH),
   t(1),
   optimization(SGD)
  {
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBase::~CNeuronBase(void)
  {
   if(CheckPointer(Connections) != POINTER_INVALID)
      delete Connections;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0)
  {
   if(CheckPointer(Connections) == POINTER_INVALID)
     {
      Connections = new CArrayCon();
      if(CheckPointer(Connections) == POINTER_INVALID)
         return false;
     }
//---
   if(!Connections.Reserve(fmax(numOutputs, 1)))
     {
      Print(__FUNCTION__ + ": Connections.Reserve failed (allocation failure?) - neuron would silently end up with 0 connections");
      return false;
     }
   for(uint c = 0; c < numOutputs; c++)
     {
      if(!Connections.CreateElementScaled(c, weighScale))
         return false;
      Connections.IncreaseTotal();
     }
//---
   m_myIndex = myIndex;
   optimization = optimization_type;
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "NeuronCPU.mqh"
class COpenCLMy   :  public COpenCL
  {
public:
                     COpenCLMy(void)   {};
                    ~COpenCLMy(void)   {};
   template<typename T>
   int               AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
  };
#include "NeuronDirectML.mqh"
class CLayer: public CArrayObj
  {
private:
   uint              iOutputs;
   int               iFileHandle;
   int               hWeights;
   int               hDeltaWeights;
   int               hOutput;
   int               hGradient;
   COpenCLMy         *OpenCL;
   CDirectMLMy       *DirectML;

public:
                     CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL);
                    ~CLayer(void) {};
   //--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below.
   bool              CreateElementScaled(int const index, double weighScale);
   //--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool
   //--- CreateElement(const int index)` - because it is the real virtual override that CArrayObj::Load()
   //--- dispatches through, and CArrayObj::Load() is how EVERY saved layer is read back (CLayer::Load ->
   //--- CNet::Load). In MQL5 an override must match the base parameter list
   //--- exactly; adding even a DEFAULTED parameter makes it a separate method that merely hides the base
   //--- one, silently leaving the base's `return(false)` stub in the vtable slot. That is exactly what a
   //--- `double weighScale = -1.0` parameter added here did: from then on every single model load failed
   //--- at the first layer ("REJECTED: only loaded 0 of N layers (failed at layer 0)") no matter how
   //--- healthy the .nnw was, so every restart retrained from era 0 and every best-checkpoint restore
   //--- silently kept the current weights. Never add parameters to this method - extend
   //--- CreateElementScaled() and call it explicitly instead.
   virtual bool      CreateElement(const int index) { return CreateElementScaled(index, -1.0); }
   virtual void      IncreaseTotal()   {  m_data_total++;   }
   virtual int       Type(void)  const { return defLayer; }
   virtual bool      Load(const int file_handle);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CLayer::CreateElementScaled(int index, double weighScale)
  {
   if(index >= m_data_max)
      return false;
//---
   bool result = false;
   CNeuronBase *temp = NULL;
   CNeuronPool *temp_p = NULL;
   CNeuronBaseOCL *temp_ocl = NULL;
   if(iFileHandle <= 0)
     {
      temp = new CNeuron();
      if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD, weighScale))
         return false;
      result = true;
     }
   else
     {
      int type = FileReadInteger(iFileHandle);
      switch(type)
        {
         case  defNeuron:
            temp = new CNeuron();
            if(CheckPointer(temp) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            result = temp.Init(iOutputs, index, ADAM);
            break;
         case  defNeuronPool:
            temp_p = new CNeuronPool();
            if(CheckPointer(temp_p) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronConv:
            temp_p = new CNeuronConv();
            if(CheckPointer(temp_p) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronLSTM:
            temp_p = new CNeuronLSTM();
            if(CheckPointer(temp_p) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronBaseOCL:
            temp_ocl = new CNeuronBaseOCL();
            if(CheckPointer(temp_ocl) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            //--- Pure-MQL5 inference (no backend): construct host-only. Load() restores the weights into
            //--- the host buffers and CNeuronBaseOCL::feedForwardCPU() computes on them. The device Init()
            //--- below REQUIRES a backend, so it only runs when one exists (training/optimization/GPU run).
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
              {
               m_data[index] = temp_ocl;
               return true;
              }
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM)
               : temp_ocl.Init(iOutputs, index, DirectML, 1, ADAM))
              {
               m_data[index] = temp_ocl;
               return true;
              }
            break;
         case  defNeuronConvOCL:
           {
            //--- placeholder dims; the real window/step/units are restored by Load() right after
            CNeuronConvOCL *temp_conv = new CNeuronConvOCL();
            if(CheckPointer(temp_conv) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            //--- Pure-MQL5 inference (no backend): construct host-only, Load() fills the host buffers,
            //--- CNeuronConvOCL::feedForwardCPU() computes on them (device Init below needs a backend).
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
              {
               m_data[index] = temp_conv;
               return true;
              }
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM)
               : temp_conv.Init(iOutputs, index, DirectML, 1, 1, 1, 1, ADAM))
              {
               m_data[index] = temp_conv;
               return true;
              }
            break;
           }
         case  defNeuronBatchNormOCL:
           {
            //--- Placeholder width of 1; CNeuronBatchNormOCL::Load() replaces both the buffers and the
            //--- whole gamma/beta/statistics block with the real ones, then verifies they agree.
            CNeuronBatchNormOCL *temp_bn = new CNeuronBatchNormOCL();
            if(CheckPointer(temp_bn) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note).
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
              {
               m_data[index] = temp_bn;
               return true;
              }
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_bn.Init(iOutputs, index, OpenCL, 1, 1, ADAM)
               : temp_bn.Init(iOutputs, index, DirectML, 1, 1, ADAM))
              {
               m_data[index] = temp_bn;
               return true;
              }
            break;
           }
         case  defNeuronPoolOCL:
           {
            CNeuronPoolOCL *temp_pool = new CNeuronPoolOCL();
            if(CheckPointer(temp_pool) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note).
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
              {
               m_data[index] = temp_pool;
               return true;
              }
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_pool.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM)
               : temp_pool.Init(iOutputs, index, DirectML, 1, 1, 1, ADAM))
              {
               m_data[index] = temp_pool;
               return true;
              }
            break;
           }
         case  defNeuronLSTMOCL:
           {
            CNeuronLSTMOCL *temp_lstm = new CNeuronLSTMOCL();
            if(CheckPointer(temp_lstm) == POINTER_INVALID)
              {
               result = false;
               break;
              }
            //--- Pure-MQL5 inference (no backend): construct host-only (see the defNeuronConvOCL note).
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
              {
               m_data[index] = temp_lstm;
               return true;
              }
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_lstm.Init(iOutputs, index, OpenCL, 1, ADAM)
               : temp_lstm.Init(iOutputs, index, DirectML, 1, ADAM))
              {
               m_data[index] = temp_lstm;
               return true;
              }
            break;
           }
         default:
            result = false;
            break;
        }
     }
   if(result)
      m_data[index] = temp;
//---
   return (result);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "ArrayLayer.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPool : public CNeuronBase
  {
protected:
   CLayer            *OutputLayer;
   int               iWindow;
   int               iStep;

   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);

public:
                     CNeuronPool(void) {};
                    ~CNeuronPool(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronPool;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConv  :  public CNeuronPool
  {
protected:
   double            param;   //PReLU param
   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);
   virtual double    activationFunction(double x);
   virtual bool      updateInputWeights(CLayer *prevLayer);
public:
                     CNeuronConv() :   param(0.01) { };
                    ~CNeuronConv(void)             { };
   //---
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   virtual double    activationFunctionDerivative(double x);
   virtual int       Type(void)   const   {  return defNeuronConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::feedForward(CObject *&SourceObject)
  {
   bool result = false;
//---
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(SourceObject.Type())
     {
      case defLayer:
         temp_l = SourceObject;
         result = feedForward(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         temp_n = SourceObject;
         result = feedForward(temp_n.getOutputLayer());
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::updateInputWeights(CObject *SourceObject)
  {
   bool result = false;
//---
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(SourceObject.Type())
     {
      case defLayer:
         temp_l = SourceObject;
         result = updateInputWeights(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         temp_n = SourceObject;
         temp_l = temp_n.getOutputLayer();
         result = updateInputWeights(temp_l);
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::feedForward(CLayer *prevLayer)
  {
   bool result = false;
//---
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return result;
//---
   int total = prevLayer.Total() - iWindow + 1;
   CNeuron *temp;
   CConnection *con;
   result = true;
   for(int i = 0; (i < total && result); i += iStep)
     {
      double sum = 0;
      for(int j = 0; (j < iWindow && result); j++)
        {
         temp = prevLayer.At(i + j);
         con = Connections.At(j);
         if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
            return false;
         double val = temp.getOutputVal();
         sum += val * con.weight;
        }
      temp = OutputLayer.At(i / iStep);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setOutputVal(activationFunction(sum));
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunction(double x)
  {
   if(x >= 0)
      return x;
   return param * x;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject)
  {
   bool result = false;
//---
   if(CheckPointer(TargetObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(TargetObject.Type())
     {
      case defLayer:
         temp_l = TargetObject;
         result = calcHiddenGradients(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         switch(Type())
           {
            case defNeuron:
               temp_n = TargetObject;
               result = temp_n.calcInputGradients(GetPointer(this), m_myIndex);
               break;
            case defNeuronLSTM:
               temp_n = TargetObject;
               temp_l = getOutputLayer();
               if(!temp_n.calcInputGradients(temp_l))
                 {
                  result = false;
                  break;
                 }
               result = calcHiddenGradients(temp_l);
               break;
            default:
               temp_l =getOutputLayer();
               temp_n = TargetObject;
               result = temp_n.calcInputGradients(temp_l);
               break;
           }
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
      return false;
//---
   gradient = 0;
   int total = OutputLayer.Total();
   CNeuron *temp;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal()));
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunctionDerivative(double x)
  {
   if(x >= 0)
      return 1;
   return param;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::updateInputWeights(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   CConnection *con;
   double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
   for(int n = 0; n < iWindow && !IsStopped(); n++)
     {
      con = Connections.At(n);
      if(CheckPointer(con) == POINTER_INVALID)
         continue;
      double delta = 0;
      int total_i = OutputLayer.Total();
      CNeuron *prev, *out;
      for(int i = 0; i < total_i; i++)
        {
         prev = prevLayer.At(n * iStep + i);
         out = OutputLayer.At(total_i - i - 1);
         if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID)
            continue;
         delta += prev.getOutputVal() * out.getGradient();
        }
      if(optimization == SGD)
         con.weight += con.deltaWeight = (delta != 0 ? eta*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0);
      else
        {
         con.mt = b1 * con.mt + (1 - b1) * delta;
         con.vt = b2 * con.vt + (1 - b2) * delta * delta + 0.00000001;
         con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
         // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why.
         con.weight += con.deltaWeight;
        }
      // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
      // AI\Network.cl's UpdateWeightsConvMomentum/UpdateWeightsConvAdam MAX_WEIGHT clamp.
      con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
     }
   if(optimization == ADAM)
      t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
  {
   iWindow = window;
   iStep = step;
   //--- LeCun-uniform init, matching CNeuronConvOCL::Init's rationale - fan-in is the conv window size.
   if(!CNeuronBase::Init(window, myIndex, optimization_type, 1.0 / MathSqrt((double)window + 1.0)))
      return false;
   OutputLayer = new CLayer(numOutputs);
   if(CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
   //--- He-scaled init for the OutputLayer's own dense units - fan-in is this pool/conv unit's own
   //--- sibling count (units_count); no OCL/DLL equivalent exists to mirror since those tiers use flat
   //--- buffers instead of this per-unit object representation, so this follows the same dense rationale
   //--- as CNet::CNet()'s defNeuron case above.
   double outputScale = MathSqrt(2.0 / ((double)units_count + 1.0));
   if(!OutputLayer.Reserve(units_count))
     {
      Print(__FUNCTION__ + ": OutputLayer.Reserve failed (allocation failure?) - neuron would silently end up with 0 outputs");
      return false;
     }
   for(int i = 0; i < units_count; i++)
     {
      if(!OutputLayer.CreateElementScaled(i, outputScale))
         return false;
      OutputLayer.IncreaseTotal();
     }
//---
   if(Type() == defNeuronPool)
     {
      if(CheckPointer(Connections) != POINTER_INVALID)
         Connections.Clear();
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronPool::~CNeuronPool(void)
  {
   delete OutputLayer;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::feedForward(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return false;
//---
   int total = prevLayer.Total() - iWindow + 1;
   CNeuron *temp;
   for(int i = 0; i <= total; i += iStep)
     {
      double sum = 0;
      for(int j = 0; j < iWindow; j++)
        {
         temp = prevLayer.At(i + j);
         if(CheckPointer(temp) == POINTER_INVALID)
            continue;
         sum += temp.getOutputVal();
        }
      temp = OutputLayer.At(i / iStep);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setOutputVal(sum / iWindow);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
      return false;
//---
   gradient = 0;
   int total = OutputLayer.Total();
   CNeuron *temp;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(temp.sumDOW(nextLayer));
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID  || CheckPointer(prevLayer.At(0)) == POINTER_INVALID)
      return false;
//---
   if(prevLayer.At(0).Type() != defNeuron)
     {
      CNeuronPool *temp = prevLayer.At(m_myIndex);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      prevLayer = temp.getOutputLayer();
      if(CheckPointer(prevLayer) == POINTER_INVALID)
         return false;
     }
//---
   CNeuronBase *prevNeuron, *outputNeuron;
   int total = prevLayer.Total();
   for(int i = 0; i < total; i++)
     {
      prevNeuron = prevLayer.At(i);
      if(CheckPointer(prevNeuron) == POINTER_INVALID)
         continue;
      double prev_gradient = 0;
      int start = i - iWindow + iStep;
      start = (start - start % iStep) / iStep;
      double stop = (i - i % iStep) / iStep + 1;
      for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
        {
         outputNeuron = OutputLayer.At(out);
         if(CheckPointer(outputNeuron) == POINTER_INVALID)
            continue;
         prev_gradient += outputNeuron.getGradient() / iWindow;
        }
      prevNeuron.setGradient(prev_gradient);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevNeuron.Type() != defNeuron)
     {
      CNeuronPool *temp = prevNeuron;
      return calcInputGradients(temp.getOutputLayer());
     }
//---
   CNeuronBase *outputNeuron;
   double prev_gradient = 0;
   int start = (int)index - iWindow + iStep;
   start = (start - start % iStep) / iStep;
   double stop = (index - index % iStep) / iStep + 1;
   for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
     {
      outputNeuron = OutputLayer.At(out);
      if(CheckPointer(outputNeuron) == POINTER_INVALID)
         continue;
      prev_gradient += outputNeuron.getGradient() / iWindow;
     }
   prevNeuron.setGradient(prev_gradient);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevLayer.At(0).Type() != defNeuron)
     {
      CNeuronPool *temp = prevLayer.At(m_myIndex);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      prevLayer = temp.getOutputLayer();
      if(CheckPointer(prevLayer) == POINTER_INVALID)
         return false;
     }
//---
   CNeuronBase *prevNeuron, *outputNeuron;
   CConnection *con;
   int total = prevLayer.Total();
   for(int i = 0; i < total; i++)
     {
      prevNeuron = prevLayer.At(i);
      if(CheckPointer(prevNeuron) == POINTER_INVALID)
         continue;
      double prev_gradient = 0;
      int start = i - iWindow + iStep;
      start = (start - start % iStep) / iStep;
      double stop = (i - i % iStep) / iStep + 1;
      for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
        {
         outputNeuron = OutputLayer.At(out);
         int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep;
         con = Connections.At(c);
         if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
            continue;
         prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight;
        }
      prevNeuron.setGradient(prev_gradient);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevNeuron.Type() != defNeuron)
     {
      CNeuronPool *temp = prevNeuron;
      return calcInputGradients(temp.getOutputLayer());
     }
//---
   CNeuronBase *outputNeuron;
   CConnection *con;
   double prev_gradient = 0;
   int start = (int)index - iWindow + iStep;
   start = (start - start % iStep) / iStep;
   double stop = (index - index % iStep) / iStep + 1;
   for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
     {
      outputNeuron = OutputLayer.At(out);
      int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep);
      con = Connections.At(c);
      if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
         continue;
      prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight;
     }
   prevNeuron.setGradient(prev_gradient);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::Save(int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
   if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE)
      return false;
//---
   return Connections.Save(file_handle);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "LayerDescription.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNet
  {
protected:
   double            dLogitAdjust[3];
   bool              bLogitAdjust;
   void              backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0);
   bool              InitOpenCL(void);
   bool              InitDirectML(void);
   //--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference.
   bool              feedForwardCPU(CArrayDouble *inputVals);
public:
                     CNet(CArrayObj *Description);
                    ~CNet(void);
   bool              feedForward(CArrayDouble *inputVals);
   //--- sampleWeight scales this example's output-layer gradient before it propagates back through the
   //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling
   //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same
   //--- example multiple times).
   void              backProp(CArrayDouble *targetVals, double sampleWeight = 1.0);
   //--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class
   //--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass
   //--- only. Minimizing softmax CE on adjusted logits is consistent for BALANCED error - which is
   //--- exactly the metric checkpoint selection already ranks on (macro-recall), so this is the first
   //--- time the loss and the selection criterion optimize the same thing.
   //--- It replaces minority REPLAY (which duplicated rare bars up to 28x and made Buy and Sell
   //--- compete for the same replicated capacity - the measured failure was each model taking one
   //--- direction to ~50% recall and abandoning the other, with the direction chosen arbitrarily) and
   //--- the post-hoc inference prior, which becomes double-counting once the offsets are trained in.
   //--- Offsets are NEGATIVE (log of a probability), so a rare class gets its logit pushed DOWN during
   //--- training, forcing the weights to produce a larger raw logit to compensate. At inference the
   //--- offsets are absent, so that surplus becomes the calibrated boost the rare class needs.
   void              SetLogitAdjustment(const double &offsets[]);
   void              ClearLogitAdjustment(void)   {  bLogitAdjust = false;  }
   bool              HasLogitAdjustment(void)     {  return bLogitAdjust;   }
   void              getResults(CArrayDouble *&resultVals) ;
   double            getRecentAverageError()  { return recentAverageError; }
   //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see
   //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array
   //--- when there is nothing to persist/restore.
   bool              Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]);
   //--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it
   //--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see
   //--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud.
   bool              Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false);
   //--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). CaptureWeights() snapshots
   //--- every neuron's weights (base/conv/LSTM) into host arrays; RestoreWeights() writes them back IN
   //--- PLACE via setWeights - reusing the existing neuron objects and their already-allocated device
   //--- buffers, exactly like BlendWeightsFrom(). This REPLACED an earlier file-based checkpoint pair
   //--- (since removed - it had no callers left) for the mid-run stability restore, because a file path RE-CREATES every neuron on
   //--- load (fresh CLayer + Init), and the multithreaded CPU-DLL backend (CDirectMLMy/WarriorCPU.dll)
   //--- cannot allocate a second full set of neuron tensors while the live set still exists - so the
   //--- file restore failed ("read 0 layers"), the model could never roll back a regressed era, and it
   //--- drifted into a Neutral collapse. In-place weight copy uses only getWeights/setWeights, which the
   //--- per-era shadow blend already exercises successfully on that backend. Snapshots WEIGHTS only (not
   //--- Adam moments); the regression handler decays eta on restore and per-step deltas are clipped, so
   //--- stale moments can't overshoot. In-memory => valid only within a single Train() run (same as the
   //--- ephemeral _ckpt.tmp was), which is exactly its scope.
   bool              CaptureWeights(void);
   bool              RestoreWeights(void);
   bool              HaveWeightSnapshot(void) const { return m_haveWeightSnapshot; }
   //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another
   //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight +
   //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS
   //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops
   //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the
   //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit
   //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over
   //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation.
   //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) -
   //--- true whenever the shadow was cloned from live via Save()/Load() and never independently
   //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a
   //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers.
   bool              BlendWeightsFrom(CNet &live, double tau);
   //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly
   //--- initialized and carry the real learning signal) of each output neuron's incoming weight block,
   //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site
   //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to
   //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more
   //--- non-majority classes than the true base rate warrants until backProp corrects it over many
   //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the
   //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns
   //--- false (no-op) rather than corrupt anything if that assumption doesn't hold.
   bool              SeedOutputLayerBias(const double &biasValues[]);
   //--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Set BEFORE Load() in an inference-only
   //--- backtest: it makes InitOpenCL()/InitDirectML() no-op (both backends stay NULL), so the OCL
   //--- neurons load their weights host-side only and feedForward() runs the double-precision MQL5
   //--- path (feedForwardCPU) reading those same host buffers. Training/optimization never set this
   //--- (they always want a backend), so their behaviour is unchanged. See ExpertSignalAIBase.mqh's
   //--- inference-only wiring and the deploy-time validation that gates it.
   void              SetCpuInference(bool v)       {  m_cpuInference = v;   }
   bool              CpuInference(void)      const {  return m_cpuInference; }
   //--- Re-assert the output layer's activation after a Load(), and report what it used to be.
   //--- WHY THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. CNeuronBase::Save/
   //--- CNeuronBaseOCL::Save write (int)activation per neuron and the matching Load() reads it straight
   //--- back into the live object, so the activation chosen in BuildFreshTopology() only ever applies to
   //--- a genuinely NEW topology. Every reload restores whatever is on disk and the next Save() writes it
   //--- back out - a wrong value can never heal on its own, while the source file reads as though it were
   //--- already fixed. That is exactly how models kept training with an unbounded NONE classification head
   //--- for a full day after BuildFreshTopology() had been reverted to SIGMOID (2026-07-29): confirmed by
   //--- parsing the binaries, `layer N: BaseOCL act=NONE out=3`, while a freshly reset model of the same
   //--- config read act=SIGMOID. Symptom was negative "OOS raw out" values (impossible under sigmoid)
   //--- escalating to a 4.1e13 logit spread with all three classes numerically identical.
   //--- Only the output layer is repaired here: it is always a plain dense layer whose activation is a
   //--- single unambiguous expression in BuildFreshTopology(). Hidden layers are deliberately left alone -
   //--- they legitimately differ per stage (PRELU dense, PRELU conv, NONE pool, TANH LSTM), so blanket
   //--- re-assertion there would corrupt exactly the topologies it was meant to protect.
   //--- Returns true when a repair was actually made, and reports the stale value through `previous`.
   bool              EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous);
   //--- Freeze/unfreeze every batch-normalization layer's running statistics (AI\NeuronBatchNorm.mqh).
   //--- Frozen, a forward pass is a pure function of its input; unfrozen (the default) it also advances
   //--- the statistics. Anything that COMPARES two forward passes must freeze first or it measures its
   //--- own side effect. No-op on a net with no normalization layers.
   void              SetBatchNormFrozen(bool frozen);
   //---
   static double     recentAverageSmoothingFactor;

private:
   CArrayLayer       *layers;
   COpenCLMy         *opencl;
   CDirectMLMy       *directml;
   double            recentAverageError;
   bool              m_cpuInference;
   //--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per
   //--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture.
   CArrayObj        *m_weightSnapshot;
   bool              m_haveWeightSnapshot;
   //--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process).
   //--- A run legitimately builds SEVERAL CNet objects - the main Net, the EMA shadow net, the OOS-sim
   //--- clone, the deploy-time MQL5-inference self-check clone - and each one probed the backends
   //--- independently. On a host without OpenCL that reprinted the same 3-line banner per net
   //--- ("OpenCL not found, error code=5100" comes from the STDLIB COpenCL::Initialize, so it can't be
   //--- silenced at our call site), which read in the log like the EA was initializing twice.
   //--- s_openclUnavailable: latched only on FAILURE, and only ever skips a probe that is already known
   //--- to fail - OpenCL availability cannot change inside a process. A host that HAS OpenCL never
   //--- latches, so every CNet still gets its own COpenCLMy. Skipping also keeps GetLastError() free of
   //--- the harmless 5100 for later callers.
   //--- s_computeTierLogged: suppresses only the repeat of the informational "tier active" line (the
   //--- tier is a property of the HOST, identical for every net). Failure messages stay loud every time.
   static bool       s_openclUnavailable;
   static bool       s_computeTierLogged;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over
bool CNet::s_openclUnavailable = false;
bool CNet::s_computeTierLogged = false;
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNet::CNet(CArrayObj *Description)
  {
   //--- Before the early returns below: CNet(NULL) is a legitimate construction (see
   //--- InitNeuralNetwork) and must still leave the adjustment cleanly disabled.
   bLogitAdjust = false;
   ArrayInitialize(dLogitAdjust, 0.0);
   if(CheckPointer(Description) == POINTER_INVALID)
      return;
//---
   int total = Description.Total();
   if(total <= 0)
      return;
//---
   layers = new CArrayLayer();
   if(CheckPointer(layers) == POINTER_INVALID)
      return;
//---
   CLayer *temp;
   CLayerDescription *desc = NULL, *next = NULL, *prev = NULL;
   CNeuronBase *neuron = NULL;
   CNeuronPool *neuron_p = NULL;
   int output_count = 0;
   int temp_count = 0;
//---
   next = Description.At(1);
   if(CheckPointer(next) != POINTER_INVALID &&
      (next.type == defNeuron || next.type == defNeuronBaseOCL || next.type == defNeuronConv || next.type == defNeuronConvOCL ||
       next.type == defNeuronLSTM || next.type == defNeuronBatchNorm || next.type == defNeuronBatchNormOCL))
     {
      //--- OpenCL first, DirectML/D3D12 next, plain CPU as the final fallback
      if(!InitOpenCL())
         InitDirectML();
     }
   //--- Batch normalization exists only in the OCL neuron model (AI\NeuronBatchNorm.mqh); the legacy
   //--- scalar CNeuron path below has no counterpart. Rather than silently build a DIFFERENT network
   //--- than the topology asked for - which is precisely the class of bug that had models training
   //--- against a stale output head for a day - refuse, loudly, and leave an empty net behind. Callers
   //--- already treat a 0-layer net as a hard failure. Reaching here means OpenCL, DirectML AND the
   //--- CPU DLL all failed to initialize, which this project does not support for training anyway.
   if(CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID)
     {
      for(int i = 0; i < total; i++)
        {
         CLayerDescription *probe = Description.At(i);
         if(CheckPointer(probe) != POINTER_INVALID &&
            (probe.type == defNeuronBatchNorm || probe.type == defNeuronBatchNormOCL))
           {
            Print(__FUNCTION__ + ": REFUSED - topology requests a batch-normalization layer but no compute"
                  " backend initialized (no OpenCL, no DirectML, no CPU DLL). Batch norm has no scalar-CPU"
                  " implementation; building without it would silently train a different architecture.");
            return;
           }
        }
     }
//---
   for(int i = 0; i < total; i++)
     {
      prev = desc;
      desc = Description.At(i);
      if((i + 1) < total)
        {
         next = Description.At(i + 1);
         if(CheckPointer(next) == POINTER_INVALID)
            return;
        }
      else
         next = NULL;
      //--- How many outgoing weights this layer carries. The convention in this engine is that the
      //--- weight matrix feeding layer L is stored ON layer L-1, so only a DENSE successor claims one:
      //--- conv/pool/LSTM own their weights internally, and batch norm is elementwise and has none at
      //--- all (just gamma/beta, which live in its own parameter block). Getting this wrong for the new
      //--- type would allocate a full dense matrix that nothing ever reads or trains.
      int outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL) ? 0 : next.count);
      temp = new CLayer(outputs);
      int neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL ? 1 : 0));
      if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
        {
         CNeuronBaseOCL *neuron_ocl = NULL;
         switch(desc.type)
           {
            case defNeuron:
            case defNeuronBaseOCL:
               neuron_ocl = new CNeuronBaseOCL();
               if(CheckPointer(neuron_ocl) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               if(CheckPointer(opencl) != POINTER_INVALID
                  ? !neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization)
                  : !neuron_ocl.Init(outputs, 0, directml, desc.count, desc.optimization))
                 {
                  delete temp;
                  return;
                 }
               neuron_ocl.SetActivationFunction(desc.activation);
               if(!temp.Add(neuron_ocl))
                 {
                  delete neuron_ocl;
                  delete temp;
                  return;
                 }
               neuron_ocl = NULL;
               break;
            case defNeuronBatchNorm:
            case defNeuronBatchNormOCL:
              {
               CNeuronBatchNormOCL *neuron_bn = new CNeuronBatchNormOCL();
               if(CheckPointer(neuron_bn) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               //--- Elementwise, so this layer is exactly as wide as the one below it. Take that width
               //--- from the layer already built rather than from desc.count: a conv or pool stage's
               //--- output size is derived HERE (the sliding-window arithmetic above), so the topology
               //--- builder in ExpertSignalAIBase.mqh has no way to know it and cannot state it. Falls
               //--- back to desc.count only for the impossible case of a batch-norm layer at index 0.
               int bnUnits = desc.count;
               if(layers.Total() > 0)
                 {
                  CLayer *below = layers.At(layers.Total() - 1);
                  if(CheckPointer(below) != POINTER_INVALID && below.Total() > 0)
                    {
                     CNeuronBaseOCL *belowNeuron = below.At(0);
                     if(CheckPointer(belowNeuron) != POINTER_INVALID && belowNeuron.Neurons() > 0)
                        bnUnits = belowNeuron.Neurons();
                    }
                 }
               bool bnInit = (CheckPointer(opencl) != POINTER_INVALID
                              ? neuron_bn.Init(outputs, 0, opencl, bnUnits, desc.batch, desc.optimization)
                              : neuron_bn.Init(outputs, 0, directml, bnUnits, desc.batch, desc.optimization));
               if(!bnInit)
                 {
                  delete neuron_bn;
                  delete temp;
                  return;
                 }
               if(!temp.Add(neuron_bn))
                 {
                  delete neuron_bn;
                  delete temp;
                  return;
                 }
               neuron_bn = NULL;
               //--- Keep the running conv/pool sizing cursor pointing at this layer's real width, so a
               //--- conv or pool stage placed ABOVE a batch-norm layer still sizes correctly.
               output_count = bnUnits;
               break;
              }
            case defNeuronConv:
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *neuron_conv = new CNeuronConvOCL();
               if(CheckPointer(neuron_conv) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               //--- number of sliding positions - same formula the CPU CNeuronConv path uses
               if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
                 {
                  int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
                  temp_count = (prevCount - desc.window) % desc.step;
                  output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               else
                 {
                  temp_count = (output_count - desc.window) % desc.step;
                  output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               bool convInit = (CheckPointer(opencl) != POINTER_INVALID
                                 ? neuron_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, output_count, desc.optimization)
                                 : neuron_conv.Init(outputs, 0, directml, desc.window, desc.step, desc.count, output_count, desc.optimization));
               if(!convInit)
                 {
                  delete neuron_conv;
                  delete temp;
                  return;
                 }
               neuron_conv.SetActivationFunction(desc.activation);
               if(!temp.Add(neuron_conv))
                 {
                  delete neuron_conv;
                  delete temp;
                  return;
                 }
               neuron_conv = NULL;
               break;
              }
            case defNeuronPool:
            case defNeuronPoolOCL:
              {
               CNeuronPoolOCL *neuron_pool = new CNeuronPoolOCL();
               if(CheckPointer(neuron_pool) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               //--- number of sliding positions - same formula the CPU CNeuronPool path uses
               if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
                 {
                  int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
                  temp_count = (prevCount - desc.window) % desc.step;
                  output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               else
                 {
                  temp_count = (output_count - desc.window) % desc.step;
                  output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               bool poolInit = (CheckPointer(opencl) != POINTER_INVALID
                                ? neuron_pool.Init(outputs, 0, opencl, desc.window, desc.step, output_count, desc.optimization)
                                : neuron_pool.Init(outputs, 0, directml, desc.window, desc.step, output_count, desc.optimization));
               if(!poolInit)
                 {
                  delete neuron_pool;
                  delete temp;
                  return;
                 }
               if(!temp.Add(neuron_pool))
                 {
                  delete neuron_pool;
                  delete temp;
                  return;
                 }
               neuron_pool = NULL;
               break;
              }
            case defNeuronLSTM:
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *neuron_lstm = new CNeuronLSTMOCL();
               if(CheckPointer(neuron_lstm) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               bool lstmInit = (CheckPointer(opencl) != POINTER_INVALID
                                 ? neuron_lstm.Init(outputs, 0, opencl, desc.count, desc.optimization)
                                 : neuron_lstm.Init(outputs, 0, directml, desc.count, desc.optimization));
               if(!lstmInit)
                 {
                  delete neuron_lstm;
                  delete temp;
                  return;
                 }
               if(!temp.Add(neuron_lstm))
                 {
                  delete neuron_lstm;
                  delete temp;
                  return;
                 }
               neuron_lstm = NULL;
               break;
              }
            default:
               return;
               break;
           }
        }
      else
         for(int n = 0; n < neurons; n++)
           {
            switch(desc.type)
              {
               case defNeuron:
                  neuron = new CNeuron();
                  if(CheckPointer(neuron) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  //--- He-scaled init, matching CNeuronBaseOCL::Init's rationale - fan-in is this
                  //--- layer's own neuron count (bias already included via the `neurons` count above).
                  neuron.Init(outputs, n, desc.optimization, MathSqrt(2.0 / (double)neurons));
                  neuron.SetActivationFunction(desc.activation);
                  break;
               case defNeuronConv:
                  neuron_p = new CNeuronConv();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  if(CheckPointer(prev) != POINTER_INVALID)
                    {
                     if(prev.type == defNeuron)
                       {
                        temp_count = (int)((prev.count - desc.window) % desc.step);
                        output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                       }
                     else
                        if(n == 0)
                          {
                           temp_count = (int)((output_count - desc.window) % desc.step);
                           output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                          }
                    }
                  if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
               case defNeuronPool:
                  neuron_p = new CNeuronPool();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  if(CheckPointer(prev) != POINTER_INVALID)
                    {
                     if(prev.type == defNeuron)
                       {
                        temp_count = (int)((prev.count - desc.window) % desc.step);
                        output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                       }
                     else
                        if(n == 0)
                          {
                           temp_count = (int)((output_count - desc.window) % desc.step);
                           output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                          }
                    }
                  if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
               case defNeuronLSTM:
                  neuron_p = new CNeuronLSTM();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  output_count = (next != NULL ? next.window : desc.step);
                  if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
              }
            if(!temp.Add(neuron))
              {
               delete temp;
               delete layers;
               return;
              }
            neuron = NULL;
           }
      if(!layers.Add(temp))
        {
         delete temp;
         delete layers;
         return;
        }
     }
//---
  }
//+------------------------------------------------------------------+
//| Tries to initialize OpenCL; on any failure (no GPU, driver     |
//| missing, kernel build error) frees it and leaves opencl==NULL  |
//| so the rest of CNet transparently runs its CPU code path.      |
//+------------------------------------------------------------------+
bool CNet::InitOpenCL(void)
  {
   //--- pure-MQL5 inference: deliberately refuse a backend so the OCL neurons compute host-side.
   if(m_cpuInference)
      return false;
   if(CheckPointer(opencl) != POINTER_INVALID)
      return true;
//--- already established (by an earlier CNet in this process) that this host has no OpenCL - skip the
//--- probe rather than re-run a known failure and reprint the stdlib's banner. See s_openclUnavailable.
   if(s_openclUnavailable)
      return false;
//---
   opencl = new COpenCLMy();
   if(CheckPointer(opencl) == POINTER_INVALID || !opencl.Initialize(cl_program, true))
     {
      if(CheckPointer(opencl) != POINTER_INVALID)
         delete opencl;
      opencl = NULL;
      s_openclUnavailable = true;
      PrintFormat("%s: OpenCL unavailable, falling back to CPU", __FUNCTION__);
      return false;
     }
//--- create kernels
   opencl.SetKernelsCount(18);
   opencl.KernelCreate(def_k_FeedForward, "FeedForward");
   opencl.KernelCreate(def_k_CaclOutputGradient, "CaclOutputGradient");
   opencl.KernelCreate(def_k_CaclHiddenGradient, "CaclHiddenGradient");
   opencl.KernelCreate(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum");
   opencl.KernelCreate(def_k_UpdateWeightsAdam, "UpdateWeightsAdam");
   opencl.KernelCreate(def_k_FeedForwardConv, "FeedForwardConv");
   opencl.KernelCreate(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv");
   opencl.KernelCreate(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum");
   opencl.KernelCreate(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam");
   opencl.KernelCreate(def_k_LSTM_Gates, "LSTM_Gates");
   opencl.KernelCreate(def_k_LSTM_State, "LSTM_State");
   opencl.KernelCreate(def_k_LSTM_GateGradient, "LSTM_GateGradient");
   opencl.KernelCreate(def_k_LSTM_WeightsGradient, "LSTM_WeightsGradient");
   opencl.KernelCreate(def_k_LSTM_InputsGradient, "LSTM_InputsGradient");
   opencl.KernelCreate(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam");
   opencl.KernelCreate(def_k_LSTM_UpdateWeightsMomentum, "LSTM_UpdateWeightsMomentum");
   opencl.KernelCreate(def_k_FeedForwardProof, "FeedForwardProof");
   opencl.KernelCreate(def_k_CalcInputGradientProof, "CalcInputGradientProof");
   return true;
  }
//+------------------------------------------------------------------+
//| Second-tier GPU fallback: only tried when OpenCL init failed.    |
//| Requires DirectML\WarriorDML.dll in the terminal's Libraries     |
//| folder (build it with DirectML\build.bat); on any failure frees  |
//| itself and leaves directml==NULL so CNet falls through to CPU.   |
//+------------------------------------------------------------------+
bool CNet::InitDirectML(void)
  {
   //--- pure-MQL5 inference: deliberately refuse a backend so the OCL neurons compute host-side.
   if(m_cpuInference)
      return false;
   //--- Tester/optimization/forward runs must not touch WarriorDML/WarriorCPU DLL imports.
   //--- This avoids agent-side file-lock/synchronization failures on rapid stop/restart cycles.
   if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD))
      return false;
   if(CheckPointer(directml) != POINTER_INVALID)
      return true;
//---
   directml = new CDirectMLMy();
   if(CheckPointer(directml) == POINTER_INVALID)
      return false;
   directml.SetCpuLoadPercent(EffectiveCpuLoadPercent());
   if(!directml.Initialize())
     {
      int err = directml.LastError();
      delete directml;
      directml = NULL;
      string reason;
      switch(err)
        {
         case 1: reason = "CreateDXGIFactory1 failed";                                  break;
         case 2: reason = "no DX12 hardware adapter found (feature level 11_0)";        break;
         case 3: reason = "compute command queue creation failed";                      break;
         case 4: reason = "command allocator creation failed";                          break;
         case 5: reason = "command list creation failed";                               break;
         case 6: reason = "fence creation failed";                                      break;
         case 7: reason = "fence event creation failed";                                break;
         case 8: reason = "HLSL kernel compile/PSO creation failed";                    break;
         default: reason = "neither WarriorDML.dll nor WarriorCPU.dll loaded (check Libraries folder / \"Allow DLL imports\")";
        }
      PrintFormat("%s: DirectML/D3D12 and CPU DLL both unavailable (%s), falling back to slow per-object CPU path", __FUNCTION__, reason);
      return false;
     }
//--- Announce the tier ONCE per process: it describes the host, not this particular net, and a run builds
//--- several nets (main + EMA shadow + sim/self-check clones). See s_computeTierLogged.
   if(!s_computeTierLogged)
     {
      s_computeTierLogged = true;
      if(directml.Tier() == COMPUTE_TIER_CPU)
        {
         //--- Report the SPLIT, not just the result. When several charts train at once this is the
         //--- number that explains their speed, and it is the one that was silently wrong before.
         int share = EffectiveCpuLoadPercent();
         PrintFormat("%s: DirectML/D3D12 unavailable, using multithreaded CPU DLL fallback (%d threads per network, target %d; %d%% of %d detected cores - fixed, independent of how many charts run)",
                     __FUNCTION__, directml.CpuThreadsUsed(), CPU_THREADS_PER_NETWORK, share, (int)TerminalInfoInteger(TERMINAL_CPU_CORES));
        }
      else
         PrintFormat("%s: DirectML/D3D12 GPU tier active", __FUNCTION__);
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::feedForward(CArrayDouble *inputVals)
  {
   if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1)
      return false;
   //--- Pure-MQL5 inference: OCL-format neurons loaded host-only, computed in double precision. Separate
   //--- path because the branches below assume either a device backend or the plain CNeuronBase model.
   if(m_cpuInference)
      return feedForwardCPU(inputVals);
//---
   CLayer *previous = NULL;
   CLayer *current = layers.At(0);
   int total = MathMin(current.Total(), inputVals.Total());
   CNeuronBase *neuron = NULL;
   bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID);
   if(!gpuActive)
     {
      for(int i = 0; i < total; i++)
        {
         neuron = current.At(i);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return false;
         neuron.setOutputVal(inputVals.At(i));
        }
     }
   else
     {
      CNeuronBaseOCL *neuron_ocl = current.At(0);
      int total_data = inputVals.Total();
      bool written;
      if(CheckPointer(opencl) != POINTER_INVALID)
        {
         //--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own
         //--- BufferWrite(), this call bypasses that class entirely (it writes straight into the
         //--- first layer's Output buffer by index), so the narrow-to-float has to happen here too.
         float array[];
         if(ArrayResize(array, total_data) < 0)
            return false;
         for(int d = 0; d < total_data; d++)
            array[d] = (float)inputVals.At(d);
         written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data);
        }
      else
        {
         double array[];
         if(ArrayResize(array, total_data) < 0)
            return false;
         for(int d = 0; d < total_data; d++)
            array[d] = inputVals.At(d);
         written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data);
        }
      if(!written)
         return false;
     }
//---
   CObject *temp = NULL;
   for(int l = 1; l < layers.Total(); l++)
     {
      previous = current;
      current = layers.At(l);
      if(CheckPointer(current) == POINTER_INVALID)
         return false;
      //---
      if(gpuActive)
        {
         CNeuronBaseOCL *current_ocl = current.At(0);
         if(!current_ocl.feedForward(previous.At(0)))
            return false;
         continue;
        }
      //---
      total = current.Total();
      if(current.At(0).Type() == defNeuron)
         total--;
      //---
      for(int n = 0; n < total; n++)
        {
         neuron = current.At(n);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return false;
         if(previous.At(0).Type() == defNeuron)
           {
            temp = previous;
            if(!neuron.feedForward(temp))
               return false;
            continue;
           }
         if(neuron.Type() == defNeuron)
           {
            if(n == 0)
              {
               CLayer *temp_l = new CLayer(total);
               if(CheckPointer(temp_l) == POINTER_INVALID)
                  return false;
               CNeuronPool *Pool = NULL;
               for(int p = 0; p < previous.Total(); p++)
                 {
                  Pool = previous.At(p);
                  if(CheckPointer(Pool) == POINTER_INVALID)
                     return false;
                  temp_l.AddArray(Pool.getOutputLayer());
                 }
               temp = temp_l;
              }
            if(!neuron.feedForward(temp))
               return false;
            if(n == total - 1)
              {
               CLayer *temp_l = temp;
               temp_l.FreeMode(false);
               temp_l.Shutdown();
               delete temp_l;
              }
            continue;
           }
         temp = previous.At(n);
         if(CheckPointer(temp) == POINTER_INVALID)
            return false;
         if(!neuron.feedForward(temp))
            return false;
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| Pure-MQL5 forward pass over an OCL-format network loaded host-only |
//| (no OpenCL/DirectML/DLL). Layer 0 is fed from inputVals; each      |
//| subsequent layer's OCL neuron computes via its virtual            |
//| feedForwardCPU() (dense/conv/pool/LSTM). See SetCpuInference().    |
//+------------------------------------------------------------------+
bool CNet::feedForwardCPU(CArrayDouble *inputVals)
  {
   CLayer *current = layers.At(0);
   if(CheckPointer(current) == POINTER_INVALID)
      return false;
   CNeuronBaseOCL *in0 = current.At(0);
   if(CheckPointer(in0) == POINTER_INVALID || !in0.SetInputsCPU(inputVals))
      return false;
   for(int l = 1; l < layers.Total(); l++)
     {
      CLayer *previous = current;
      current = layers.At(l);
      if(CheckPointer(current) == POINTER_INVALID)
         return false;
      CNeuronBaseOCL *cur = current.At(0);
      CNeuronBaseOCL *prev = previous.At(0);
      if(CheckPointer(cur) == POINTER_INVALID || CheckPointer(prev) == POINTER_INVALID)
         return false;
      if(!cur.feedForwardCPU(prev))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::backProp(CArrayDouble *targetVals, double sampleWeight)
  {
   if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
      return;
   if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
     {
      backPropOCL(targetVals, sampleWeight);
      return;
     }
//---
   CLayer *outputLayer = layers.At(layers.Total() - 1);
   if(CheckPointer(outputLayer) == POINTER_INVALID)
      return;
//---
   //--- Defensive: this is the pure-MQL5 CPU backward pass and it walks CNeuron/CNeuronBase objects
   //--- with SCALAR getOutputVal()/getGradient()/setGradient() accessors. CNeuronBaseOCL & friends do
   //--- NOT derive from CNeuronBase and expose only ARRAY accessors over a device buffer, so they can
   //--- never be walked here. That is normally impossible to reach: the early return above hands any
   //--- backend-backed net to backPropOCL(), and CNet::Create() only ever constructs OCL neurons when
   //--- a backend exists. The one way to hold OCL neurons with no backend is pure-MQL5 inference mode
   //--- (SetCpuInference -> CLayer::CreateElementScaled's host-only branch), which is inference-only
   //--- and never backprops (OnlineLearnStep() guards on Net.CpuInference()). Bail out loudly rather
   //--- than mis-cast if that ever changes.
   //--- 2026-07-28: a set of inline "handle OCL neuron types" branches was added throughout this
   //--- function to cover that impossible case. They could not compile - they called the scalar
   //--- accessors on CNeuronBaseOCL - and were removed; this guard replaces them.
   CObject *probe = outputLayer.At(0);
   if(CheckPointer(probe) != POINTER_INVALID)
     {
      int t0 = probe.Type();
      if(t0 == defNeuronBaseOCL || t0 == defNeuronConvOCL || t0 == defNeuronPoolOCL || t0 == defNeuronLSTMOCL ||
         t0 == defNeuronBatchNormOCL)
        {
         Print(__FUNCTION__ + ": REFUSED - CPU backward pass reached a net built from OpenCL/DirectML neurons with no compute backend attached. Nothing was trained this step.");
         return;
        }
     }
//---
   double error = 0.0;
   int total = outputLayer.Total() - 1;
   //--- 3-output classification case: true softmax + categorical-cross-entropy gradient
   //--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section
   //--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID
   //--- (bounded, avoids the historical logit-runaway collapse documented at
   //--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD
   //--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly,
   //--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/
   //--- Neutral together during training: raising one class's softmax probability now structurally
   //--- lowers the other two's (via the shared normalizing sum), giving real competition instead of
   //--- three independent binary regressions that can all drift toward "predict Neutral" together -
   //--- root cause of the "overshoot to all-Neutral" convergence failure this replaces.
   bool useSoftmaxGrad = (total == 3);
   double smax[3];
   if(useSoftmaxGrad)
     {
      //--- Logit adjustment added BEFORE the max-subtraction so the shift stays numerically safe.
      double logit[3];
      double maxLogit = -DBL_MAX;
      for(int n = 0; n < 3; n++)
        {
         CNeuron *nrn = outputLayer.At(n);
         logit[n] = CLASS_LOGIT_SCALE * nrn.getOutputVal() + (bLogitAdjust ? dLogitAdjust[n] : 0.0);
         maxLogit = MathMax(maxLogit, logit[n]);
        }
      double sum = 0.0;
      for(int n = 0; n < 3; n++)
        {
         smax[n] = exp(logit[n] - maxLogit);
         sum += smax[n];
        }
      for(int n = 0; n < 3; n++)
         smax[n] /= sum;
     }
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      CNeuron *neuron = outputLayer.At(n);
      double target = targetVals.At(n);
      double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
      double delta = clampedTarget - neuron.getOutputVal();
      error += delta * delta;
      if(useSoftmaxGrad)
         neuron.setGradient(clampedTarget - smax[n]);
      else
         neuron.calcOutputGradients(targetVals.At(n));
      //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
      //--- sampleWeight is derived) - scales the just-computed output gradient in place, before the
      //--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted
      //--- signal without needing its own separate weighting logic.
      if(sampleWeight != 1.0)
         neuron.setGradient(neuron.getGradient() * sampleWeight);
     }
   error /= total;
   error = sqrt(error);
   recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
//---
   CNeuronBase *neuron = NULL;
   CObject *temp = NULL;
   for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--)
     {
      CLayer *hiddenLayer = layers.At(layerNum);
      CLayer *nextLayer = layers.At(layerNum + 1);
      total = hiddenLayer.Total();
      for(int n = 0; n < total && !IsStopped(); ++n)
        {
         neuron = hiddenLayer.At(n);
         if(nextLayer.At(0).Type() == defNeuron)
           {
            temp = nextLayer;
            neuron.calcHiddenGradients(temp);
            continue;
           }
         if(neuron.Type() == defNeuron)
           {
            double g = 0;
            for(int i = 0; i < nextLayer.Total(); i++)
              {
               temp = nextLayer.At(i);
               neuron.calcHiddenGradients(temp);
               g += neuron.getGradient();
              }
            neuron.setGradient(g);
            continue;
           }
         temp = nextLayer.At(n);
         neuron.calcHiddenGradients(temp);
        }
     }
//---
   for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--)
     {
      CLayer *layer = layers.At(layerNum);
      CLayer *prevLayer = layers.At(layerNum - 1);
      total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0);
      int n_conv = 0;
      for(int n = 0; n < total && !IsStopped(); n++)
        {
         neuron = layer.At(n);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return;
         if(neuron.Type() == defNeuronPool)
            continue;
         switch(prevLayer.At(0).Type())
           {
            case defNeuron:
               temp = prevLayer;
               neuron.updateInputWeights(temp);
               break;
            case defNeuronConv:
            case defNeuronPool:
            case defNeuronLSTM:
               if(neuron.Type() == defNeuron)
                 {
                  for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++)
                    {
                     temp = prevLayer.At(n_conv);
                     neuron.updateInputWeights(temp);
                    }
                 }
               else
                 {
                  temp = prevLayer.At(n);
                  neuron.updateInputWeights(temp);
                 }
               break;
            default:
               temp = NULL;
               break;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight)
  {
   if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID ||
      (CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID))
      return;
   CLayer *currentLayer = (CLayer*)layers.At(layers.Total() - 1);
   if(CheckPointer(currentLayer) == POINTER_INVALID)
      return;
//---
   double error = 0.0;
   int total = targetVals.Total();
   double result[];
   CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)currentLayer.At(0);
   if(neuron.getOutputVal(result) < total)
      return;
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      double target = targetVals.At(n);
      // Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever
      // target==0, so a one-hot classification target only ever counted the true class's own
      // error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric,
      // which is what CNet::backProp()'s CPU-fallback path computes for every output
      // unconditionally, and is what actually drives the dError<0.1 convergence gate in
      // ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp
      // / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was.
      double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n];
      error += MathPow(delta, 2);
     }
   error /= total;
   error = sqrt(error);
   recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
   if(!neuron.calcOutputGradients(targetVals))
      return;
   //--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true
   //--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5
   //--- from the raw outputs already read back into result[] above - see the matching CNet::backProp()
   //--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared
   //--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels
   //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted
   //--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely
   //--- on the MQL5 side, so none of the 3 compute backends need to change.
   if(total == 3)
     {
      //--- Logit adjustment (see SetLogitAdjustment): tau*log(prior_c) per class, added to the logit
      //--- before the softmax. Backward pass ONLY - the forward pass and every inference path stay
      //--- untouched, which is the whole point: the network learns to absorb the offset, so at
      //--- inference its RAW argmax is already the balanced-error-optimal decision.
      double logit[3];
      double maxLogit = -DBL_MAX;
      for(int n = 0; n < 3; n++)
        {
         logit[n] = CLASS_LOGIT_SCALE * result[n] + (bLogitAdjust ? dLogitAdjust[n] : 0.0);
         maxLogit = MathMax(maxLogit, logit[n]);
        }
      double smax[3];
      double sm = 0.0;
      for(int n = 0; n < 3; n++)
        {
         smax[n] = exp(logit[n] - maxLogit);
         sm += smax[n];
        }
      double gradOverwrite[3];
      for(int n = 0; n < 3; n++)
        {
         smax[n] /= sm;
         double target = targetVals.At(n);
         double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
         gradOverwrite[n] = clampedTarget - smax[n];
        }
      neuron.setGradient(gradOverwrite);
     }
   //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
   //--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta
   //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the
   //--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers
   //--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends.
   if(sampleWeight != 1.0)
     {
      double gradVals[];
      int gradCount = neuron.getGradient(gradVals);
      if(gradCount > 0)
        {
         for(int g = 0; g < gradCount; g++)
            gradVals[g] *= sampleWeight;
         neuron.setGradient(gradVals);
        }
     }
//--- Calc Hidden Gradients
   CObject *temp = NULL;
   total = layers.Total();
   for(int layerNum = total - 2; layerNum > 0; layerNum--)
     {
      CLayer *nextLayer = currentLayer;
      currentLayer = layers.At(layerNum);
      neuron = currentLayer.At(0);
      neuron.calcHiddenGradients(nextLayer.At(0));
     }
//--- Layer-1 LSTM special case. The loop above deliberately stops at layerNum > 0 because the INPUT
//--- layer needs no gradient of its own - true for every layer type whose updateInputWeights() derives
//--- its own weight deltas from (own gradient x previous output) inside the kernel: dense
//--- (UpdateWeightsMomentum/Adam) and conv (UpdateWeightsConvMomentum/Adam) both do.
//--- CNeuronLSTMOCL is the ONE exception: LSTM_UpdateWeightsMomentum/Adam do not derive anything, they
//--- only CONSUME the WeightsGradient buffer, and that buffer is filled purely as a SIDE EFFECT of
//--- CNeuronLSTMOCL::calcInputGradients() - which is invoked by the layer BELOW, via its
//--- calcHiddenGradients(target=thisLSTM) dispatch. So an LSTM sitting at layer index 1 (the LSTM_2L
//--- preset: input -> LSTM -> dense -> dense -> output) never gets calcInputGradients() called at all:
//--- WeightsGradient stays at its BufferInit(total, 0) zeros forever, Adam's mt/vt therefore stay 0 and
//--- every weight delta is exactly 0. The LSTM silently never trains - it stays a frozen random
//--- recurrent projection while only the dense taper above it learns, which shows up as ~0% Buy/Sell
//--- recall and a raw-output range that barely moves off its initialisation.
//--- HYBRID_2L (input -> conv -> pool -> LSTM -> dense -> dense -> output) is unaffected: its LSTM is at
//--- index 3, so the pool layer below it makes the call in the normal course of the loop.
//--- Done here rather than by relaxing the loop bound so the loop's currentLayer/nextLayer bookkeeping
//--- is untouched, and so no other topology pays for an extra kernel dispatch it does not need.
   if(total >= 3)
     {
      CLayer *firstHidden = layers.At(1);
      CLayer *inputLayer = layers.At(0);
      if(CheckPointer(firstHidden) != POINTER_INVALID && CheckPointer(inputLayer) != POINTER_INVALID &&
         CheckPointer(firstHidden.At(0)) != POINTER_INVALID && CheckPointer(inputLayer.At(0)) != POINTER_INVALID &&
         firstHidden.At(0).Type() == defNeuronLSTMOCL)
        {
         CNeuronLSTMOCL *lstm = firstHidden.At(0);
         CNeuronBaseOCL *inputNeuron = inputLayer.At(0);
         //--- Safe to run now and only now: the loop above has already filled this LSTM's own Gradient
         //--- (it processed layerNum == 1), which is exactly what LSTMGateGradient reads.
         if(!lstm.calcInputGradients(inputNeuron))
            printf("%s: LSTM at layer 1 failed to compute its weight gradients - it will not train this step", __FUNCTION__);
        }
     }
//---
   CLayer *prevLayer = layers.At(total - 1);
   for(int layerNum = total - 1; layerNum > 0; layerNum--)
     {
      currentLayer = prevLayer;
      prevLayer = layers.At(layerNum - 1);
      neuron = currentLayer.At(0);
      neuron.updateInputWeights(prevLayer.At(0));
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::getResults(CArrayDouble *&resultVals)
  {
   if(CheckPointer(resultVals) == POINTER_INVALID)
     {
      resultVals = new CArrayDouble();
      if(CheckPointer(resultVals) == POINTER_INVALID)
         return;
     }
//---
   resultVals.Clear();
   if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0)
      return;
//---
   CLayer *output = layers.At(layers.Total() - 1);
   if(CheckPointer(output) == POINTER_INVALID)
      return;
//---
   if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
     {
      switch(output.At(0).Type())
        {
         case defNeuronBaseOCL:
         case defNeuronConvOCL:
         case defNeuronPoolOCL:
         case defNeuronLSTMOCL:
           {
            CNeuronBaseOCL *temp = output.At(0);
            temp.getOutputVal(resultVals);
            return;
           }
        }
     }
   //--- pure-MQL5 inference: OCL output neuron computed host-side, read without a device BufferRead.
   if(m_cpuInference)
     {
      switch(output.At(0).Type())
        {
         case defNeuronBaseOCL:
         case defNeuronConvOCL:
         case defNeuronPoolOCL:
         case defNeuronLSTMOCL:
           {
            CNeuronBaseOCL *temp = output.At(0);
            temp.GetOutputsCPU(resultVals);
            return;
           }
        }
     }
   CNeuronBase *neuron = NULL;
   CLayer *temp = NULL;
   int total = output.Total();
   if(output.At(0).Type() == defNeuron)
      total--;
//---
   for(int i = 0; i < total; i++)
     {
      CObject *obj = output.At(i);
      if(CheckPointer(obj) == POINTER_INVALID)
         continue;
      if(obj.Type() == defNeuron)
        {
         neuron = (CNeuronBase*)obj;
         resultVals.Add(neuron.getOutputVal());
         continue;
        }
      if(obj.Type() == defNeuronPool)
        {
         CNeuronPool *n = (CNeuronPool*)obj;
         temp = n.getOutputLayer();
         for(int ii = 0; ii < temp.Total(); ii++)
           {
            CObject *poolObj = temp.At(ii);
            if(CheckPointer(poolObj) == POINTER_INVALID)
               continue;
            CNeuronBase *poolNeuron = (CNeuronBase*)poolObj;
            resultVals.Add(poolNeuron.getOutputVal());
           }
        }
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[])
  {
   //--- only the shared FILE_COMMON production weights are protected from being overwritten by
   //--- backtest/optimization noise - a LOCAL (common=false) file is exactly what the tester's
   //--- per-agent cross-pass weight cache uses (see CExpertSignalAIBase::InitNeuralNetwork), and
   //--- must be allowed to write even inside the tester/optimizer.
   if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
      return true;
   if(file_name == NULL)
      return false;
   //--- Never persist a layerless network. FileOpen(FILE_WRITE) below TRUNCATES on open, so writing an
   //--- empty net here would replace a good ~18MB model on disk with a ~330-byte header-only stub (0
   //--- layers) - silent, total loss of trained weights. This guard is checked BEFORE FileOpen so a bad
   //--- in-memory state (e.g. a failed checkpoint restore that emptied the layers) can't clobber the file.
   if(CheckPointer(layers) == POINTER_INVALID || layers.Total() == 0)
     {
      Print("CNet::Save: refusing to write ", file_name, " - network has 0 layers (would overwrite a valid model with an empty stub)");
      return false;
     }
//--- ATOMIC SAVE (crash-safe) - see System\AtomicFile.mqh for why every write here is staged through a
//--- temp file. Short version: FileOpen(FILE_WRITE) truncates on open, so writing straight to file_name
//--- would zero the good model the instant the save starts, and an interrupted save then leaves a
//--- truncated .nnw that the next load rebuilds from era 0 (the recurring data loss).
   int commonFlag = (common ? FILE_COMMON : 0);
   string tmpName = "";
   int handle = AtomicWriteBegin(file_name, commonFlag, tmpName);
   if(handle == INVALID_HANDLE)
      return false;
//---
   bool ok = true;
   if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0 || FileWriteLong(handle, era) <= 0 ||
      FileWriteInteger(handle, trainingComplete ? 1 : 0) <= 0)
      ok = false;
//--- AutoTuneIndicators "winning" AD indicator params, same append-only pattern as the era/
//--- trainingComplete fields above: count-prefixed so older readers can still stop before this block
   int paramsCount = ArraySize(indicatorParams);
   if(ok && FileWriteInteger(handle, paramsCount) <= 0)
      ok = false;
   for(int p = 0; ok && p < paramsCount; p++)
      if(FileWriteDouble(handle, indicatorParams[p]) <= 0)
         ok = false;
   if(ok && !layers.Save(handle))
      ok = false;
   return AtomicWriteEnd(handle, file_name, tmpName, commonFlag, ok, "CNet::Save");
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet)
  {
   //--- see the matching comment in Save() - only shared FILE_COMMON production weights are blocked
   if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
      return false;
//---
   if(file_name == NULL)
      return false;
//---
   if(!quiet)
      Print(file_name);
   //--- FILE_SHARE_READ|FILE_SHARE_WRITE: this is a READ - it never needs exclusive access, and demanding
   //--- it (the default) makes the open fail with 5004 whenever any other process holds the file, which is
   //--- the normal state of a deployed model while a live chart runs this EA. See CopySharedFile().
   int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ | FILE_SHARE_READ | FILE_SHARE_WRITE);
   if(handle == INVALID_HANDLE)
     {
      //--- Every OTHER rejection below prints why; this one previously didn't, which is how a real cause
      //--- (a transient Windows sharing violation from a concurrent writer - e.g. a live chart's atomic
      //--- Save() renaming this same file mid-read) looked identical to "no file yet" in the journal. Name
      //--- it explicitly so a recurrence is diagnosable instead of a silent fall-through to an untrained net.
      if(!quiet)
         Print("CNet::Load: " + file_name + " - FileOpen failed, error " + IntegerToString(GetLastError()) +
               ". If this file is known to exist, this is likely a transient lock (e.g. another process " +
               "writing the same file concurrently) rather than a missing/corrupt file.");
      return false;
     }
//--- Capture the on-disk size up front: it discriminates the two failure modes a rejected read can have.
//--- A healthy multi-layer model is multi-MB; a size of only a few hundred bytes/KB = a TRUNCATED file
//--- (interrupted save). A full-size file that STILL fails to load layers = a backend ALLOCATION failure
//--- (e.g. the CPU-DLL still holding a prior net's tensors after a skipped teardown), NOT truncation.
   ulong fileSizeBytes = FileSize(handle);
//---
   error = FileReadDouble(handle);
   undefine = FileReadDouble(handle);
   forecast = FileReadDouble(handle);
   time = (datetime)FileReadLong(handle);
   era = FileReadLong(handle);
   // Older save files predate this field - FileReadInteger returns 0 past EOF,
   // which correctly defaults to "not complete" so a resumed run keeps training
   // instead of silently trusting an unfinished/unknown model as done.
   trainingComplete = (FileReadInteger(handle) != 0);
   // Older save files predate this block too - FileReadInteger returns 0 past EOF, which
   // correctly yields an empty indicatorParams (nothing to restore) instead of misreading weights.
   ArrayFree(indicatorParams);
   int paramsCount = FileReadInteger(handle);
   if(paramsCount > 0)
     {
      ArrayResize(indicatorParams, paramsCount);
      for(int p = 0; p < paramsCount; p++)
         indicatorParams[p] = FileReadDouble(handle);
     }
//---
   if(CheckPointer(layers) != POINTER_INVALID)
      layers.Clear();
   else
      layers = new CArrayLayer();
   int i = 0, num;
//---
   if(!InitOpenCL())
      InitDirectML();
//--- check
//--- read and check start marker - 0xFFFFFFFFFFFFFFFF
//--- DIAGNOSTIC: every failure below returns a bare false, and the caller can only surface GetLastError()
//--- which on a no-GPU/CPU-DLL box is polluted with the harmless 5100 from the OpenCL probe above - so a
//--- genuinely corrupt/partial/incompatible file looked identical to a missing one for several sessions of
//--- "restart -> era 0" hunting. Name the exact failure so the next restart is diagnosable at a glance.
   long temp = FileReadLong(handle);
   if(temp != -1)
     {
      FileClose(handle);
      if(!quiet)
         Print("CNet::Load: " + file_name + " - REJECTED: bad start marker (read " + IntegerToString(temp) +
               ", expected -1) => the header is corrupt or the file was truncated by an interrupted save. This is NOT a compute/OpenCL problem.");
      return(false);
     }
   //--- read and check array type
   int savedType = FileReadInteger(handle, INT_VALUE);
   if(savedType != layers.Type())
     {
      FileClose(handle);
      if(!quiet)
         Print("CNet::Load: " + file_name + " - REJECTED: layer-array type mismatch (read " + IntegerToString(savedType) +
               ", expected " + IntegerToString(layers.Type()) + ") => file format/version incompatible. This is NOT a compute/OpenCL problem.");
      return(false);
     }
//--- read array length
   num = FileReadInteger(handle, INT_VALUE);
//--- read array
   int failedLayer = -1; // index of the first layer that failed to load, or -1 if all loaded
   if(num != 0)
     {
      for(i = 0; i < num; i++)
        {
         //--- create new element
         CLayer *Layer = new CLayer(0, handle, opencl, directml);
         if(CheckPointer(Layer) == POINTER_INVALID)
           {
            failedLayer = i;
            break;
           }
         //--- On a partial/corrupt file, Layer.Load() or layers.Add() can fail after the CLayer was
         //--- already allocated. Deleting it before breaking prevents the orphaned-CLayer leak MT5 reports
         //--- at unload ("1 object of class 'CLayer'") - the layer was never added to `layers`, so the
         //--- CNet destructor (which frees `layers`) can't reclaim it.
         if(!Layer.Load(handle))
           {
            delete Layer;
            failedLayer = i;
            break;
           }
         if(!layers.Add(Layer))
           {
            delete Layer;
            failedLayer = i;
            break;
           }
        }
     }
   FileClose(handle);
//--- result: a 0-layer file is not a usable model (it's the empty stub the Save guard now prevents, but
//--- older stubs may still be on disk) - report failure so the caller rebuilds/retrains instead of
//--- running inference on a layerless network.
   if(num == 0)
     {
      if(!quiet)
         Print("CNet::Load: " + file_name + " - REJECTED: 0-layer stub (empty model, header says " +
               IntegerToString(num) + " layers). An older empty-save stub; the current Save guard prevents these. Use reset-weights to start clean. NOT a compute/OpenCL problem.");
      return(false);
     }
   if(failedLayer >= 0 || layers.Total() != num)
     {
      if(!quiet)
        {
         string backend = (CheckPointer(opencl) != POINTER_INVALID ? "OpenCL" : (CheckPointer(directml) != POINTER_INVALID ? "CPU-DLL" : "pure-MQL5"));
         //--- Which failure mode this is depends on WHERE it stopped, not just on the file size:
         //---  - failed at layer 0 with a full-size file: nothing was read yet, so the file cannot be the
         //---    cause. This is a CODE fault in the read path - historically CLayer::CreateElement losing
         //---    its virtual override of CArrayObj::CreateElement (see the note on that method), which
         //---    made CArrayObj::Load() hit the base's `return(false)` stub on the very first neuron.
         //---  - failed at a LATER layer: the header/earlier layers read fine, so the file really does end
         //---    early => truncated by an interrupted save (Save() is atomic now, so this should be a
         //---    legacy file only).
         //---  - tiny file: truncated, whatever the layer index.
         string likely;
         if(fileSizeBytes < 65536)
            likely = "the file is only " + IntegerToString((int)fileSizeBytes) + " bytes => TRUNCATED (interrupted save, pre-atomic-Save file)";
         else
            if(failedLayer == 0)
               likely = "the file is " + IntegerToString((int)(fileSizeBytes / 1024)) + " KB and it failed on the FIRST layer, before any layer data mattered"
                        + " => this is a READ-PATH CODE fault, not a bad file and not a " + backend + " allocation problem."
                        + " Check that CLayer::CreateElement still exactly matches CArrayObj::CreateElement's signature (const int) so it overrides it,"
                        + " and that CLayer::Load/CNeuronBaseOCL::Load are in sync with Save()";
            else
               likely = "the file is " + IntegerToString((int)(fileSizeBytes / 1024)) + " KB and layers 0.." + IntegerToString(failedLayer - 1) +
                        " read fine => the file ends early (TRUNCATED by an interrupted save) or a neuron tensor could not be allocated on the " + backend;
         Print("CNet::Load: " + file_name + " - REJECTED: only loaded " + IntegerToString(layers.Total()) +
               " of " + IntegerToString(num) + " layers (failed at layer " + IntegerToString(failedLayer) +
               "). Diagnosis: " + likely + ".");
        }
      return(false);
     }
   return true;
  }
//+------------------------------------------------------------------+
//| See this method's declaration comment for the EMA shadow-weight  |
//| deployment rationale.                                            |
//+------------------------------------------------------------------+
bool CNet::BlendWeightsFrom(CNet &live, double tau)
  {
   if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID)
      return false;
   int layerTotal = MathMin(layers.Total(), live.layers.Total());
   for(int l = 0; l < layerTotal; l++)
     {
      CLayer *shadowLayer = layers.At(l);
      CLayer *liveLayer = live.layers.At(l);
      if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID)
         continue;
      int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total());
      for(int n = 0; n < neuronTotal; n++)
        {
         //--- Type() first, through the true common base (CObject) - NOT through a CNeuronBaseOCL*-typed
         //--- pointer. On the plain-CPU tier (no OpenCL/DirectML - the only option on Marketplace, which
         //--- forbids DLLs) CNet::CNet() builds legacy CNeuronBase-hierarchy neurons (CNeuron/CNeuronConv/
         //--- CNeuronPool/CNeuronLSTM), an UNRELATED class hierarchy from CNeuronBaseOCL/CNeuronConvOCL/
         //--- CNeuronLSTMOCL. Assigning one of those objects to a CNeuronBaseOCL* and calling its virtual
         //--- methods (as this used to do unconditionally) reads the wrong vtable/member layout - undefined
         //--- behaviour, not just a silent no-op, every single online-learning step on that tier.
         CObject *shadowObj = shadowLayer.At(n);
         CObject *liveObj = liveLayer.At(n);
         if(CheckPointer(shadowObj) == POINTER_INVALID || CheckPointer(liveObj) == POINTER_INVALID)
            continue;
         if(shadowObj.Type() != liveObj.Type())
            continue;
         double shadowW[], liveW[];
         switch(shadowObj.Type())
           {
            case defNeuronBaseOCL:
              {
               CNeuronBaseOCL *shadowNeuron = shadowObj;
               CNeuronBaseOCL *liveNeuron = liveObj;
               if(shadowNeuron.getWeights(shadowW) > 0 && liveNeuron.getWeights(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowNeuron.setWeights(shadowW);
                 }
              }
              break;
            case defNeuronBatchNormOCL:
              {
               //--- getWeightsBN packs the outgoing dense matrix and gamma/beta/statistics into one
               //--- flat array, so the shared blend loop below needs no special case of its own.
               CNeuronBatchNormOCL *shadowBN = shadowObj;
               CNeuronBatchNormOCL *liveBN = liveObj;
               if(shadowBN.getWeightsBN(shadowW) > 0 && liveBN.getWeightsBN(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowBN.setWeightsBN(shadowW);
                 }
              }
              break;
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *shadowConv = shadowObj;
               CNeuronConvOCL *liveConv = liveObj;
               if(shadowConv.getWeightsConv(shadowW) > 0 && liveConv.getWeightsConv(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowConv.setWeightsConv(shadowW);
                 }
              }
              break;
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *shadowLstm = shadowObj;
               CNeuronLSTMOCL *liveLstm = liveObj;
               if(shadowLstm.getWeightsLSTM(shadowW) > 0 && liveLstm.getWeightsLSTM(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowLstm.setWeightsLSTM(shadowW);
                 }
              }
              break;
            //--- Plain-CPU tier (no backend at all): dense/conv/pool/LSTM legacy neurons store weights
            //--- per-connection (CConnection.weight) rather than in one contiguous buffer - blend those
            //--- directly instead of silently dropping the shadow-EMA deployment on this tier.
            case defNeuron:
            case defNeuronConv:
            case defNeuronPool:
            case defNeuronLSTM:
              {
               CNeuronBase *shadowBase = shadowObj;
               CNeuronBase *liveBase = liveObj;
               CArrayCon *shadowCon = shadowBase.getConnections();
               CArrayCon *liveCon = liveBase.getConnections();
               if(CheckPointer(shadowCon) == POINTER_INVALID || CheckPointer(liveCon) == POINTER_INVALID)
                  break;
               int ct = MathMin(shadowCon.Total(), liveCon.Total());
               for(int ci = 0; ci < ct; ci++)
                 {
                  CConnection *sc = shadowCon.At(ci);
                  CConnection *lc = liveCon.At(ci);
                  if(CheckPointer(sc) == POINTER_INVALID || CheckPointer(lc) == POINTER_INVALID)
                     continue;
                  sc.weight = (1.0 - tau) * sc.weight + tau * lc.weight;
                 }
              }
              break;
           }
        }
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Snapshot every neuron's weights into host memory (see the header  |
//| declaration). Layer-major order; one CArrayDouble per neuron. Used |
//| as the mid-run best-era checkpoint - restored by RestoreWeights(). |
//+------------------------------------------------------------------+
bool CNet::CaptureWeights(void)
  {
   if(CheckPointer(layers) == POINTER_INVALID)
      return false;
   if(CheckPointer(m_weightSnapshot) == POINTER_INVALID)
     {
      m_weightSnapshot = new CArrayObj();
      if(CheckPointer(m_weightSnapshot) == POINTER_INVALID)
         return false;
     }
   m_weightSnapshot.Clear(); // FreeMode deletes the previous snapshot's per-neuron arrays
   m_haveWeightSnapshot = false;
   for(int l = 0; l < layers.Total(); l++)
     {
      CLayer *layer = (CLayer*)layers.At(l);
      if(CheckPointer(layer) == POINTER_INVALID)
         return false;
      for(int n = 0; n < layer.Total(); n++)
        {
         CObject *obj = layer.At(n);
         if(CheckPointer(obj) == POINTER_INVALID)
            continue;
         double w[];
         int got = 0;
         switch(obj.Type())
           {
            case defNeuronBaseOCL:
              {
               CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
               got = neuron.getWeights(w);
               break;
              }
            case defNeuronBatchNormOCL:
              {
               //--- Dense matrix + gamma/beta/statistics as one array - see getWeightsBN. Without this
               //--- the plateau ladder would restore the weights around this layer while leaving its
               //--- own parameters at whatever the diverged era left behind.
               CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
               got = bn.getWeightsBN(w);
               break;
              }
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *c = (CNeuronConvOCL*)obj;
               got = c.getWeightsConv(w);
               break;
              }
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj;
               got = ls.getWeightsLSTM(w);
               break;
              }
            case defNeuron:
            case defNeuronConv:
            case defNeuronPool:
            case defNeuronLSTM:
              {
               CNeuronBase *neuron = (CNeuronBase*)obj;
               CArrayCon *conns = neuron.getConnections();
               if(CheckPointer(conns) != POINTER_INVALID)
                 {
                  got = conns.Total();
                  ArrayResize(w, got);
                  for(int i = 0; i < got; i++)
                    {
                     CConnection *con = conns.At(i);
                     w[i] = (CheckPointer(con) != POINTER_INVALID) ? con.weight : 0.0;
                    }
                 }
               break;
              }
            default:
               break;
           }
         CArrayDouble *snap = new CArrayDouble();
         if(CheckPointer(snap) == POINTER_INVALID)
            return false;
         if(got > 0)
            snap.AssignArray(w);
         if(!m_weightSnapshot.Add(snap))
           {
            delete snap;
            return false;
           }
        }
     }
   m_haveWeightSnapshot = true;
   return true;
  }
//+------------------------------------------------------------------+
//| Write the CaptureWeights() snapshot back into the live neurons IN  |
//| PLACE (setWeights - no neuron re-creation, no new device tensors). |
//| Aborts (returning false, live weights untouched past that point)  |
//| only on a neuron-count mismatch, which never happens within a run  |
//| (fixed topology). See the header declaration for the full why.    |
//+------------------------------------------------------------------+
bool CNet::RestoreWeights(void)
  {
   if(!m_haveWeightSnapshot || CheckPointer(m_weightSnapshot) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
      return false;
   int idx = 0;
   for(int l = 0; l < layers.Total(); l++)
     {
      CLayer *layer = (CLayer*)layers.At(l);
      if(CheckPointer(layer) == POINTER_INVALID)
         return false;
      for(int n = 0; n < layer.Total(); n++)
        {
         if(idx >= m_weightSnapshot.Total())
            return false; // topology/count mismatch - stop rather than mis-map weights
         CObject *obj = layer.At(n);
         CArrayDouble *snap = m_weightSnapshot.At(idx);
         idx++;
         if(CheckPointer(obj) == POINTER_INVALID || CheckPointer(snap) == POINTER_INVALID)
            continue;
         int cnt = snap.Total();
         if(cnt <= 0)
            continue; // no weights captured for this neuron (e.g. output layer) - nothing to restore
         double w[];
         ArrayResize(w, cnt);
         for(int i = 0; i < cnt; i++)
            w[i] = snap.At(i);
         switch(obj.Type())
           {
            case defNeuronBaseOCL:
              {
               CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
               neuron.setWeights(w);
               break;
              }
            case defNeuronBatchNormOCL:
              {
               CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
               bn.setWeightsBN(w);
               break;
              }
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *c = (CNeuronConvOCL*)obj;
               c.setWeightsConv(w);
               break;
              }
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj;
               ls.setWeightsLSTM(w);
               break;
              }
            case defNeuron:
            case defNeuronConv:
            case defNeuronPool:
            case defNeuronLSTM:
              {
               CNeuronBase *neuron = (CNeuronBase*)obj;
               CArrayCon *conns = neuron.getConnections();
               if(CheckPointer(conns) == POINTER_INVALID)
                  break;
               int count = MathMin(cnt, conns.Total());
               for(int i = 0; i < count; i++)
                 {
                  CConnection *con = conns.At(i);
                  if(CheckPointer(con) != POINTER_INVALID)
                     con.weight = w[i];
                 }
               break;
              }
            default:
               break;
           }
        }
     }
   return (idx == m_weightSnapshot.Total());
  }
//+------------------------------------------------------------------+
//| See this method's declaration comment for the cold-start bias    |
//| rationale. The weight block that produces the output layer's     |
//| values is stored on the layer BEFORE it (see CNeuronBaseOCL::    |
//| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE     |
//| neuron, laid out as (sourceNeurons+1) values per destination      |
//| neuron, the last of which is that neuron's bias term), so this    |
//| reaches one layer back from the output layer to edit it.          |
//+------------------------------------------------------------------+
bool CNet::SeedOutputLayerBias(const double &biasValues[])
  {
   int outputs = ArraySize(biasValues);
   if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
      return false;
   CLayer *sourceLayer = layers.At(layers.Total() - 2);
   if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0)
      return false;
   CObject *sourceObj = sourceLayer.At(0);
   //--- Batch norm is accepted here as well as a plain dense layer. It is a CNeuronBaseOCL subclass
   //--- with the identical (inputs+1)*outputs weight layout - it just happens to be the layer that
   //--- carries the head's weight matrix once normalization is enabled (see AI\NeuronBatchNorm.mqh and
   //--- BuildFreshTopology's batch-norm insertion). Without this the exact-type check silently failed
   //--- and the cold-start output-bias seed stopped being applied for every batch-norm topology.
   if(CheckPointer(sourceObj) == POINTER_INVALID ||
      (sourceObj.Type() != defNeuronBaseOCL && sourceObj.Type() != defNeuronBatchNormOCL))
      return false;
   CNeuronBaseOCL *sourceNeuron = (CNeuronBaseOCL*)sourceObj;
   if(CheckPointer(sourceNeuron) == POINTER_INVALID)
      return false;
   int inputs = sourceNeuron.Neurons();
   double weights[];
   int count = sourceNeuron.getWeights(weights);
   if(count != (inputs + 1) * outputs)
      return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess
   for(int i = 0; i < outputs; i++)
      weights[(inputs + 1) * i + inputs] = biasValues[i];
   return sourceNeuron.setWeights(weights);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::SetBatchNormFrozen(bool frozen)
  {
   if(CheckPointer(layers) == POINTER_INVALID)
      return;
   for(int l = 0; l < layers.Total(); l++)
     {
      CLayer *layer = (CLayer*)layers.At(l);
      if(CheckPointer(layer) == POINTER_INVALID)
         continue;
      for(int n = 0; n < layer.Total(); n++)
        {
         CObject *obj = layer.At(n);
         if(CheckPointer(obj) == POINTER_INVALID || obj.Type() != defNeuronBatchNormOCL)
            continue;
         CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
         bn.SetStatsFrozen(frozen);
        }
     }
  }
//+------------------------------------------------------------------+
//| See the declaration comment for why a loaded model's activation   |
//| cannot be trusted and must be re-asserted from the topology spec. |
//+------------------------------------------------------------------+
bool CNet::EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous)
  {
   previous = intended;
   if(CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
      return false;
   CLayer *outputLayer = layers.At(layers.Total() - 1);
   if(CheckPointer(outputLayer) == POINTER_INVALID || outputLayer.Total() <= 0)
      return false;
   CObject *obj = outputLayer.At(0);
   if(CheckPointer(obj) == POINTER_INVALID)
      return false;
   //--- Both neuron models expose the same two accessors (CNeuronBase gained Activation() for exactly
   //--- this), so one branch per family is enough - and anything else (conv/pool/LSTM can never be an
   //--- output layer in this project's topologies) is left untouched rather than force-cast.
   int t = obj.Type();
   if(t == defNeuronBaseOCL || t == defNeuronConvOCL || t == defNeuronPoolOCL || t == defNeuronLSTMOCL ||
      t == defNeuronBatchNormOCL)
     {
      CNeuronBaseOCL *n = (CNeuronBaseOCL*)obj;
      previous = n.Activation();
      if(previous == intended)
         return false;
      n.SetActivationFunction(intended);
      return true;
     }
   if(t == defNeuron || t == defNeuronConv || t == defNeuronPool || t == defNeuronLSTM)
     {
      //--- The scalar CPU model stores activation per NEURON, not per layer, so every neuron in the
      //--- output layer has to be corrected - not just index 0.
      bool repaired = false;
      for(int i = 0; i < outputLayer.Total(); i++)
        {
         CObject *cell = outputLayer.At(i);
         if(CheckPointer(cell) == POINTER_INVALID || cell.Type() != defNeuron)
            continue;
         CNeuronBase *n = (CNeuronBase*)cell;
         if(n.Activation() == intended)
            continue;
         if(!repaired)
            previous = n.Activation();
         n.SetActivationFunction(intended);
         repaired = true;
        }
      return repaired;
     }
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Save(const int file_handle)
  {
   if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle))
      return false;
   if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Load(const int file_handle)
  {
   if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle))
      return false;
   iWindow = FileReadInteger(file_handle, INT_VALUE);
   iStep = FileReadInteger(file_handle, INT_VALUE);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::Save(const int file_handle)
  {
   if(!CNeuronPool::Save(file_handle))
      return false;
   if(FileWriteDouble(file_handle, param) < 8)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::Load(const int file_handle)
  {
   if(!CNeuronPool::Load(file_handle))
      return false;
   param = FileReadDouble(file_handle);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLSTM    :  public CNeuronPool
  {
protected:
   CLayer            *ForgetGate;
   CLayer            *InputGate;
   CLayer            *OutputGate;
   CLayer            *NewContent;
   CArrayDouble      *Memory;
   CArrayDouble      *PrevMemory;
   CArrayDouble      *Input;
   CArrayDouble      *InputGradient;
   //---
   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);
   virtual bool      updateInputWeights(CLayer *prevLayer);
   virtual bool      updateInputWeights(CLayer *gate, CArrayDouble *input_data);
   virtual bool      InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type);
   virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence);

public:
                     CNeuronLSTM(void);
                    ~CNeuronLSTM(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronLSTM;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTM::CNeuronLSTM(void)
  {
   ForgetGate     =  new CLayer();
   InputGate      =  new CLayer();
   OutputGate     =  new CLayer();
   NewContent     =  new CLayer();
   Memory         =  new CArrayDouble();
   PrevMemory     =  new CArrayDouble();
   Input          =  new CArrayDouble();
   InputGradient  =  new CArrayDouble();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTM::~CNeuronLSTM(void)
  {
   if(CheckPointer(ForgetGate) != POINTER_INVALID)
      delete ForgetGate;
   if(CheckPointer(InputGate) != POINTER_INVALID)
      delete InputGate;
   if(CheckPointer(OutputGate) != POINTER_INVALID)
      delete OutputGate;
   if(CheckPointer(NewContent) != POINTER_INVALID)
      delete NewContent;
   if(CheckPointer(Memory) != POINTER_INVALID)
      delete Memory;
   if(CheckPointer(PrevMemory) != POINTER_INVALID)
      delete PrevMemory;
   if(CheckPointer(Input) != POINTER_INVALID)
      delete Input;
   if(CheckPointer(InputGradient) != POINTER_INVALID)
      delete InputGradient;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
  {
   if(units_count <= 0)
      return false;
//--- Init Layers
   if(!CNeuronPool::Init(numOutputs, myIndex, window, step, units_count, optimization_type))
      return false;
   if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(InputGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(NewContent, units_count, window + units_count, optimization_type))
      return false;
   if(!Memory.Reserve(units_count))
      return false;
   if(!PrevMemory.Reserve(units_count))
      return false;
   CNeuron *temp;
   for(int i = 0; i < units_count; i++)
     {
      if(!Memory.Add(0))
         return false;
      if(!PrevMemory.Add(0))
         return false;
      temp = OutputLayer.At(i);
      temp.setOutputVal(0);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(layer) == POINTER_INVALID)
     {
      layer = new CLayer(numOutputs);
      if(CheckPointer(layer) == POINTER_INVALID)
         return false;
     }
   else
      layer.Clear();
   if(!layer.Reserve(numUnits))
      return false;
//---
   //--- LeCun-uniform init, matching CNeuronLSTMOCL::Init's gate weighScale rationale - fan-in is
   //--- hidden units + input width (window+units_count, i.e. this InitLayer() call's own numOutputs
   //--- param, since each gate neuron's Connections array IS its fan-in weight vector here, not a
   //--- fan-out to a next layer).
   double gateScale = 1.0 / MathSqrt((double)numOutputs + 1.0);
   CNeuron *temp;
   for(int i = 0; i < numUnits; i++)
     {
      temp = new CNeuron();
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      if(!temp.Init(numOutputs + 1, i, optimization_type, gateScale))
         return false;
      if(!layer.Add(temp))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::feedForward(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0)
      return false;
   CNeuronBase *temp;
   CConnection *temp_con;
   if(CheckPointer(Input) == POINTER_INVALID)
     {
      Input = new CArrayDouble();
      if(CheckPointer(Input) == POINTER_INVALID)
         return false;
     }
   else
      Input.Clear();
//--- Concatenate input sequence
   int total = prevLayer.Total();
   if(!Input.Reserve(total + OutputLayer.Total()))
      return false;
   for(int i = 0; i < total; i++)
     {
      temp = prevLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
         return false;
     }
   total = OutputLayer.Total();
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
         return false;
     }
   int total_data = Input.Total();
//--- Calculated forget gate
   CArrayDouble *forget_gate = CalculateGate(ForgetGate, Input);
   if(CheckPointer(forget_gate) == POINTER_INVALID)
      return false;
//--- Calculated input gate
   CArrayDouble *input_gate = CalculateGate(InputGate, Input);
   if(CheckPointer(input_gate) == POINTER_INVALID)
      return false;
//--- Calculated output gate
   CArrayDouble *output_gate = CalculateGate(OutputGate, Input);
   if(CheckPointer(output_gate) == POINTER_INVALID)
      return false;
//--- Calculated new content
   CArrayDouble *new_content = new CArrayDouble();
   if(CheckPointer(new_content) == POINTER_INVALID)
      return false;
   total = NewContent.Total();
   for(int i = 0; i < total; i++)
     {
      temp = NewContent.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      double val = 0;
      for(int c = 0; c < total_data; c++)
        {
         temp_con = temp.Connections.At(c);
         if(CheckPointer(temp_con) == POINTER_INVALID)
            return false;
         val += temp_con.weight * Input.At(c);
        }
      val = TanhFunction(val);
      temp.setOutputVal(val);
      if(!new_content.Add(val))
         return false;
     }
//--- Calculated output sequences
   for(int i = 0; i < total; i++)
     {
      if(PrevMemory.Total() <= i)
         PrevMemory.Add(Memory.At(i));
      else
         PrevMemory.Update(i, Memory.At(i));
      double value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i);
      if(!Memory.Update(i, value))
         return false;
      temp = OutputLayer.At(i);
      value = TanhFunction(value) * output_gate.At(i);
      temp.setOutputVal(value);
     }
//---
   delete forget_gate;
   delete input_gate;
   delete new_content;
   delete output_gate;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CArrayDouble *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayDouble *sequence)
  {
   CNeuronBase *temp;
   CConnection *temp_con;
   CArrayDouble *result = new CArrayDouble();
   if(CheckPointer(gate) == POINTER_INVALID)
      return NULL;
   int total = gate.Total();
   int total_data = sequence.Total();
   for(int i = 0; i < total; i++)
     {
      temp = gate.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
        {
         delete result;
         return NULL;
        }
      double val = 0;
      for(int c = 0; c < total_data; c++)
        {
         temp_con = temp.Connections.At(c);
         if(CheckPointer(temp_con) == POINTER_INVALID)
           {
            delete result;
            return NULL;
           }
         val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c));
        }
      val = SigmoidFunction(val);
      temp.setOutputVal(val);
      if(!result.Add(val))
        {
         delete result;
         return NULL;
        }
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(InputGradient) == POINTER_INVALID)
     {
      InputGradient = new CArrayDouble();
      if(CheckPointer(InputGradient) == POINTER_INVALID)
         return false;
     }
   else
      InputGradient.Clear();
//---
   int total = OutputLayer.Total();
   CNeuron *temp;
   CArrayDouble *MemoryGradient = new CArrayDouble();
   CNeuron *gate;
   CConnection *con;
//---
   if(nextLayer != OutputLayer)
      for(int i = 0; i < total; i++)
        {
         temp = OutputLayer.At(i);
         if(CheckPointer(temp) == POINTER_INVALID)
            return false;
         temp.setGradient(temp.sumDOW(nextLayer));
        }
//--- Calculated memory  and output gate gradients
   if(CheckPointer(MemoryGradient) == POINTER_INVALID)
      return false;
   if(!MemoryGradient.Reserve(total))
      return false;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      gate = OutputGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      double value = temp.getGradient() * gate.getOutputVal();
      value = TanhFunctionDerivative(Memory.At(i)) * value;
      if(i >= MemoryGradient.Total())
        {
         if(!MemoryGradient.Add(value))
            return false;
        }
      else
        {
         value = MemoryGradient.At(i) + value;
         if(!MemoryGradient.Update(i, value))
            return false;
        }
      gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0);
      //--- Calcculated gates and new content gradients
      gate = ForgetGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
      gate = InputGate.At(i);
      temp = NewContent.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
      temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0);
     }
//--- Calculated input gradients
   int total_inp = temp.getConnections().Total();
   for(int n = 0; n < total_inp; n++)
     {
      double value = 0;
      for(int i = 0; i < total; i++)
        {
         temp = ForgetGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = InputGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = OutputGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = NewContent.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
        }
      if(InputGradient.Total() >= n)
        {
         if(!InputGradient.Add(value))
            return false;
        }
      else
         if(!InputGradient.Update(n, value))
            return false;
     }
//--- Calculated gradients for prev. state
   int shift = total_inp - total;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(InputGradient.At(shift + i));
     }
//--- Calculated memory  and output gate gradients
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      gate = OutputGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      double value = temp.getGradient() * gate.getPrevVal();
      value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value;
      if(!MemoryGradient.Update(i, value))
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0));
      //--- Calcculated gates and new content gradients
      gate = ForgetGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
      gate = InputGate.At(i);
      temp = NewContent.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
      temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0));
     }
//---
   delete MemoryGradient;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID)
      return false;
//---
   if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input)
      || !updateInputWeights(NewContent, Input))
     {
      return false;
     }
   if(optimization == ADAM)
      t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayDouble *input_data)
  {
   if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID)
      return false;
   CNeuronBase *neuron;
   CConnection *con;
   int total_n = gate.Total();
   int total_data = input_data.Total();
   double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
   for(int n = 0; n < total_n; n++)
     {
      neuron = gate.At(n);
      if(CheckPointer(neuron) == POINTER_INVALID)
         return false;
      double g = neuron.getGradient();
      double g2 = g * g;
      for(int i = 0; i < total_data; i++)
        {
         con = neuron.getConnections().At(i);
         if(CheckPointer(con) == POINTER_INVALID)
            return false;
         double data = input_data.At(i);
         if(optimization == SGD)
            con.weight += con.deltaWeight = (g != 0 && data != 0 ? eta * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight;
         else
           {
            con.mt = b1 * con.mt + (1 - b1) * g;
            con.vt = b2 * con.vt + (1 - b2) * g2 + 0.00000001;
            con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
            // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why.
            con.weight += con.deltaWeight;
           }
         // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
         // AI\Network.cl's LSTM_UpdateWeightsAdam MAX_WEIGHT clamp.
         con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index)
      return false;
//---
   prevNeuron.setGradient(InputGradient.At(index));
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return false;
//---
   int total = prevLayer.Total();
   if(total <= 0)
      return false;
   CNeuronBase *neuron;
   bool result = true;
   for(int i = 0; (i < total && result); i++)
     {
      neuron = prevLayer.At(i);
      if(CheckPointer(neuron) == POINTER_INVALID)
        {
         result = false;
         break;
        }
      result = calcInputGradients(neuron, i);
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Save(const int file_handle)
  {
   if(!CNeuronPool::Save(file_handle))
      return false;
   if(!ForgetGate.Save(file_handle))
      return false;
   if(!InputGate.Save(file_handle))
      return false;
   if(!OutputGate.Save(file_handle))
      return false;
   if(!NewContent.Save(file_handle))
      return false;
   if(!Memory.Save(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Load(const int file_handle)
  {
   if(!CNeuronPool::Load(file_handle))
      return false;
   if(!ForgetGate.Load(file_handle))
      return false;
   if(!InputGate.Load(file_handle))
      return false;
   if(!OutputGate.Load(file_handle))
      return false;
   if(!NewContent.Load(file_handle))
      return false;
   if(!Memory.Load(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunction(double x)
  {
   switch(activation)
     {
      case NONE:
         return(x);
         break;
      case TANH:
         return TanhFunction(x);
         break;
      case SIGMOID:
         return SigmoidFunction(x);
         break;
      case PRELU:
         // fixed 0.01 slope, matches CNeuronConv::activationFunction's `param` default -
         // this case was previously missing here, so any generic Dense (defNeuron) layer
         // given PRELU silently fell through to the `return x` below (pure linear identity,
         // no nonlinearity at all) instead of leaky-ReLU.
         return(x >= 0 ? x : 0.01 * x);
         break;
     }
//---
   return x;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunctionDerivative(double x)
  {
   switch(activation)
     {
      case NONE:
         return(1);
         break;
      case TANH:
         return TanhFunctionDerivative(x);
         break;
      case SIGMOID:
         return SigmoidFunctionDerivative(x);
         break;
      case PRELU:
         // See activationFunction() above - was previously missing here too, defaulting
         // to a derivative of 1 (which happens to be right for x>=0, but wrong for x<0,
         // where it must be the 0.01 leak slope, not 1).
         return(x >= 0 ? 1.0 : 0.01);
         break;
     }
//---
   return 1;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
template<typename T>
int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags)
  {
   int result = -1;
   for(int i = 0; i < m_buffers_total; i++)
     {
      if(m_buffers[i] != INVALID_HANDLE)
         continue;
      result = i;
      break;
     }
//---
   if(result < 0)
     {
      if(ArrayResize(m_buffers, m_buffers_total + 1) > 0)
        {
         m_buffers_total = ArraySize(m_buffers);
         result = m_buffers_total - 1;
         m_buffers[result] = INVALID_HANDLE;
        }
      else
         return result;
     }
//---
   if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags))
      return -1;
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL, CDirectMLMy *directml = NULL) :  hWeights(-1),
   hDeltaWeights(-1),
   hOutput(-1),
   hGradient(-1)
  {
   iOutputs = outputs;
   iFileHandle = handle;
   OpenCL = opencl;
   DirectML = directml;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CLayer::Load(const int file_handle)
  {
   iFileHandle = file_handle;
   if(!CArrayObj::Load(file_handle))
      return false;
   // A corrupt/truncated save can produce a 0-element layer here - CArrayObj::Load itself would
   // still return true for it, so m_data[0] must be guarded before indexing or MQL5 raises an
   // "array out of range" runtime error that aborts the whole CNet::Load call.
   if(Total() <= 0 || CheckPointer(m_data[0]) == POINTER_INVALID)
     {
      Print(__FUNCTION__ + ": layer loaded with no elements (corrupt or truncated save file)");
      return false;
     }
//---
   switch(m_data[0].Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronPoolOCL:
      case defNeuronLSTMOCL:
      case defNeuronBatchNormOCL:
        {
         CNeuronBaseOCL *temp = m_data[0];
         iOutputs = temp.getConnections();
         break;
        }
      default:
        {
         CNeuronBase *temp = m_data[0];
         iOutputs = temp.getConnections().Total();
         break;
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
//| Install the per-class logit offsets - see the declaration.        |
//| Expects tau*log(prior_c) already computed by the caller, which is |
//| the only place that knows the training set's class distribution.  |
//+------------------------------------------------------------------+
void CNet::SetLogitAdjustment(const double &offsets[])
  {
   if(ArraySize(offsets) < 3)
     {
      //--- Refuse rather than half-apply: a partially filled offset vector would silently bias two
      //--- classes against a third, which is far worse than running unadjusted.
      bLogitAdjust = false;
      return;
     }
   for(int i = 0; i < 3; i++)
     {
      //--- Guard against a non-finite offset reaching the softmax (a zero prior would give -inf and
      //--- turn every gradient into NaN). A class with no examples at all simply gets no push.
      double v = offsets[i];
      if(!MathIsValidNumber(v))
         v = 0.0;
      dLogitAdjust[i] = v;
     }
   bLogitAdjust = true;
  }
CNet::~CNet(void)
  {
   if(CheckPointer(layers) != POINTER_INVALID)
      delete layers;
   if(CheckPointer(m_weightSnapshot) != POINTER_INVALID)
      delete m_weightSnapshot; // CArrayObj (FreeMode) deletes its per-neuron CArrayDouble elements
   if(CheckPointer(opencl) != POINTER_INVALID)
     {
      opencl.Shutdown();
      delete opencl;
     }
   if(CheckPointer(directml) != POINTER_INVALID)
      delete directml;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "BufferDouble.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBaseOCL    :  public CObject
  {
protected:
   COpenCLMy         *OpenCL;
   CDirectMLMy       *DirectML;
   CBufferDouble     *Output;
   CBufferDouble     *PrevOutput;
   CBufferDouble     *Weights;
   CBufferDouble     *DeltaWeights;
   CBufferDouble     *Gradient;
   CBufferDouble     *FirstMomentum;
   CBufferDouble     *SecondMomentum;
   //---
   const double      alpha;
   int               t;
   //---
   int               m_myIndex;
   ENUM_ACTIVATION   activation;
   ENUM_OPTIMIZATION optimization;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);
   //--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only
   //--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists -
   //--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU).
   bool              BackendBufferCreate(CBufferDouble *buf)
     {
      if(CheckPointer(buf) == POINTER_INVALID)
         return false;
      if(CheckPointer(OpenCL) != POINTER_INVALID)
         return buf.BufferCreate(OpenCL);
      if(CheckPointer(DirectML) != POINTER_INVALID)
         return buf.BufferCreate(DirectML);
      return true; // no backend: keep host m_data, skip device allocation
     }

public:
                     CNeuronBaseOCL(void);
                    ~CNeuronBaseOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }
   //---
   virtual int       getOutputIndex(void)          {  return Output.GetIndex();        }
   virtual int       getPrevOutIndex(void)         {  return PrevOutput.GetIndex();    }
   virtual int       getGradientIndex(void)        {  return Gradient.GetIndex();      }
   virtual int       getWeightsIndex(void)         {  return Weights.GetIndex();       }
   virtual int       getDeltaWeightsIndex(void)    {  return DeltaWeights.GetIndex();  }
   virtual int       getFirstMomentumIndex(void)   {  return FirstMomentum.GetIndex(); }
   virtual int       getSecondMomentumIndex(void)  {  return SecondMomentum.GetIndex();}
   //---
   virtual int       getOutputVal(double &values[])   {  return Output.GetData(values);      }
   virtual int       getOutputVal(CArrayDouble *values)   {  return Output.GetData(values);  }
   virtual int       getPrevVal(double &values[])     {  return PrevOutput.GetData(values);  }
   virtual int       getGradient(double &values[])    {  return Gradient.GetData(values);    }
   //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by
   //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call
   //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same
   //--- buffer to propagate into the hidden layers.
   virtual bool      setGradient(const double &values[])
     {
      int count = ArraySize(values);
      for(int i = 0; i < count; i++)
         if(!Gradient.Update(i, values[i]))
            return false;
      return Gradient.BufferWrite();
     }
   // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created,
   // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here.
   virtual int       getWeights(double &values[])     {  return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values));     }
   // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment
   // (see that method's declaration comment) - writes a full replacement weight array back to this
   // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element,
   // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment
   // then pushes it to the device - the shape callers use when blending two already-read-out arrays.
   virtual bool      setWeights(double &values[])
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
         return false;
      if(!Weights.AssignArray(values))
         return false;
      return Weights.BufferWrite();
     }
   virtual int       Neurons(void)                    {  return Output.Total();              }
   virtual ENUM_ACTIVATION Activation(void)           {  return activation;                  }
   virtual int       getConnections(void)             {  return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0);   }
   //--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to
   //--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/
   //--- getOutputVal() which route through BufferRead() (a device read that fails without a backend).
   double            OutputHost(int i)                {  return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0;   }
   double            WeightHost(int i)                {  return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0;  }
   int               WeightsCount(void)               {  return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0;   }
   //--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only
   //--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense
   //--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math.
   virtual bool      feedForwardCPU(CNeuronBaseOCL *NeuronOCL);
   //--- Host-side input write for the CPU inference path's layer 0, and host-side output read for
   //--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates.
   bool              SetInputsCPU(CArrayDouble *inputVals);
   int               GetOutputsCPU(CArrayDouble *values);
   //---
   virtual bool      feedForward(CObject *SourceObject);
   virtual bool      calcHiddenGradients(CObject *TargetObject);
   virtual bool      calcOutputGradients(CArrayDouble *Target);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CObject *SourceObject);
   virtual void      SetOptimization(ENUM_OPTIMIZATION value) {  optimization = value; }
   virtual ENUM_OPTIMIZATION Optimization(void) const { return optimization; }
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronBaseOCL;                  }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBaseOCL::CNeuronBaseOCL(void)   :  alpha(momentum),
   activation(TANH),
   optimization(SGD),
   t(1)
  {
   OpenCL = NULL;
   DirectML = NULL;
   Output = new CBufferDouble();
   PrevOutput = new CBufferDouble();
   Weights = new CBufferDouble();
   DeltaWeights = new CBufferDouble();
   Gradient = new CBufferDouble();
   FirstMomentum = new CBufferDouble();
   SecondMomentum = new CBufferDouble();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBaseOCL::~CNeuronBaseOCL(void)
  {
   if(CheckPointer(Output) != POINTER_INVALID)
      delete Output;
   if(CheckPointer(PrevOutput) != POINTER_INVALID)
      delete PrevOutput;
   if(CheckPointer(Weights) != POINTER_INVALID)
      delete Weights;
   if(CheckPointer(DeltaWeights) != POINTER_INVALID)
      delete DeltaWeights;
   if(CheckPointer(Gradient) != POINTER_INVALID)
      delete Gradient;
   if(CheckPointer(FirstMomentum) != POINTER_INVALID)
      delete FirstMomentum;
   if(CheckPointer(SecondMomentum) != POINTER_INVALID)
      delete SecondMomentum;
   OpenCL = NULL;
   DirectML = NULL;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
      return false;
   OpenCL = open_cl;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(OpenCL))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
      // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
      // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
      // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
      // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
      // for the fact that they zero out roughly half their input distribution, whereas the
      // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
      // was ~2x too conservative here. Applied to every layer through this one shared Init()
      // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
      // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
      // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
      // extremes on every backend, so the imprecision there is not worth the much larger, riskier
      // change of threading activation awareness through every neuron subtype's Init() overload.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(OpenCL))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(OpenCL))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(OpenCL))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(OpenCL))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same  |
//| buffer layout, buffers just get created on the DML backend.      |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0)
      return false;
   DirectML = direct_ml;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(DirectML))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(DirectML))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(DirectML))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(DirectML))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(DirectML))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
#include "NeuronOCLConvPool.mqh"
#include "NeuronBatchNorm.mqh"
//+------------------------------------------------------------------+
//| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from      |
//| scratch from the standard LSTM equations - NOT ported from the    |
//| NeuroNet_DNG reference (see the note above the LSTM kernels in    |
//| Network.cl for why). Single-timestep-truncated BPTT: gradient     |
//| does not flow back into h_prev/c_prev from an earlier step.       |
//| Adam-only - Init fails for any other optimization type.           |
//+------------------------------------------------------------------+
class CNeuronLSTMOCL : public CNeuronBaseOCL
  {
protected:
   int               m_iInputs;
   CBufferDouble     *WeightsLSTM;
   CBufferDouble     *FirstMomentumLSTM;
   CBufferDouble     *SecondMomentumLSTM;
   CBufferDouble     *DeltaWeightsLSTM;
   CBufferDouble     *WeightsGradient;
   CBufferDouble     *Concatenated;
   CBufferDouble     *ConcatenatedGradient;
   CBufferDouble     *Memory;
   CBufferDouble     *HiddenCache;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      SetInputs(int count);

public:
                     CNeuronLSTMOCL(void) : m_iInputs(-1)
     {
      WeightsLSTM = NULL;
      FirstMomentumLSTM = NULL;
      SecondMomentumLSTM = NULL;
      DeltaWeightsLSTM = NULL;
      WeightsGradient = NULL;
      Concatenated = NULL;
      ConcatenatedGradient = NULL;
      Memory = NULL;
      HiddenCache = NULL;
     }
                    ~CNeuronLSTMOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronLSTMOCL;  }
   // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the
   // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
   virtual int       getWeightsLSTM(double &values[])     {  return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values));  }
   virtual bool      setWeightsLSTM(double &values[])
     {
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
      if(!WeightsLSTM.AssignArray(values))
         return false;
      return WeightsLSTM.BufferWrite();
     }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTMOCL::~CNeuronLSTMOCL(void)
  {
   if(CheckPointer(WeightsLSTM) != POINTER_INVALID)
      delete WeightsLSTM;
   if(CheckPointer(FirstMomentumLSTM) != POINTER_INVALID)
      delete FirstMomentumLSTM;
   if(CheckPointer(SecondMomentumLSTM) != POINTER_INVALID)
      delete SecondMomentumLSTM;
   if(CheckPointer(DeltaWeightsLSTM) != POINTER_INVALID)
      delete DeltaWeightsLSTM;
   if(CheckPointer(WeightsGradient) != POINTER_INVALID)
      delete WeightsGradient;
   if(CheckPointer(Concatenated) != POINTER_INVALID)
      delete Concatenated;
   if(CheckPointer(ConcatenatedGradient) != POINTER_INVALID)
      delete ConcatenatedGradient;
   if(CheckPointer(Memory) != POINTER_INVALID)
      delete Memory;
   if(CheckPointer(HiddenCache) != POINTER_INVALID)
      delete HiddenCache;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type))
      return false;
   uint H = numNeurons;
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(OpenCL))
      return false;
   m_iInputs = -1;
   return true;
  }
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above.        |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type))
      return false;
   uint H = numNeurons;
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(DirectML))
      return false;
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(DirectML))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(DirectML))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(DirectML))
      return false;
   m_iInputs = -1;
   return true;
  }
//+------------------------------------------------------------------+
//| Lazily sized on the first feedForward call, once the previous     |
//| layer's neuron count (the input size) is known.                   |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::SetInputs(int count)
  {
   m_iInputs = count;
   uint H = (uint)Neurons();
   int total = (int)(4 * H * (H + count + 1));
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
     {
      WeightsLSTM = new CBufferDouble();
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(!WeightsLSTM.Reserve(total))
      return false;
   // Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full
   // rationale; fan-in here is hidden units + input width (each gate reads both).
   double weighScale = 1.0 / MathSqrt((double)(H + count) + 1.0);
   for(int i = 0; i < total; i++)
     {
      double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
      if(weigh == 0)
         weigh = 0.001;
      if(!WeightsLSTM.Add(weigh))
         return false;
     }
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
     {
      FirstMomentumLSTM = new CBufferDouble();
      if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(!FirstMomentumLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
     {
      SecondMomentumLSTM = new CBufferDouble();
      if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(!SecondMomentumLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   // Momentum accumulator - only meaningfully used when optimization==SGD
   // (see updateInputWeights()), but allocated unconditionally regardless of
   // the configured optimizer, matching CNeuronBaseOCL::Init's pattern for
   // the dense-layer DeltaWeights/FirstMomentum/SecondMomentum trio above.
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
     {
      DeltaWeightsLSTM = new CBufferDouble();
      if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(!DeltaWeightsLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(WeightsGradient) == POINTER_INVALID)
     {
      WeightsGradient = new CBufferDouble();
      if(CheckPointer(WeightsGradient) == POINTER_INVALID)
         return false;
     }
   if(!WeightsGradient.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(m_iInputs <= 0)
     {
      if(!SetInputs(NeuronOCL.Neurons()))
         return false;
     }
   else
      if(m_iInputs != NeuronOCL.Neurons())
         return false;
   int H = Neurons();
   int I = m_iInputs;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.LSTMGates(WeightsLSTM.GetIndex(), getOutputIndex(), NeuronOCL.getOutputIndex(), Concatenated.GetIndex(), H, I) ||
         !DirectML.LSTMState(Concatenated.GetIndex(), Memory.GetIndex(), getOutputIndex(), HiddenCache.GetIndex(), getOutputIndex(), H))
        {
         printf("Error of execution DirectML LSTM feedForward");
         return false;
        }
      return Output.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset2[2] = {0, 0};
   uint size2[2] = {(uint)H, 4};
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_matrix_w, WeightsLSTM.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_hidden_prev, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_inputs, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_Gates, 2, offset2, size2))
     {
      printf("Error of execution kernel LSTM_Gates: %d", GetLastError());
      return false;
     }
   uint offset1[1] = {0};
   uint size1[1] = {(uint)H};
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_memory, Memory.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_prev, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_cache, HiddenCache.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_output, getOutputIndex());
   OpenCL.SetArgument(def_k_LSTM_State, def_k_lstms_hidden_size, H);
   if(!OpenCL.Execute(def_k_LSTM_State, 1, offset1, size1))
     {
      printf("Error of execution kernel LSTM_State: %d", GetLastError());
      return false;
     }
   //--- Output (== hidden_prev for the next timestep) stays GPU-resident; see the note in
   //--- CNeuronBaseOCL::feedForward().
   return true;
  }
//+------------------------------------------------------------------+
//| Pure-MQL5 double-precision mirror of Network.cl's LSTM_Gates +    |
//| LSTM_State kernels, host buffers only (CPU inference). Recurrent  |
//| state is carried EXACTLY as the kernels do: hidden_prev is this   |
//| neuron's Output, the cell state is Memory[0..H); both persist     |
//| across bars. Weight layout: 4 gates (forget,input,output,cand),   |
//| each H*(H+I+1), per unit [H recurrent | I input | 1 bias].        |
//| NOTE: like every backend this advances state per call; over a     |
//| long backtest fp rounding vs the training backend can drift, but  |
//| within the validated per-step tolerance (see ValidateCpuInference)|
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID ||
      CheckPointer(WeightsLSTM) == POINTER_INVALID || CheckPointer(Memory) == POINTER_INVALID)
      return false;
   int H = Neurons();
   int I = m_iInputs;
   if(I <= 0 || NeuronOCL.Neurons() != I || H <= 0)
      return false;
   int per_gate = H * (H + I + 1);
   if(WeightsLSTM.Total() < 4 * per_gate || Memory.Total() < 2 * H || Output.Total() < H)
      return false;
   //--- Gates: read hidden_prev (this Output) and inputs (prev Output) BEFORE Output is overwritten by
   //--- the state step below - exactly the two-kernel ordering (Concatenated is a separate buffer there).
   double gates[];
   if(ArrayResize(gates, 4 * H) != 4 * H)
      return false;
   for(int gate = 0; gate < 4; gate++)
      for(int id = 0; id < H; id++)
        {
         int shift = gate * per_gate + id * (H + I + 1);
         double sum = 0.0;
         for(int k = 0; k < H; k++)
            sum += Output.At(k) * WeightsLSTM.At(shift + k);              // hidden_prev
         for(int k = 0; k < I; k++)
            sum += NeuronOCL.OutputHost(k) * WeightsLSTM.At(shift + H + k); // inputs
         sum += WeightsLSTM.At(shift + H + I);                            // bias
         gates[gate * H + id] = (gate < 3) ? (1.0 / (1.0 + exp(-sum))) : tanh(sum);
        }
   //--- State: c_t = f*c_prev + i*g ; h = o*tanh(c_t). memory[H+id] keeps c_prev (unused in inference).
   for(int id = 0; id < H; id++)
     {
      double f  = gates[id];
      double ii = gates[H + id];
      double o  = gates[2 * H + id];
      double g  = gates[3 * H + id];
      double c_prev = Memory.At(id);
      double c_t = f * c_prev + ii * g;
      if(!Memory.Update(H + id, c_prev) || !Memory.Update(id, c_t))
         return false;
      if(!Output.Update(id, o * tanh(c_t)))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Writes the gradient into NeuronOCL (the earlier/input-side layer) |
//| - same inverted-call convention as CNeuronConvOCL::calcInputGradients.|
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || m_iInputs <= 0)
      return false;
   int H = Neurons();
   int I = m_iInputs;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.LSTMGateGradient(getGradientIndex(), Memory.GetIndex(), Concatenated.GetIndex(), ConcatenatedGradient.GetIndex(), H) ||
         !DirectML.LSTMWeightsGradient(ConcatenatedGradient.GetIndex(), HiddenCache.GetIndex(), NeuronOCL.getOutputIndex(), WeightsGradient.GetIndex(), H, I) ||
         !DirectML.LSTMInputsGradient(ConcatenatedGradient.GetIndex(), WeightsLSTM.GetIndex(), NeuronOCL.getGradientIndex(), H, I))
        {
         printf("Error of execution DirectML LSTM calcInputGradients");
         return false;
        }
      double temp[];
      return NeuronOCL.getGradient(temp) > 0;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset1[1] = {0};
   uint size1[1] = {(uint)H};
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_gradient, getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_memory, Memory.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_GateGradient, def_k_lstmgg_hidden_size, H);
   if(!OpenCL.Execute(def_k_LSTM_GateGradient, 1, offset1, size1))
     {
      printf("Error of execution kernel LSTM_GateGradient: %d", GetLastError());
      return false;
     }
   uint sizeW[1] = {(uint)WeightsLSTM.Total()};
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_cache, HiddenCache.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_inputs, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_weights_gradient, WeightsGradient.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_WeightsGradient, 1, offset1, sizeW))
     {
      printf("Error of execution kernel LSTM_WeightsGradient: %d", GetLastError());
      return false;
     }
   uint sizeI[1] = {(uint)I};
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_matrix_w, WeightsLSTM.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_inputs_gradient, NeuronOCL.getGradientIndex());
   OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_InputsGradient, 1, offset1, sizeI))
     {
      printf("Error of execution kernel LSTM_InputsGradient: %d", GetLastError());
      return false;
     }
   //--- NeuronOCL's Gradient stays GPU-resident; see the note in
   //--- CNeuronConvOCL::calcInputGradients().
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
      return false;
   int total = WeightsLSTM.Total();
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(optimization == SGD)
        {
         if(!DirectML.LSTMUpdateWeightsMomentum(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), DeltaWeightsLSTM.GetIndex(), eta, alpha, total,
                                                0))
           {
            printf("Error of execution DirectML LSTM_UpdateWeightsMomentum");
            return false;
           }
        }
      else
        {
         double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
         if(!DirectML.LSTMUpdateWeightsAdam(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), FirstMomentumLSTM.GetIndex(), SecondMomentumLSTM.GetIndex(), lt, b1, b2, total))
           {
            printf("Error of execution DirectML LSTM_UpdateWeightsAdam");
            return false;
           }
         t++;
        }
      //--- WeightsLSTM stays DLL-resident; feedForward reads it via GetIndex() (same as the OpenCL
      //--- branch below). Save()/BlendWeightsFrom() BufferRead() on demand.
      return true;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset1[1] = {0};
   uint size1[1] = {(uint)total};
   if(optimization == SGD)
     {
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_w, WeightsLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_weights_gradient, WeightsGradient.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_dw, DeltaWeightsLSTM.GetIndex());
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_learning_rates, (float)eta);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_momentum, (float)alpha);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_optimizer, 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsMomentum, 1, offset1, size1))
        {
         printf("Error of execution kernel LSTM_UpdateWeightsMomentum: %d", GetLastError());
         return false;
        }
     }
   else
     {
      double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_w, WeightsLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_weights_gradient, WeightsGradient.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_m, FirstMomentumLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_v, SecondMomentumLSTM.GetIndex());
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_l, (float)lt);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b1, (float)b1);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b2, (float)b2);
      ResetLastError();
      if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsAdam, 1, offset1, size1))
        {
         printf("Error of execution kernel LSTM_UpdateWeightsAdam: %d", GetLastError());
         return false;
        }
      t++;
     }
   //--- WeightsLSTM stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights().
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      return false;
   if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < INT_VALUE)
      return false;
   if(m_iInputs <= 0)
      return true;
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID || !WeightsLSTM.BufferRead() || !WeightsLSTM.Save(file_handle))
      return false;
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID || !FirstMomentumLSTM.BufferRead() || !FirstMomentumLSTM.Save(file_handle))
      return false;
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID || !SecondMomentumLSTM.BufferRead() || !SecondMomentumLSTM.Save(file_handle))
      return false;
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID || !DeltaWeightsLSTM.BufferRead() || !DeltaWeightsLSTM.Save(file_handle))
      return false;
   if(CheckPointer(Memory) == POINTER_INVALID || !Memory.BufferRead() || !Memory.Save(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      return false;
   m_iInputs = FileReadInteger(file_handle, INT_VALUE);
   uint H = (uint)Neurons();
   //--- scratch buffers (not persisted) were sized for the placeholder Init() unit
   //--- count; resize them now that the real neuron count is known.
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   Concatenated.BufferFree();
   if(!Concatenated.BufferInit(4 * H, 0))
      return false;
   if(!BackendBufferCreate(Concatenated))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   ConcatenatedGradient.BufferFree();
   if(!ConcatenatedGradient.BufferInit(4 * H, 0))
      return false;
   if(!BackendBufferCreate(ConcatenatedGradient))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   HiddenCache.BufferFree();
   if(!HiddenCache.BufferInit(H, 0))
      return false;
   if(!BackendBufferCreate(HiddenCache))
      return false;
//---
   if(m_iInputs <= 0)
      return true;
   int total = (int)(4 * H * (H + m_iInputs + 1));
//---
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
     {
      WeightsLSTM = new CBufferDouble();
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(WeightsLSTM.GetIndex() >= 0)
      WeightsLSTM.BufferFree();
   if(!WeightsLSTM.Load(file_handle))
      return false;
   if(!BackendBufferCreate(WeightsLSTM))
      return false;
//---
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
     {
      FirstMomentumLSTM = new CBufferDouble();
      if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(FirstMomentumLSTM.GetIndex() >= 0)
      FirstMomentumLSTM.BufferFree();
   if(!FirstMomentumLSTM.Load(file_handle))
      return false;
   if(!BackendBufferCreate(FirstMomentumLSTM))
      return false;
//---
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
     {
      SecondMomentumLSTM = new CBufferDouble();
      if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(SecondMomentumLSTM.GetIndex() >= 0)
      SecondMomentumLSTM.BufferFree();
   if(!SecondMomentumLSTM.Load(file_handle))
      return false;
   if(!BackendBufferCreate(SecondMomentumLSTM))
      return false;
//---
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
     {
      DeltaWeightsLSTM = new CBufferDouble();
      if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(DeltaWeightsLSTM.GetIndex() >= 0)
      DeltaWeightsLSTM.BufferFree();
   if(!DeltaWeightsLSTM.Load(file_handle))
      return false;
   if(!BackendBufferCreate(DeltaWeightsLSTM))
      return false;
//---
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(Memory.GetIndex() >= 0)
      Memory.BufferFree();
   if(!Memory.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Memory))
      return false;
//---
   if(CheckPointer(WeightsGradient) == POINTER_INVALID)
     {
      WeightsGradient = new CBufferDouble();
      if(CheckPointer(WeightsGradient) == POINTER_INVALID)
         return false;
     }
   if(!WeightsGradient.BufferInit(total, 0))
      return false;
   if(!BackendBufferCreate(WeightsGradient))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
      case defNeuronBatchNormOCL:
         temp = SourceObject;
         return feedForward(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
                                NeuronOCL.Neurons(), NativeActivationCode(activation)))
        {
         printf("Error of execution DirectML FeedForward");
         return false;
        }
      return Output.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Output.Total();
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel FeedForward: %d", GetLastError());
      return false;
     }
   //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
   //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
   //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
   //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
   //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
   return true;
  }
//+------------------------------------------------------------------+
//| Pure-MQL5 dense forward pass - exact double-precision mirror of   |
//| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, |
//| which (Gizlyk convention) owns both the inputs (its Output) and   |
//| the weights connecting them to THIS layer (its Weights), laid out |
//| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host     |
//| buffers only; used when no compute backend exists (CPU inference).|
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return false;
   int inputs   = NeuronOCL.Neurons();
   int outCount = Output.Total();
   int wTotal   = NeuronOCL.WeightsCount();
   if(wTotal < (inputs + 1) * outCount)
      return false; // weight buffer smaller than the dense layout requires - refuse rather than misread
   for(int i = 0; i < outCount; i++)
     {
      int shift = (inputs + 1) * i;
      double sum = 0.0;
      for(int k = 0; k < inputs; k++)
         sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k);
      sum += NeuronOCL.WeightHost(shift + inputs); // bias
      switch(activation)
        {
         case TANH:
            sum = tanh(sum);
            break;
         case SIGMOID:
            sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum))));
            break;
         case PRELU:
            if(sum < 0.0)
               sum *= 0.01;
            break;
         // NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case.
        }
      if(!Output.Update(i, sum))
         return false;
     }
   return true;
  }
//+------------------------------------------------------------------+
//| Host-side input write for the CPU inference path's layer 0 (there |
//| is no device buffer to write into, unlike CNet::feedForward's     |
//| BufferWrite branch). Copies inputVals into the Output host array. |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals)
  {
   if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return false;
   int total = MathMin(Output.Total(), inputVals.Total());
   for(int i = 0; i < total; i++)
      if(!Output.Update(i, inputVals.At(i)))
         return false;
   return true;
  }
//+------------------------------------------------------------------+
//| Host-side output read for getResults() on the CPU inference path  |
//| (no device BufferRead available). Returns the count copied.       |
//+------------------------------------------------------------------+
int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values)
  {
   if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
      return 0;
   values.Clear();
   int n = Output.Total();
   for(int i = 0; i < n; i++)
      if(!values.Add(Output.At(i)))
         return i;
   return n;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
                                       NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1))
        {
         printf("Error of execution DirectML CalcHiddenGradient");
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Neurons() + 1;
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
      return false;
     }
   //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
   //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
  {
   if(CheckPointer(Target) == POINTER_INVALID)
      return false;
   int count = Target.Total();
   for(int i = 0; i < count; i++)
      if(!Gradient.Update(i, Target.At(i)))
         return false;
   Gradient.BufferWrite();
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
        {
         printf("Error of execution DirectML CalcOutputGradient");
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = count;
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
   ResetLastError();
   if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
      return false;
     }
   //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
   //--- BufferRead()s itself - see the note in feedForward() above.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      int inputs = NeuronOCL.Neurons();
      int neurons = Neurons();
      if(optimization == SGD)
        {
         if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                             NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons,
                                             0))
           {
            printf("Error of execution DirectML UpdateWeightsMomentum");
            return false;
           }
        }
      else
        {
         double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
         if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                         NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
                                         inputs, lt, b1, b2, neurons))
           {
            printf("Error of execution DirectML UpdateWeightsAdam");
            return false;
           }
         t++;
        }
      //--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex()
      //--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand.
      return true;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Neurons();
   global_work_size[1] = NeuronOCL.Neurons();
   if(optimization == SGD)
     {
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta);
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
         return false;
        }
     }
   else
     {
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
         return false;
      double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2))
         return false;
      uint rest = global_work_size[1] % 4;
      global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
         return false;
        }
      t++;
     }
   //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
   //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
  {
   if(CheckPointer(TargetObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   CNeuronConvOCL *tempConv = NULL;
   CNeuronLSTMOCL *tempLstm = NULL;
   CNeuronPoolOCL *tempPool = NULL;
   CNeuronBatchNormOCL *tempBN = NULL;
   switch(TargetObject.Type())
     {
      case defNeuronBaseOCL:
         temp = TargetObject;
         return calcHiddenGradients(temp);
         break;
      case defNeuronBatchNormOCL:
         //--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step
         //--- and writes into this->Gradient. Routing it through the dense branch instead would run
         //--- CaclHiddenGradient against a weight matrix batch norm does not have.
         tempBN = TargetObject;
         return tempBN.calcInputGradients(GetPointer(this));
         break;
      case defNeuronConvOCL:
         //--- Conv owns the backward step (calcInputGradients), called on itself with
         //--- "this" (the earlier layer) passed in so it writes into this->Gradient.
         tempConv = TargetObject;
         return tempConv.calcInputGradients(GetPointer(this));
         break;
      case defNeuronLSTMOCL:
         tempLstm = TargetObject;
         return tempLstm.calcInputGradients(GetPointer(this));
         break;
      case defNeuronPoolOCL:
         tempPool = TargetObject;
         return tempPool.calcInputGradients(GetPointer(this));
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
      case defNeuronBatchNormOCL:
         temp = SourceObject;
         return updateInputWeights(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Save(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
   if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
      return false;
   if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
      return false;
   if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
      return false;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      FileWriteInteger(file_handle, 0);
      return true;
     }
   else
      FileWriteInteger(file_handle, 1);
//---
   if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
      return false;
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
         return false;
      if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Load(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
//---
   activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
   optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
   t = FileReadInteger(file_handle, INT_VALUE);
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(Output.GetIndex() >= 0)
      Output.BufferFree();
   if(!Output.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Output))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(PrevOutput.GetIndex() >= 0)
      PrevOutput.BufferFree();
   if(!PrevOutput.Load(file_handle))
      return false;
   if(!BackendBufferCreate(PrevOutput))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(Gradient.GetIndex() >= 0)
      Gradient.BufferFree();
   if(!Gradient.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Gradient))
      return false;
//---
   if(FileReadInteger(file_handle) == 0)
      return true;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      Weights = new CBufferDouble();
      if(CheckPointer(Weights) == POINTER_INVALID)
         return false;
     }
   if(Weights.GetIndex() >= 0)
      Weights.BufferFree();
   if(!Weights.Load(file_handle))
      return false;
   if(!BackendBufferCreate(Weights))
      return false;
//---
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID)
        {
         DeltaWeights = new CBufferDouble();
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
            return false;
        }
      if(DeltaWeights.GetIndex() >= 0)
         DeltaWeights.BufferFree();
      if(!DeltaWeights.Load(file_handle))
         return false;
      if(!BackendBufferCreate(DeltaWeights))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID)
        {
         FirstMomentum = new CBufferDouble();
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
            return false;
        }
      if(FirstMomentum.GetIndex() >= 0)
         FirstMomentum.BufferFree();
      if(!FirstMomentum.Load(file_handle))
         return false;
      if(!BackendBufferCreate(FirstMomentum))
         return false;
      //---
      if(CheckPointer(SecondMomentum) == POINTER_INVALID)
        {
         SecondMomentum = new CBufferDouble();
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
            return false;
        }
      if(SecondMomentum.GetIndex() >= 0)
         SecondMomentum.BufferFree();
      if(!SecondMomentum.Load(file_handle))
         return false;
      if(!BackendBufferCreate(SecondMomentum))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+