Warrior_EA/AI/Network.mqh

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include <Arrays\ArrayDouble.mqh>
#include <Arrays\ArrayInt.mqh>
#include <Arrays\ArrayObj.mqh>
#include <OpenCL\OpenCL.mqh>
//--- kept local to this file (rather than Enumerations\InputEnums.mqh) since AI\Network.mqh is
//--- included well before Variables\Inputs.mqh in Warrior_EA.mq5's include chain - this input has
//--- to be self-contained here regardless.
enum CPU_LOAD_PRESET
  {
   CPU_LOAD_10 = 10,  // 10% of detected cores
   CPU_LOAD_20 = 20,  // 20% of detected cores
   CPU_LOAD_30 = 30,  // 30% of detected cores
   CPU_LOAD_40 = 40,  // 40% of detected cores
   CPU_LOAD_50 = 50,  // 50% of detected cores
   CPU_LOAD_60 = 60,  // 60% of detected cores
   CPU_LOAD_70 = 70,  // 70% of detected cores
   CPU_LOAD_80 = 80,  // 80% of detected cores
   CPU_LOAD_90 = 90,  // 90% of detected cores
   CPU_LOAD_MAX = 100 // Max - all detected cores
  };
//--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available,
//--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Caps how
//--- many of the auto-detected CPU cores WarriorCPU.dll's worker thread pool actually uses - has
//--- no effect at all when a GPU tier (OpenCL or DirectML) is active, since neither one calls into
//--- WarriorCPU.dll.
//--- Applied directly, undivided, to every CNet's own WarriorCPU.dll worker pool (live Net, shadow
//--- Net, and - under AIType=HYBRID - each of PAI/CONV/LSTM's own pair) even though several such
//--- pools can be alive at once. An earlier version divided this by how many CNet instances were
//--- alive (g_netPeerCount/EffectiveCpuLoadPercent()), reasoning that N pools each sized at 100%
//--- would N-way oversubscribe the CPU - but MQL5 gives one chart's EA a single execution thread,
//--- and every WarriorCPU.dll entry point (CPU_FeedForward/CPU_CalcOutputGradient/etc.) blocks that
//--- thread until its own ParallelFor() completes, so only ONE pool is EVER actively computing at a
//--- time - the rest sit idle (blocked on a condvar, ~0% CPU) regardless of how many exist. Dividing
//--- was pure waste: it capped whichever pool happened to be running at a fraction of the cores the
//--- user asked for, without preventing any contention that was never actually possible.
input CPU_LOAD_PRESET TargetCPULoad = CPU_LOAD_MAX; // CPU worker thread cap when no GPU accel (% cores)
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
//--- kept local to this file for the same include-order reason as CPU_LOAD_PRESET/TargetCPULoad
//--- above - AI\Network.mqh is parsed well before Variables\Inputs.mqh.
//--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference
//--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's
//--- abstract "0.001" mention, which the book's own worked examples don't actually use either.
//--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005)
//--- this project had separately tuned lr to before this input existed, so no behavior conflict.
//--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak
//--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to
//--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this
//--- reverts to the literature/book default.
input double AdamLearningRate = 0.0003; // Adam learning rate (eta ceiling) - book default 3.0e-4
input double AdamBeta1         = 0.9;   // Adam beta1 (1st-moment decay) - book default 0.9
input double AdamBeta2         = 0.999; // Adam beta2 (2nd-moment decay) - book default 0.999
//--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method
//--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses
//--- Adam's book-default rate as its starting point too. The book also states no numeric default for
//--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses
//--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for.
input double SgdLearningRate   = 0.0003; // SGD+momentum learning rate - book default 3.0e-4
input double SgdMomentum       = 0.9;    // SGD+momentum decay - reuses Adam's beta1 default
#define  lr AdamLearningRate
#define  b1 AdamBeta1
#define  b2 AdamBeta2
#define  momentum SgdMomentum
double eta = lr;
#define defConnect         0x7781
#define defArrayConnects   0x7782
#define defNeuronBase      0x7783
#define defNeuron          0x7784
#define defNeuronConv      0x7785
#define defNeuronPool     0x7786
#define defLayer           0x7787
#define defArrayLayer      0x7788
#define defNet             0x7789
#define defNeuronLSTM      0x7791
//---
#define defBufferDouble    0x7882
#define defNeuronBaseOCL   0x7883
#define defNeuronLSTMOCL   0x7884
#define defNeuronConvOCL   0x7885
#define defNeuronPoolOCL   0x7886
//---
#define def_k_FeedForward     0
#define def_k_ff_matrix_w     0
#define def_k_ff_matrix_i     1
#define def_k_ff_matrix_o     2
#define def_k_ff_inputs       3
#define def_k_ff_activation   4
//---
#define def_k_CaclOutputGradient 1
#define def_k_cog_matrix_t       0
#define def_k_cog_matrix_o       1
#define def_k_cog_matrix_ig      2
#define def_k_cog_activation     3
//---
#define def_k_CaclHiddenGradient 2
#define def_k_chg_matrix_w       0
#define def_k_chg_matrix_g       1
#define def_k_chg_matrix_o       2
#define def_k_chg_matrix_ig      3
#define def_k_chg_outputs        4
#define def_k_chg_activation     5
//---
#define def_k_UpdateWeightsMomentum      3
#define def_k_uwm_matrix_w        0
#define def_k_uwm_matrix_g        1
#define def_k_uwm_matrix_i        2
#define def_k_uwm_matrix_dw       3
#define def_k_uwm_inputs          4
#define def_k_uwm_learning_rates  5
#define def_k_uwm_momentum        6
//---
#define def_k_UpdateWeightsAdam   4
#define def_k_uwa_matrix_w        0
#define def_k_uwa_matrix_g        1
#define def_k_uwa_matrix_i        2
#define def_k_uwa_matrix_m        3
#define def_k_uwa_matrix_v        4
#define def_k_uwa_inputs          5
#define def_k_uwa_l               6
#define def_k_uwa_b1              7
#define def_k_uwa_b2              8
//---
#define def_k_FeedForwardProof    15
#define def_k_ffp_matrix_i        0
#define def_k_ffp_matrix_o        1
#define def_k_ffp_inputs          2
#define def_k_ffp_window          3
#define def_k_ffp_step            4
//---
#define def_k_CalcInputGradientProof 16
#define def_k_cigp_matrix_i       0
#define def_k_cigp_matrix_g       1
#define def_k_cigp_matrix_o       2
#define def_k_cigp_matrix_ig      3
#define def_k_cigp_outputs        4
#define def_k_cigp_window         5
#define def_k_cigp_step           6
//---
#define def_k_FeedForwardConv     5
#define def_k_ffc_matrix_w        0
#define def_k_ffc_matrix_i        1
#define def_k_ffc_matrix_o        2
#define def_k_ffc_inputs          3
#define def_k_ffc_step            4
#define def_k_ffc_window_in       5
#define def_k_ffc_window_out      6
#define def_k_ffc_activation      7
//---
#define def_k_CalcHiddenGradientConv 6
#define def_k_chgc_matrix_w       0
#define def_k_chgc_matrix_g       1
#define def_k_chgc_matrix_o       2
#define def_k_chgc_matrix_ig      3
#define def_k_chgc_outputs        4
#define def_k_chgc_step           5
#define def_k_chgc_window_in      6
#define def_k_chgc_window_out     7
#define def_k_chgc_activation     8
//---
#define def_k_UpdateWeightsConvMomentum 7
#define def_k_uwcm_matrix_w       0
#define def_k_uwcm_matrix_g       1
#define def_k_uwcm_matrix_i       2
#define def_k_uwcm_matrix_dw      3
#define def_k_uwcm_inputs         4
#define def_k_uwcm_learning_rates 5
#define def_k_uwcm_momentum       6
#define def_k_uwcm_window_in      7
#define def_k_uwcm_window_out     8
#define def_k_uwcm_step           9
//---
#define def_k_UpdateWeightsConvAdam 8
#define def_k_uwca_matrix_w       0
#define def_k_uwca_matrix_g       1
#define def_k_uwca_matrix_i       2
#define def_k_uwca_matrix_m       3
#define def_k_uwca_matrix_v       4
#define def_k_uwca_inputs         5
#define def_k_uwca_l              6
#define def_k_uwca_b1             7
#define def_k_uwca_b2             8
#define def_k_uwca_window_in      9
#define def_k_uwca_window_out     10
#define def_k_uwca_step           11
//---
// LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient
// flows back into h_prev/c_prev from a prior step). Supports both Adam and
// SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) -
// see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch.
// Derived from scratch from the standard LSTM equations - NOT ported from
// the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites
// the live weights buffer instead of writing to weights_gradient.
#define def_k_LSTM_Gates          9
#define def_k_lstmg_matrix_w      0
#define def_k_lstmg_hidden_prev   1
#define def_k_lstmg_inputs        2
#define def_k_lstmg_concatenated  3
#define def_k_lstmg_hidden_size   4
#define def_k_lstmg_input_size    5
//---
#define def_k_LSTM_State          10
#define def_k_lstms_concatenated  0
#define def_k_lstms_memory        1
#define def_k_lstms_hidden_prev   2
#define def_k_lstms_hidden_cache  3
#define def_k_lstms_output        4
#define def_k_lstms_hidden_size   5
//---
#define def_k_LSTM_GateGradient   11
#define def_k_lstmgg_gradient     0
#define def_k_lstmgg_memory       1
#define def_k_lstmgg_concatenated 2
#define def_k_lstmgg_concatenated_gradient 3
#define def_k_lstmgg_hidden_size  4
//---
#define def_k_LSTM_WeightsGradient 12
#define def_k_lstmwg_concatenated_gradient 0
#define def_k_lstmwg_hidden_cache 1
#define def_k_lstmwg_inputs       2
#define def_k_lstmwg_weights_gradient 3
#define def_k_lstmwg_hidden_size  4
#define def_k_lstmwg_input_size   5
//---
#define def_k_LSTM_InputsGradient 13
#define def_k_lstmig_concatenated_gradient 0
#define def_k_lstmig_matrix_w     1
#define def_k_lstmig_inputs_gradient 2
#define def_k_lstmig_hidden_size  3
#define def_k_lstmig_input_size   4
//---
#define def_k_LSTM_UpdateWeightsAdam 14
#define def_k_lstmuwa_matrix_w    0
#define def_k_lstmuwa_weights_gradient 1
#define def_k_lstmuwa_matrix_m    2
#define def_k_lstmuwa_matrix_v    3
#define def_k_lstmuwa_l           4
#define def_k_lstmuwa_b1          5
#define def_k_lstmuwa_b2          6
//---
// SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see
// AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body.
#define def_k_LSTM_UpdateWeightsMomentum 17
#define def_k_lstmuwm_matrix_w    0
#define def_k_lstmuwm_weights_gradient 1
#define def_k_lstmuwm_matrix_dw   2
#define def_k_lstmuwm_learning_rates 3
#define def_k_lstmuwm_momentum    4
//---
// b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see
// AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era
// same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root
// cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE
// gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever
// could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled
// kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam
// signatures), so they're safe to expose as ordinary inputs.
// Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had
// already gone unstable (real collapses were happening at weight magnitudes several orders of
// magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh
// engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually
// reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below.
#define MAX_WEIGHT                100.0
// Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in
// DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see
// WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright
// +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of
// training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi-
// hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles.
// 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates,
// ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's
// sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a
// weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the
// discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the
// calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the
// spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt
// dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay
// strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins
// argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth.
#define WEIGHT_DECAY              0.001
// Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full
// rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden
// Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras),
// most likely from 5x back-to-back oversampling replay building artificially correlated momentum.
// Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the
// post-update weight value and is far too loose (1e6) to prevent this.
#define MAX_WEIGHT_DELTA          0.1
// Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see
// SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its
// activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that
// neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated
// unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in
// Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine.
#define MIN_ACTIVATION_DERIVATIVE 1.0e-4
// Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND
// read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored against a
// different distribution than it was trained on). The classification outputs are SIGMOID-bounded to
// [0,1], so the raw logit spread can never exceed 1 and the softmax winner caps at e/(e+2)=0.576 -
// the one-hot 1.0 target is unreachable, per-sample gradients never decay below ~0.42, and training
// can only orbit, never converge (observed as IS error frozen at sqrt(1/3)=0.58 with all three
// outputs saturated at 0). Scaling the logits by 6 stretches the spread to [0,6], raising the
// ceiling to e^6/(e^6+2)=0.995: targets effectively reachable, gradients can vanish, and the focal
// modulation's pt finally spans (0,1) instead of (0.21,0.58). The gradient deliberately stays
// (target - softmax) WITHOUT the extra 6x chain-rule factor - the scale is defined as part of the
// loss, keeping gradient magnitudes (and thus eta tuning) unchanged.
#define CLASS_LOGIT_SCALE         6.0

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#resource "Network.cl" as string cl_program
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
enum ENUM_ACTIVATION
  {
   NONE,
   TANH,
   SIGMOID,
   PRELU     // fixed param=0.01, matches CNeuronConv's CPU activationFunction
  };
//+------------------------------------------------------------------+
//| Translates ENUM_ACTIVATION to the "activation" int code every     |
//| native compute backend actually understands (Network.cl's         |
//| kernels, and the mirrored Activation()/inline switches in         |
//| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and  |
//| deliberately anything else (incl. NONE) falls through every one   |
//| of those switches unmatched, which is exactly linear passthrough -|
//| there's no case 3 anywhere on the native side, so PRELU relies on |
//| the FeedForwardConv-family kernels specifically, and NONE never   |
//| needs a case at all. This is NOT the same numbering as            |
//| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a    |
//| raw (int)activation cast at a kernel call site silently sends the |
//| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native     |
//| SIGMOID). Only use this at actual kernel-dispatch call sites -     |
//| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw         |
//| ENUM_ACTIVATION value instead, and must keep using (int)activation |
//| directly so saved topology files round-trip through Load() as-is. |
//+------------------------------------------------------------------+
int NativeActivationCode(ENUM_ACTIVATION value)
  {
   switch(value)
     {
      case TANH:    return 0;
      case SIGMOID: return 1;
      case PRELU:   return 2;
      default:      return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches
     }
  }
//---
enum ENUM_OPTIMIZATION
  {
   SGD,  // SGD + Momentum (heavy-ball, simpler, needs more eras)
   ADAM  // Adam (adaptive step, faster convergence, can overfit)
  };
//---
enum ENUM_BUFFERS
  {
   WEIGHTS,
   DELTA_WEIGHTS,
   OUTPUT,
   GRADIENT,
   FIRST_MOMENTUM,
   SECOND_MOMENTUM
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "NeuronPrimitives.mqh"
class CLayer;
//---
class CNeuronBase    :  public CObject
  {
protected:
   double            outputVal;
   double            prevVal;
   uint              m_myIndex;
   double            gradient;
   CArrayCon         *Connections;
   ENUM_ACTIVATION   activation;
   ENUM_OPTIMIZATION optimization;
   int               t;
   //---
   virtual bool      feedForward(CLayer *prevLayer)               {  return false;     }
   virtual bool      calcHiddenGradients(CLayer *&nextLayer)     {  return false;     }
   virtual bool      updateInputWeights(CLayer *&prevLayer)       {  return false;     }
   virtual double    activationFunction(double x);
   virtual double    SigmoidFunction(double x)                    {  return MathPow(1 + exp(-x), -1); }
   virtual double    TanhFunction(double x)                       {  return tanh(x);   }
   virtual CLayer    *getOutputLayer(void)                        {  return NULL;      }
public:
                     CNeuronBase(void);
                    ~CNeuronBase(void);
   virtual bool      Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }
   //---
   static double            alpha;
   //---
   virtual void      setOutputVal(double val)                     {  prevVal = outputVal;   outputVal = val;    }
   virtual double    getOutputVal()                               {  return outputVal; }
   virtual double    getPrevVal()                                 {  return prevVal;   }
   virtual void      setGradient(double val)                      {  gradient = val;     }
   virtual double    getGradient()                                {  return gradient;  }
   virtual CArrayCon *getConnections()                            {  return Connections;}
   virtual double    activationFunctionDerivative(double x);
   virtual double    SigmoidFunctionDerivative(double x)          {  return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x));   }
   virtual double    TanhFunctionDerivative(double x)             {  return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x));  }
   //---
   virtual bool      feedForward(CObject *&SourceObject);
   virtual bool      calcHiddenGradients(CObject *&TargetObject);
   virtual bool      updateInputWeights(CObject *&SourceObject);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle)
     {
      activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
      optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
      return(Connections.Load(file_handle));
     }
   //---
   virtual int       Type(void)        const                      {  return defNeuronBase;                  }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::alpha = momentum;     // momentum
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBase::CNeuronBase(void)  :
   outputVal(1),
   gradient(0),
   activation(TANH),
   t(1),
   optimization(SGD)
  {
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBase::~CNeuronBase(void)
  {
   if(CheckPointer(Connections) != POINTER_INVALID)
      delete Connections;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(Connections) == POINTER_INVALID)
     {
      Connections = new CArrayCon();
      if(CheckPointer(Connections) == POINTER_INVALID)
         return false;
     }
//---
   if(Connections.Reserve(fmax(numOutputs, 1)))
      for(uint c = 0; c < numOutputs; c++)
        {
         if(!Connections.CreateElement(c))
            return false;
         Connections.IncreaseTotal();
        }
//---
   m_myIndex = myIndex;
   optimization = optimization_type;
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "NeuronCPU.mqh"
class COpenCLMy   :  public COpenCL
  {
public:
                     COpenCLMy(void)   {};
                    ~COpenCLMy(void)   {};
   template<typename T>
   int               AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
  };
#include "NeuronDirectML.mqh"
class CLayer: public CArrayObj
  {
private:
   uint              iOutputs;
   int               iFileHandle;
   int               hWeights;
   int               hDeltaWeights;
   int               hOutput;
   int               hGradient;
   COpenCLMy         *OpenCL;
   CDirectMLMy       *DirectML;

public:
                     CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL);
                    ~CLayer(void) {};
   //---
   virtual bool      CreateElement(int const index);
   virtual void      IncreaseTotal()   {  m_data_total++;   }
   virtual int       Type(void)  const { return defLayer; }
   virtual bool      Load(const int file_handle);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CLayer::CreateElement(int index)
  {
   if(index >= m_data_max)
      return false;
//---
   bool result = false;
   CNeuronBase *temp = NULL;
   CNeuronPool *temp_p = NULL;
   CNeuronBaseOCL *temp_ocl = NULL;
   if(iFileHandle <= 0)
     {
      temp = new CNeuron();
      if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD))
         return false;
      result = true;
     }
   else
     {
      int type = FileReadInteger(iFileHandle);
      switch(type)
        {
         case  defNeuron:
            temp = new CNeuron();
            if(CheckPointer(temp) == POINTER_INVALID)
               result = false;
            result = temp.Init(iOutputs, index, ADAM);
            break;
         case  defNeuronPool:
            temp_p = new CNeuronPool();
            if(CheckPointer(temp_p) == POINTER_INVALID)
               result = false;
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronConv:
            temp_p = new CNeuronConv();
            if(CheckPointer(temp_p) == POINTER_INVALID)
               result = false;
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronLSTM:
            temp_p = new CNeuronLSTM();
            if(CheckPointer(temp_p) == POINTER_INVALID)
               result = false;
            if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
              {
               temp = temp_p;
               result = true;
              }
            break;
         case  defNeuronBaseOCL:
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
               return false;
            temp_ocl = new CNeuronBaseOCL();
            if(CheckPointer(temp_ocl) == POINTER_INVALID)
               result = false;
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM)
               : temp_ocl.Init(iOutputs, index, DirectML, 1, ADAM))
              {
               m_data[index] = temp_ocl;
               return true;
              }
            break;
         case  defNeuronConvOCL:
           {
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
               return false;
            //--- placeholder dims; the real window/step/units are restored by Load() right after
            CNeuronConvOCL *temp_conv = new CNeuronConvOCL();
            if(CheckPointer(temp_conv) == POINTER_INVALID)
               result = false;
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM)
               : temp_conv.Init(iOutputs, index, DirectML, 1, 1, 1, 1, ADAM))
              {
               m_data[index] = temp_conv;
               return true;
              }
            break;
           }
         case  defNeuronPoolOCL:
           {
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
               return false;
            CNeuronPoolOCL *temp_pool = new CNeuronPoolOCL();
            if(CheckPointer(temp_pool) == POINTER_INVALID)
               result = false;
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_pool.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM)
               : temp_pool.Init(iOutputs, index, DirectML, 1, 1, 1, ADAM))
              {
               m_data[index] = temp_pool;
               return true;
              }
            break;
           }
         case  defNeuronLSTMOCL:
           {
            if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
               return false;
            CNeuronLSTMOCL *temp_lstm = new CNeuronLSTMOCL();
            if(CheckPointer(temp_lstm) == POINTER_INVALID)
               result = false;
            if(CheckPointer(OpenCL) != POINTER_INVALID
               ? temp_lstm.Init(iOutputs, index, OpenCL, 1, ADAM)
               : temp_lstm.Init(iOutputs, index, DirectML, 1, ADAM))
              {
               m_data[index] = temp_lstm;
               return true;
              }
            break;
           }
         default:
            result = false;
            break;
        }
     }
   if(result)
      m_data[index] = temp;
//---
   return (result);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "ArrayLayer.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronPool : public CNeuronBase
  {
protected:
   CLayer            *OutputLayer;
   int               iWindow;
   int               iStep;

   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);

public:
                     CNeuronPool(void) {};
                    ~CNeuronPool(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronPool;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronConv  :  public CNeuronPool
  {
protected:
   double            param;   //PReLU param
   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);
   virtual double    activationFunction(double x);
   virtual bool      updateInputWeights(CLayer *&prevLayer);
public:
                     CNeuronConv() :   param(0.01) { };
                    ~CNeuronConv(void)             { };
   //---
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   virtual double    activationFunctionDerivative(double x);
   virtual int       Type(void)   const   {  return defNeuronConv;   }
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::feedForward(CObject *&SourceObject)
  {
   bool result = false;
//---
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(SourceObject.Type())
     {
      case defLayer:
         temp_l = SourceObject;
         result = feedForward(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         temp_n = SourceObject;
         result = feedForward(temp_n.getOutputLayer());
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::updateInputWeights(CObject *&SourceObject)
  {
   bool result = false;
//---
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(SourceObject.Type())
     {
      case defLayer:
         temp_l = SourceObject;
         result = updateInputWeights(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         temp_n = SourceObject;
         temp_l = temp_n.getOutputLayer();
         result = updateInputWeights(temp_l);
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::feedForward(CLayer *prevLayer)
  {
   bool result = false;
//---
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return result;
//---
   int total = prevLayer.Total() - iWindow + 1;
   CNeuron *temp;
   CConnection *con;
   result = true;
   for(int i = 0; (i < total && result); i += iStep)
     {
      double sum = 0;
      for(int j = 0; (j < iWindow && result); j++)
        {
         temp = prevLayer.At(i + j);
         con = Connections.At(j);
         if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
            return false;
         double val = temp.getOutputVal();
         sum += val * con.weight;
        }
      temp = OutputLayer.At(i / iStep);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setOutputVal(activationFunction(sum));
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunction(double x)
  {
   if(x >= 0)
      return x;
   return param * x;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject)
  {
   bool result = false;
//---
   if(CheckPointer(TargetObject) == POINTER_INVALID)
      return result;
//---
   CLayer *temp_l;
   CNeuronPool *temp_n;
   switch(TargetObject.Type())
     {
      case defLayer:
         temp_l = TargetObject;
         result = calcHiddenGradients(temp_l);
         break;
      case defNeuronConv:
      case defNeuronPool:
      case defNeuronLSTM:
         switch(Type())
           {
            case defNeuron:
               temp_n = TargetObject;
               result = temp_n.calcInputGradients(GetPointer(this), m_myIndex);
               break;
            case defNeuronLSTM:
               temp_n = TargetObject;
               temp_l = getOutputLayer();
               if(!temp_n.calcInputGradients(temp_l))
                 {
                  result = false;
                  break;
                 }
               result = calcHiddenGradients(temp_l);
               break;
            default:
               temp_l =getOutputLayer();
               temp_n = TargetObject;
               result = temp_n.calcInputGradients(temp_l);
               break;
           }
         break;
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
      return false;
//---
   gradient = 0;
   int total = OutputLayer.Total();
   CNeuron *temp;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal()));
     }
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunctionDerivative(double x)
  {
   if(x >= 0)
      return 1;
   return param;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::updateInputWeights(CLayer *&prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   CConnection *con;
   double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
   for(int n = 0; n < iWindow && !IsStopped(); n++)
     {
      con = Connections.At(n);
      if(CheckPointer(con) == POINTER_INVALID)
         continue;
      double delta = 0;
      int total_i = OutputLayer.Total();
      CNeuron *prev, *out;
      for(int i = 0; i < total_i; i++)
        {
         prev = prevLayer.At(n * iStep + i);
         out = OutputLayer.At(total_i - i - 1);
         if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID)
            continue;
         delta += prev.getOutputVal() * out.getGradient();
        }
      if(optimization == SGD)
         con.weight += con.deltaWeight = (delta != 0 ? eta*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0);
      else
        {
         con.mt = b1 * con.mt + (1 - b1) * delta;
         con.vt = b2 * con.vt + (1 - b2) * delta * delta + 0.00000001;
         con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
         // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why.
         con.weight += con.deltaWeight;
        }
      // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
      // AI\Network.cl's UpdateWeightsConvMomentum/UpdateWeightsConvAdam MAX_WEIGHT clamp.
      con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
     }
   if(optimization == ADAM)
      t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
  {
   iWindow = window;
   iStep = step;
   if(!CNeuronBase::Init(window, myIndex, optimization_type))
      return false;
   OutputLayer = new CLayer(numOutputs);
   if(CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
   if(OutputLayer.Reserve(units_count))
      for(int i = 0; i < units_count; i++)
        {
         if(!OutputLayer.CreateElement(i))
            return false;
         OutputLayer.IncreaseTotal();
        }
//---
   if(Type() == defNeuronPool)
     {
      if(CheckPointer(Connections) != POINTER_INVALID)
         Connections.Clear();
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronPool::~CNeuronPool(void)
  {
   delete OutputLayer;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::feedForward(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return false;
//---
   int total = prevLayer.Total() - iWindow + 1;
   CNeuron *temp;
   for(int i = 0; i <= total; i += iStep)
     {
      double sum = 0;
      for(int j = 0; j < iWindow; j++)
        {
         temp = prevLayer.At(i + j);
         if(CheckPointer(temp) == POINTER_INVALID)
            continue;
         sum += temp.getOutputVal();
        }
      temp = OutputLayer.At(i / iStep);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setOutputVal(sum / iWindow);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
      return false;
//---
   gradient = 0;
   int total = OutputLayer.Total();
   CNeuron *temp;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(temp.sumDOW(nextLayer));
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID  || CheckPointer(prevLayer.At(0)) == POINTER_INVALID)
      return false;
//---
   if(prevLayer.At(0).Type() != defNeuron)
     {
      CNeuronPool *temp = prevLayer.At(m_myIndex);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      prevLayer = temp.getOutputLayer();
      if(CheckPointer(prevLayer) == POINTER_INVALID)
         return false;
     }
//---
   CNeuronBase *prevNeuron, *outputNeuron;
   int total = prevLayer.Total();
   for(int i = 0; i < total; i++)
     {
      prevNeuron = prevLayer.At(i);
      if(CheckPointer(prevNeuron) == POINTER_INVALID)
         continue;
      double prev_gradient = 0;
      int start = i - iWindow + iStep;
      start = (start - start % iStep) / iStep;
      double stop = (i - i % iStep) / iStep + 1;
      for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
        {
         outputNeuron = OutputLayer.At(out);
         if(CheckPointer(outputNeuron) == POINTER_INVALID)
            continue;
         prev_gradient += outputNeuron.getGradient() / iWindow;
        }
      prevNeuron.setGradient(prev_gradient);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevNeuron.Type() != defNeuron)
     {
      CNeuronPool *temp = prevNeuron;
      return calcInputGradients(temp.getOutputLayer());
     }
//---
   CNeuronBase *outputNeuron;
   double prev_gradient = 0;
   int start = (int)index - iWindow + iStep;
   start = (start - start % iStep) / iStep;
   double stop = (index - index % iStep) / iStep + 1;
   for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
     {
      outputNeuron = OutputLayer.At(out);
      if(CheckPointer(outputNeuron) == POINTER_INVALID)
         continue;
      prev_gradient += outputNeuron.getGradient() / iWindow;
     }
   prevNeuron.setGradient(prev_gradient);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevLayer.At(0).Type() != defNeuron)
     {
      CNeuronPool *temp = prevLayer.At(m_myIndex);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      prevLayer = temp.getOutputLayer();
      if(CheckPointer(prevLayer) == POINTER_INVALID)
         return false;
     }
//---
   CNeuronBase *prevNeuron, *outputNeuron;
   CConnection *con;
   int total = prevLayer.Total();
   for(int i = 0; i < total; i++)
     {
      prevNeuron = prevLayer.At(i);
      if(CheckPointer(prevNeuron) == POINTER_INVALID)
         continue;
      double prev_gradient = 0;
      int start = i - iWindow + iStep;
      start = (start - start % iStep) / iStep;
      double stop = (i - i % iStep) / iStep + 1;
      for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
        {
         outputNeuron = OutputLayer.At(out);
         int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep;
         con = Connections.At(c);
         if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
            continue;
         prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight;
        }
      prevNeuron.setGradient(prev_gradient);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
      return false;
//---
   if(prevNeuron.Type() != defNeuron)
     {
      CNeuronPool *temp = prevNeuron;
      return calcInputGradients(temp.getOutputLayer());
     }
//---
   CNeuronBase *outputNeuron;
   CConnection *con;
   double prev_gradient = 0;
   int start = (int)index - iWindow + iStep;
   start = (start - start % iStep) / iStep;
   double stop = (index - index % iStep) / iStep + 1;
   for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
     {
      outputNeuron = OutputLayer.At(out);
      int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep);
      con = Connections.At(c);
      if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
         continue;
      prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight;
     }
   prevNeuron.setGradient(prev_gradient);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBase::Save(int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
   if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE)
      return false;
//---
   return Connections.Save(file_handle);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "LayerDescription.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNet
  {
protected:
   void              backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0);
   bool              InitOpenCL(void);
   bool              InitDirectML(void);
public:
                     CNet(CArrayObj *Description);
                    ~CNet(void);
   bool              feedForward(CArrayDouble *inputVals);
   //--- sampleWeight scales this example's output-layer gradient before it propagates back through the
   //--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling
   //--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same
   //--- example multiple times).
   void              backProp(CArrayDouble *targetVals, double sampleWeight = 1.0);
   void              getResults(CArrayDouble *&resultVals) ;
   double            getRecentAverageError()  { return recentAverageError; }
   //--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see
   //--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array
   //--- when there is nothing to persist/restore.
   bool              Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]);
   bool              Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[]);
   //--- ephemeral in-run weight checkpoints (agent-local scratch file, never FILE_COMMON): unlike
   //--- Save()/Load() these are NOT blocked in the tester/optimizer, since they exist only to let a
   //--- single Train() call snapshot/restore weights mid-run (stability-based early stopping) and
   //--- never persist a model across separate backtest or optimization passes.
   bool              SaveCheckpoint(string file_name);
   bool              LoadCheckpoint(string file_name);
   //--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another
   //--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight +
   //--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS
   //--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops
   //--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the
   //--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit
   //--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over
   //--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation.
   //--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) -
   //--- true whenever the shadow was cloned from live via Save()/Load() and never independently
   //--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a
   //--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers.
   bool              BlendWeightsFrom(CNet &live, double tau);
   //--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly
   //--- initialized and carry the real learning signal) of each output neuron's incoming weight block,
   //--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site
   //--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to
   //--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more
   //--- non-majority classes than the true base rate warrants until backProp corrects it over many
   //--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the
   //--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns
   //--- false (no-op) rather than corrupt anything if that assumption doesn't hold.
   bool              SeedOutputLayerBias(const double &biasValues[]);
   //---
   static double     recentAverageSmoothingFactor;

private:
   CArrayLayer       *layers;
   COpenCLMy         *opencl;
   CDirectMLMy       *directml;
   double            recentAverageError;
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNet::CNet(CArrayObj *Description)
  {
   if(CheckPointer(Description) == POINTER_INVALID)
      return;
//---
   int total = Description.Total();
   if(total <= 0)
      return;
//---
   layers = new CArrayLayer();
   if(CheckPointer(layers) == POINTER_INVALID)
      return;
//---
   CLayer *temp;
   CLayerDescription *desc = NULL, *next = NULL, *prev = NULL;
   CNeuronBase *neuron = NULL;
   CNeuronPool *neuron_p = NULL;
   int output_count = 0;
   int temp_count = 0;
//---
   next = Description.At(1);
   if(CheckPointer(next) != POINTER_INVALID &&
      (next.type == defNeuron || next.type == defNeuronBaseOCL || next.type == defNeuronConv || next.type == defNeuronConvOCL ||
       next.type == defNeuronLSTM))
     {
      //--- OpenCL first, DirectML/D3D12 next, plain CPU as the final fallback
      if(!InitOpenCL())
         InitDirectML();
     }
//---
   for(int i = 0; i < total; i++)
     {
      prev = desc;
      desc = Description.At(i);
      if((i + 1) < total)
        {
         next = Description.At(i + 1);
         if(CheckPointer(next) == POINTER_INVALID)
            return;
        }
      else
         next = NULL;
      int outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL) ? 0 : next.count);
      temp = new CLayer(outputs);
      int neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL ? 1 : 0));
      if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
        {
         CNeuronBaseOCL *neuron_ocl = NULL;
         switch(desc.type)
           {
            case defNeuron:
            case defNeuronBaseOCL:
               neuron_ocl = new CNeuronBaseOCL();
               if(CheckPointer(neuron_ocl) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               if(CheckPointer(opencl) != POINTER_INVALID
                  ? !neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization)
                  : !neuron_ocl.Init(outputs, 0, directml, desc.count, desc.optimization))
                 {
                  delete temp;
                  return;
                 }
               neuron_ocl.SetActivationFunction(desc.activation);
               if(!temp.Add(neuron_ocl))
                 {
                  delete neuron_ocl;
                  delete temp;
                  return;
                 }
               neuron_ocl = NULL;
               break;
            case defNeuronConv:
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *neuron_conv = new CNeuronConvOCL();
               if(CheckPointer(neuron_conv) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               //--- number of sliding positions - same formula the CPU CNeuronConv path uses
               if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
                 {
                  int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
                  temp_count = (prevCount - desc.window) % desc.step;
                  output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               else
                 {
                  temp_count = (output_count - desc.window) % desc.step;
                  output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               bool convInit = (CheckPointer(opencl) != POINTER_INVALID
                                 ? neuron_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, output_count, desc.optimization)
                                 : neuron_conv.Init(outputs, 0, directml, desc.window, desc.step, desc.count, output_count, desc.optimization));
               if(!convInit)
                 {
                  delete neuron_conv;
                  delete temp;
                  return;
                 }
               neuron_conv.SetActivationFunction(desc.activation);
               if(!temp.Add(neuron_conv))
                 {
                  delete neuron_conv;
                  delete temp;
                  return;
                 }
               neuron_conv = NULL;
               break;
              }
            case defNeuronPool:
            case defNeuronPoolOCL:
              {
               CNeuronPoolOCL *neuron_pool = new CNeuronPoolOCL();
               if(CheckPointer(neuron_pool) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               //--- number of sliding positions - same formula the CPU CNeuronPool path uses
               if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
                 {
                  int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
                  temp_count = (prevCount - desc.window) % desc.step;
                  output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               else
                 {
                  temp_count = (output_count - desc.window) % desc.step;
                  output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
                 }
               bool poolInit = (CheckPointer(opencl) != POINTER_INVALID
                                ? neuron_pool.Init(outputs, 0, opencl, desc.window, desc.step, output_count, desc.optimization)
                                : neuron_pool.Init(outputs, 0, directml, desc.window, desc.step, output_count, desc.optimization));
               if(!poolInit)
                 {
                  delete neuron_pool;
                  delete temp;
                  return;
                 }
               if(!temp.Add(neuron_pool))
                 {
                  delete neuron_pool;
                  delete temp;
                  return;
                 }
               neuron_pool = NULL;
               break;
              }
            case defNeuronLSTM:
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *neuron_lstm = new CNeuronLSTMOCL();
               if(CheckPointer(neuron_lstm) == POINTER_INVALID)
                 {
                  delete temp;
                  return;
                 }
               bool lstmInit = (CheckPointer(opencl) != POINTER_INVALID
                                 ? neuron_lstm.Init(outputs, 0, opencl, desc.count, desc.optimization)
                                 : neuron_lstm.Init(outputs, 0, directml, desc.count, desc.optimization));
               if(!lstmInit)
                 {
                  delete neuron_lstm;
                  delete temp;
                  return;
                 }
               if(!temp.Add(neuron_lstm))
                 {
                  delete neuron_lstm;
                  delete temp;
                  return;
                 }
               neuron_lstm = NULL;
               break;
              }
            default:
               return;
               break;
           }
        }
      else
         for(int n = 0; n < neurons; n++)
           {
            switch(desc.type)
              {
               case defNeuron:
                  neuron = new CNeuron();
                  if(CheckPointer(neuron) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  neuron.Init(outputs, n, desc.optimization);
                  neuron.SetActivationFunction(desc.activation);
                  break;
               case defNeuronConv:
                  neuron_p = new CNeuronConv();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  if(CheckPointer(prev) != POINTER_INVALID)
                    {
                     if(prev.type == defNeuron)
                       {
                        temp_count = (int)((prev.count - desc.window) % desc.step);
                        output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                       }
                     else
                        if(n == 0)
                          {
                           temp_count = (int)((output_count - desc.window) % desc.step);
                           output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                          }
                    }
                  if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
               case defNeuronPool:
                  neuron_p = new CNeuronPool();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  if(CheckPointer(prev) != POINTER_INVALID)
                    {
                     if(prev.type == defNeuron)
                       {
                        temp_count = (int)((prev.count - desc.window) % desc.step);
                        output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                       }
                     else
                        if(n == 0)
                          {
                           temp_count = (int)((output_count - desc.window) % desc.step);
                           output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
                          }
                    }
                  if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
               case defNeuronLSTM:
                  neuron_p = new CNeuronLSTM();
                  if(CheckPointer(neuron_p) == POINTER_INVALID)
                    {
                     delete temp;
                     delete layers;
                     return;
                    }
                  output_count = (next != NULL ? next.window : desc.step);
                  if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization))
                     neuron = neuron_p;
                  break;
              }
            if(!temp.Add(neuron))
              {
               delete temp;
               delete layers;
               return;
              }
            neuron = NULL;
           }
      if(!layers.Add(temp))
        {
         delete temp;
         delete layers;
         return;
        }
     }
//---
  }
//+------------------------------------------------------------------+
//| Tries to initialize OpenCL; on any failure (no GPU, driver     |
//| missing, kernel build error) frees it and leaves opencl==NULL  |
//| so the rest of CNet transparently runs its CPU code path.      |
//+------------------------------------------------------------------+
bool CNet::InitOpenCL(void)
  {
   if(CheckPointer(opencl) != POINTER_INVALID)
      return true;
//---
   opencl = new COpenCLMy();
   if(CheckPointer(opencl) == POINTER_INVALID || !opencl.Initialize(cl_program, true))
     {
      if(CheckPointer(opencl) != POINTER_INVALID)
         delete opencl;
      opencl = NULL;
      PrintFormat("%s: OpenCL unavailable, falling back to CPU", __FUNCTION__);
      return false;
     }
//--- create kernels
   opencl.SetKernelsCount(18);
   opencl.KernelCreate(def_k_FeedForward, "FeedForward");
   opencl.KernelCreate(def_k_CaclOutputGradient, "CaclOutputGradient");
   opencl.KernelCreate(def_k_CaclHiddenGradient, "CaclHiddenGradient");
   opencl.KernelCreate(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum");
   opencl.KernelCreate(def_k_UpdateWeightsAdam, "UpdateWeightsAdam");
   opencl.KernelCreate(def_k_FeedForwardConv, "FeedForwardConv");
   opencl.KernelCreate(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv");
   opencl.KernelCreate(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum");
   opencl.KernelCreate(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam");
   opencl.KernelCreate(def_k_LSTM_Gates, "LSTM_Gates");
   opencl.KernelCreate(def_k_LSTM_State, "LSTM_State");
   opencl.KernelCreate(def_k_LSTM_GateGradient, "LSTM_GateGradient");
   opencl.KernelCreate(def_k_LSTM_WeightsGradient, "LSTM_WeightsGradient");
   opencl.KernelCreate(def_k_LSTM_InputsGradient, "LSTM_InputsGradient");
   opencl.KernelCreate(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam");
   opencl.KernelCreate(def_k_LSTM_UpdateWeightsMomentum, "LSTM_UpdateWeightsMomentum");
   opencl.KernelCreate(def_k_FeedForwardProof, "FeedForwardProof");
   opencl.KernelCreate(def_k_CalcInputGradientProof, "CalcInputGradientProof");
   return true;
  }
//+------------------------------------------------------------------+
//| Second-tier GPU fallback: only tried when OpenCL init failed.    |
//| Requires DirectML\WarriorDML.dll in the terminal's Libraries     |
//| folder (build it with DirectML\build.bat); on any failure frees  |
//| itself and leaves directml==NULL so CNet falls through to CPU.   |
//+------------------------------------------------------------------+
bool CNet::InitDirectML(void)
  {
   if(CheckPointer(directml) != POINTER_INVALID)
      return true;
//---
   directml = new CDirectMLMy();
   if(CheckPointer(directml) == POINTER_INVALID)
      return false;
   directml.SetCpuLoadPercent((int)TargetCPULoad);
   if(!directml.Initialize())
     {
      int err = directml.LastError();
      delete directml;
      directml = NULL;
      string reason;
      switch(err)
        {
         case 1: reason = "CreateDXGIFactory1 failed";                                  break;
         case 2: reason = "no DX12 hardware adapter found (feature level 11_0)";        break;
         case 3: reason = "compute command queue creation failed";                      break;
         case 4: reason = "command allocator creation failed";                          break;
         case 5: reason = "command list creation failed";                               break;
         case 6: reason = "fence creation failed";                                      break;
         case 7: reason = "fence event creation failed";                                break;
         case 8: reason = "HLSL kernel compile/PSO creation failed";                    break;
         default: reason = "neither WarriorDML.dll nor WarriorCPU.dll loaded (check Libraries folder / \"Allow DLL imports\")";
        }
      PrintFormat("%s: DirectML/D3D12 and CPU DLL both unavailable (%s), falling back to slow per-object CPU path", __FUNCTION__, reason);
      return false;
     }
   if(directml.Tier() == COMPUTE_TIER_CPU)
      PrintFormat("%s: DirectML/D3D12 unavailable, using multithreaded CPU DLL fallback (%d threads)", __FUNCTION__, directml.CpuThreadsUsed());
   else
      PrintFormat("%s: DirectML/D3D12 GPU tier active", __FUNCTION__);
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::feedForward(CArrayDouble *inputVals)
  {
   if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1)
      return false;
//---
   CLayer *previous = NULL;
   CLayer *current = layers.At(0);
   int total = MathMin(current.Total(), inputVals.Total());
   CNeuronBase *neuron = NULL;
   bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID);
   if(!gpuActive)
     {
      for(int i = 0; i < total; i++)
        {
         neuron = current.At(i);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return false;
         neuron.setOutputVal(inputVals.At(i));
        }
     }
   else
     {
      CNeuronBaseOCL *neuron_ocl = current.At(0);
      int total_data = inputVals.Total();
      bool written;
      if(CheckPointer(opencl) != POINTER_INVALID)
        {
         //--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own
         //--- BufferWrite(), this call bypasses that class entirely (it writes straight into the
         //--- first layer's Output buffer by index), so the narrow-to-float has to happen here too.
         float array[];
         if(ArrayResize(array, total_data) < 0)
            return false;
         for(int d = 0; d < total_data; d++)
            array[d] = (float)inputVals.At(d);
         written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data);
        }
      else
        {
         double array[];
         if(ArrayResize(array, total_data) < 0)
            return false;
         for(int d = 0; d < total_data; d++)
            array[d] = inputVals.At(d);
         written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data);
        }
      if(!written)
         return false;
     }
//---
   CObject *temp = NULL;
   for(int l = 1; l < layers.Total(); l++)
     {
      previous = current;
      current = layers.At(l);
      if(CheckPointer(current) == POINTER_INVALID)
         return false;
      //---
      if(gpuActive)
        {
         CNeuronBaseOCL *current_ocl = current.At(0);
         if(!current_ocl.feedForward(previous.At(0)))
            return false;
         continue;
        }
      //---
      total = current.Total();
      if(current.At(0).Type() == defNeuron)
         total--;
      //---
      for(int n = 0; n < total; n++)
        {
         neuron = current.At(n);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return false;
         if(previous.At(0).Type() == defNeuron)
           {
            temp = previous;
            if(!neuron.feedForward(temp))
               return false;
            continue;
           }
         if(neuron.Type() == defNeuron)
           {
            if(n == 0)
              {
               CLayer *temp_l = new CLayer(total);
               if(CheckPointer(temp_l) == POINTER_INVALID)
                  return false;
               CNeuronPool *Pool = NULL;
               for(int p = 0; p < previous.Total(); p++)
                 {
                  Pool = previous.At(p);
                  if(CheckPointer(Pool) == POINTER_INVALID)
                     return false;
                  temp_l.AddArray(Pool.getOutputLayer());
                 }
               temp = temp_l;
              }
            if(!neuron.feedForward(temp))
               return false;
            if(n == total - 1)
              {
               CLayer *temp_l = temp;
               temp_l.FreeMode(false);
               temp_l.Shutdown();
               delete temp_l;
              }
            continue;
           }
         temp = previous.At(n);
         if(CheckPointer(temp) == POINTER_INVALID)
            return false;
         if(!neuron.feedForward(temp))
            return false;
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::backProp(CArrayDouble *targetVals, double sampleWeight)
  {
   if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
      return;
   if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
     {
      backPropOCL(targetVals, sampleWeight);
      return;
     }
//---
   CLayer *outputLayer = layers.At(layers.Total() - 1);
   if(CheckPointer(outputLayer) == POINTER_INVALID)
      return;
//---
   double error = 0.0;
   int total = outputLayer.Total() - 1;
   //--- 3-output classification case: true softmax + categorical-cross-entropy gradient
   //--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section
   //--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID
   //--- (bounded, avoids the historical logit-runaway collapse documented at
   //--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD
   //--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly,
   //--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/
   //--- Neutral together during training: raising one class's softmax probability now structurally
   //--- lowers the other two's (via the shared normalizing sum), giving real competition instead of
   //--- three independent binary regressions that can all drift toward "predict Neutral" together -
   //--- root cause of the "overshoot to all-Neutral" convergence failure this replaces.
   bool useSoftmaxGrad = (total == 3);
   double smax[3];
   if(useSoftmaxGrad)
     {
      double maxLogit = -DBL_MAX;
      for(int n = 0; n < 3; n++)
        {
         CNeuron *nrn = outputLayer.At(n);
         maxLogit = MathMax(maxLogit, CLASS_LOGIT_SCALE * nrn.getOutputVal());
        }
      double sum = 0.0;
      for(int n = 0; n < 3; n++)
        {
         CNeuron *nrn = outputLayer.At(n);
         smax[n] = exp(CLASS_LOGIT_SCALE * nrn.getOutputVal() - maxLogit);
         sum += smax[n];
        }
      for(int n = 0; n < 3; n++)
         smax[n] /= sum;
     }
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      CNeuron *neuron = outputLayer.At(n);
      double target = targetVals.At(n);
      double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
      double delta = clampedTarget - neuron.getOutputVal();
      error += delta * delta;
      if(useSoftmaxGrad)
         neuron.setGradient(clampedTarget - smax[n]);
      else
         neuron.calcOutputGradients(targetVals.At(n));
      //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
      //--- sampleWeight is derived) - scales the just-computed output gradient in place, before the
      //--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted
      //--- signal without needing its own separate weighting logic.
      if(sampleWeight != 1.0)
         neuron.setGradient(neuron.getGradient() * sampleWeight);
     }
   error /= total;
   error = sqrt(error);
   recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
//---
   CNeuronBase *neuron = NULL;
   CObject *temp = NULL;
   for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--)
     {
      CLayer *hiddenLayer = layers.At(layerNum);
      CLayer *nextLayer = layers.At(layerNum + 1);
      total = hiddenLayer.Total();
      for(int n = 0; n < total && !IsStopped(); ++n)
        {
         neuron = hiddenLayer.At(n);
         if(nextLayer.At(0).Type() == defNeuron)
           {
            temp = nextLayer;
            neuron.calcHiddenGradients(temp);
            continue;
           }
         if(neuron.Type() == defNeuron)
           {
            double g = 0;
            for(int i = 0; i < nextLayer.Total(); i++)
              {
               temp = nextLayer.At(i);
               neuron.calcHiddenGradients(temp);
               g += neuron.getGradient();
              }
            neuron.setGradient(g);
            continue;
           }
         temp = nextLayer.At(n);
         neuron.calcHiddenGradients(temp);
        }
     }
//---
   for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--)
     {
      CLayer *layer = layers.At(layerNum);
      CLayer *prevLayer = layers.At(layerNum - 1);
      total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0);
      int n_conv = 0;
      for(int n = 0; n < total && !IsStopped(); n++)
        {
         neuron = layer.At(n);
         if(CheckPointer(neuron) == POINTER_INVALID)
            return;
         if(neuron.Type() == defNeuronPool)
            continue;
         switch(prevLayer.At(0).Type())
           {
            case defNeuron:
               temp = prevLayer;
               neuron.updateInputWeights(temp);
               break;
            case defNeuronConv:
            case defNeuronPool:
            case defNeuronLSTM:
               if(neuron.Type() == defNeuron)
                 {
                  for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++)
                    {
                     temp = prevLayer.At(n_conv);
                     neuron.updateInputWeights(temp);
                    }
                 }
               else
                 {
                  temp = prevLayer.At(n);
                  neuron.updateInputWeights(temp);
                 }
               break;
            default:
               temp = NULL;
               break;
           }
        }
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight)
  {
   if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID ||
      (CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID))
      return;
   CLayer *currentLayer = layers.At(layers.Total() - 1);
   if(CheckPointer(currentLayer) == POINTER_INVALID)
      return;
//---
   double error = 0.0;
   int total = targetVals.Total();
   double result[];
   CNeuronBaseOCL *neuron = currentLayer.At(0);
   if(neuron.getOutputVal(result) < total)
      return;
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      double target = targetVals.At(n);
      // Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever
      // target==0, so a one-hot classification target only ever counted the true class's own
      // error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric,
      // which is what CNet::backProp()'s CPU-fallback path computes for every output
      // unconditionally, and is what actually drives the dError<0.1 convergence gate in
      // ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp
      // / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was.
      double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n];
      error += MathPow(delta, 2);
     }
   error /= total;
   error = sqrt(error);
   recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
   if(!neuron.calcOutputGradients(targetVals))
      return;
   //--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true
   //--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5
   //--- from the raw outputs already read back into result[] above - see the matching CNet::backProp()
   //--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared
   //--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels
   //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted
   //--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely
   //--- on the MQL5 side, so none of the 3 compute backends need to change.
   if(total == 3)
     {
      double maxLogit = CLASS_LOGIT_SCALE * MathMax(result[0], MathMax(result[1], result[2]));
      double smax[3];
      double sm = 0.0;
      for(int n = 0; n < 3; n++)
        {
         smax[n] = exp(CLASS_LOGIT_SCALE * result[n] - maxLogit);
         sm += smax[n];
        }
      double gradOverwrite[3];
      for(int n = 0; n < 3; n++)
        {
         smax[n] /= sm;
         double target = targetVals.At(n);
         double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
         gradOverwrite[n] = clampedTarget - smax[n];
        }
      neuron.setGradient(gradOverwrite);
     }
   //--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
   //--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta
   //--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the
   //--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers
   //--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends.
   if(sampleWeight != 1.0)
     {
      double gradVals[];
      int gradCount = neuron.getGradient(gradVals);
      if(gradCount > 0)
        {
         for(int g = 0; g < gradCount; g++)
            gradVals[g] *= sampleWeight;
         neuron.setGradient(gradVals);
        }
     }
//--- Calc Hidden Gradients
   CObject *temp = NULL;
   total = layers.Total();
   for(int layerNum = total - 2; layerNum > 0; layerNum--)
     {
      CLayer *nextLayer = currentLayer;
      currentLayer = layers.At(layerNum);
      neuron = currentLayer.At(0);
      neuron.calcHiddenGradients(nextLayer.At(0));
     }
//---
   CLayer *prevLayer = layers.At(total - 1);
   for(int layerNum = total - 1; layerNum > 0; layerNum--)
     {
      currentLayer = prevLayer;
      prevLayer = layers.At(layerNum - 1);
      neuron = currentLayer.At(0);
      neuron.updateInputWeights(prevLayer.At(0));
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CNet::getResults(CArrayDouble *&resultVals)
  {
   if(CheckPointer(resultVals) == POINTER_INVALID)
     {
      resultVals = new CArrayDouble();
      if(CheckPointer(resultVals) == POINTER_INVALID)
         return;
     }
//---
   resultVals.Clear();
   if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0)
      return;
//---
   CLayer *output = layers.At(layers.Total() - 1);
   if(CheckPointer(output) == POINTER_INVALID)
      return;
//---
   if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
     {
      switch(output.At(0).Type())
        {
         case defNeuronBaseOCL:
         case defNeuronConvOCL:
         case defNeuronPoolOCL:
         case defNeuronLSTMOCL:
           {
            CNeuronBaseOCL *temp = output.At(0);
            temp.getOutputVal(resultVals);
            return;
           }
        }
     }
   CNeuronBase *neuron = NULL;
   CLayer *temp = NULL;
   int total = output.Total();
   if(output.At(0).Type() == defNeuron)
      total--;
//---
   for(int i = 0; i < total; i++)
     {
      neuron = output.At(i);
      if(CheckPointer(neuron) == POINTER_INVALID)
         continue;
      if(neuron.Type() == defNeuron)
        {
         resultVals.Add(neuron.getOutputVal());
         continue;
        }
      CNeuronPool *n = neuron;
      temp = n.getOutputLayer();
      for(int ii = 0; ii < temp.Total(); ii++)
        {
         neuron = temp.At(ii);
         if(CheckPointer(neuron) == POINTER_INVALID)
            continue;
         resultVals.Add(neuron.getOutputVal());
        }
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[])
  {
   //--- only the shared FILE_COMMON production weights are protected from being overwritten by
   //--- backtest/optimization noise - a LOCAL (common=false) file is exactly what the tester's
   //--- per-agent cross-pass weight cache uses (see CExpertSignalAIBase::InitNeuralNetwork), and
   //--- must be allowed to write even inside the tester/optimizer.
   if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
      return true;
   if(file_name == NULL)
      return false;
//---
   int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_WRITE);
   if(handle == INVALID_HANDLE)
      return false;
//---
   if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0 || FileWriteLong(handle, era) <= 0 ||
      FileWriteInteger(handle, trainingComplete ? 1 : 0) <= 0)
     {
      FileClose(handle);
      return false;
     }
//--- AutoTuneIndicators "winning" AD indicator params, same append-only pattern as the era/
//--- trainingComplete fields above: count-prefixed so older readers can still stop before this block
   int paramsCount = ArraySize(indicatorParams);
   if(FileWriteInteger(handle, paramsCount) <= 0)
     {
      FileClose(handle);
      return false;
     }
   for(int p = 0; p < paramsCount; p++)
      if(FileWriteDouble(handle, indicatorParams[p]) <= 0)
        {
         FileClose(handle);
         return false;
        }
   bool result = layers.Save(handle);
   FileFlush(handle);
   FileClose(handle);
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[])
  {
   //--- see the matching comment in Save() - only shared FILE_COMMON production weights are blocked
   if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
      return false;
//---
   if(file_name == NULL)
      return false;
//---
   Print(file_name);
   int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ);
   if(handle == INVALID_HANDLE)
      return false;
//---
   error = FileReadDouble(handle);
   undefine = FileReadDouble(handle);
   forecast = FileReadDouble(handle);
   time = (datetime)FileReadLong(handle);
   era = FileReadLong(handle);
   // Older save files predate this field - FileReadInteger returns 0 past EOF,
   // which correctly defaults to "not complete" so a resumed run keeps training
   // instead of silently trusting an unfinished/unknown model as done.
   trainingComplete = (FileReadInteger(handle) != 0);
   // Older save files predate this block too - FileReadInteger returns 0 past EOF, which
   // correctly yields an empty indicatorParams (nothing to restore) instead of misreading weights.
   ArrayFree(indicatorParams);
   int paramsCount = FileReadInteger(handle);
   if(paramsCount > 0)
     {
      ArrayResize(indicatorParams, paramsCount);
      for(int p = 0; p < paramsCount; p++)
         indicatorParams[p] = FileReadDouble(handle);
     }
//---
   if(CheckPointer(layers) != POINTER_INVALID)
      layers.Clear();
   else
      layers = new CArrayLayer();
   int i = 0, num;
//---
   if(!InitOpenCL())
      InitDirectML();
//--- check
//--- read and check start marker - 0xFFFFFFFFFFFFFFFF
   long temp = FileReadLong(handle);
   if(temp == -1)
     {
      //--- read and check array type
      if(FileReadInteger(handle, INT_VALUE) != layers.Type())
        {
         FileClose(handle);
         return(false);
        }
     }
   else
     {
      FileClose(handle);
      return(false);
     }
//--- read array length
   num = FileReadInteger(handle, INT_VALUE);
//--- read array
   if(num != 0)
     {
      for(i = 0; i < num; i++)
        {
         //--- create new element
         CLayer *Layer = new CLayer(0, handle, opencl, directml);
         if(!Layer.Load(handle))
            break;
         if(!layers.Add(Layer))
            break;
        }
     }
   FileClose(handle);
//--- result
   return (layers.Total() == num);
  }
//+------------------------------------------------------------------+
//| Ephemeral weight-only checkpoint, deliberately NOT gated by       |
//| MQL_TESTER/MQL_OPTIMIZATION - see class declaration comment.      |
//+------------------------------------------------------------------+
bool CNet::SaveCheckpoint(string file_name)
  {
   if(file_name == NULL || CheckPointer(layers) == POINTER_INVALID)
      return false;
   int handle = FileOpen(file_name, FILE_BIN | FILE_WRITE);
   if(handle == INVALID_HANDLE)
      return false;
   bool result = layers.Save(handle);
   FileFlush(handle);
   FileClose(handle);
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNet::LoadCheckpoint(string file_name)
  {
   if(file_name == NULL)
      return false;
   int handle = FileOpen(file_name, FILE_BIN | FILE_READ);
   if(handle == INVALID_HANDLE)
      return false;
   if(CheckPointer(layers) != POINTER_INVALID)
      layers.Clear();
   else
      layers = new CArrayLayer();
   int i = 0, num;
   long temp = FileReadLong(handle);
   if(temp == -1)
     {
      if(FileReadInteger(handle, INT_VALUE) != layers.Type())
        {
         FileClose(handle);
         return(false);
        }
     }
   else
     {
      FileClose(handle);
      return(false);
     }
   num = FileReadInteger(handle, INT_VALUE);
   if(num != 0)
     {
      for(i = 0; i < num; i++)
        {
         CLayer *Layer = new CLayer(0, handle, opencl, directml);
         if(!Layer.Load(handle))
            break;
         if(!layers.Add(Layer))
            break;
        }
     }
   FileClose(handle);
   return (layers.Total() == num);
  }
//+------------------------------------------------------------------+
//| See this method's declaration comment for the EMA shadow-weight  |
//| deployment rationale.                                            |
//+------------------------------------------------------------------+
bool CNet::BlendWeightsFrom(CNet &live, double tau)
  {
   if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID)
      return false;
   int layerTotal = MathMin(layers.Total(), live.layers.Total());
   for(int l = 0; l < layerTotal; l++)
     {
      CLayer *shadowLayer = layers.At(l);
      CLayer *liveLayer = live.layers.At(l);
      if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID)
         continue;
      int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total());
      for(int n = 0; n < neuronTotal; n++)
        {
         CNeuronBaseOCL *shadowNeuron = shadowLayer.At(n);
         CNeuronBaseOCL *liveNeuron = liveLayer.At(n);
         if(CheckPointer(shadowNeuron) == POINTER_INVALID || CheckPointer(liveNeuron) == POINTER_INVALID)
            continue;
         if(shadowNeuron.Type() != liveNeuron.Type())
            continue;
         double shadowW[], liveW[];
         switch(shadowNeuron.Type())
           {
            case defNeuronBaseOCL:
               if(shadowNeuron.getWeights(shadowW) > 0 && liveNeuron.getWeights(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowNeuron.setWeights(shadowW);
                 }
               break;
            case defNeuronConvOCL:
              {
               CNeuronConvOCL *shadowConv = shadowNeuron;
               CNeuronConvOCL *liveConv = liveNeuron;
               if(shadowConv.getWeightsConv(shadowW) > 0 && liveConv.getWeightsConv(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowConv.setWeightsConv(shadowW);
                 }
              }
              break;
            case defNeuronLSTMOCL:
              {
               CNeuronLSTMOCL *shadowLstm = shadowNeuron;
               CNeuronLSTMOCL *liveLstm = liveNeuron;
               if(shadowLstm.getWeightsLSTM(shadowW) > 0 && liveLstm.getWeightsLSTM(liveW) > 0)
                 {
                  int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
                  for(int wi = 0; wi < wt; wi++)
                     shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
                  shadowLstm.setWeightsLSTM(shadowW);
                 }
              }
              break;
           }
        }
     }
   return true;
  }
//+------------------------------------------------------------------+
//| See this method's declaration comment for the cold-start bias    |
//| rationale. The weight block that produces the output layer's     |
//| values is stored on the layer BEFORE it (see CNeuronBaseOCL::    |
//| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE     |
//| neuron, laid out as (sourceNeurons+1) values per destination      |
//| neuron, the last of which is that neuron's bias term), so this    |
//| reaches one layer back from the output layer to edit it.          |
//+------------------------------------------------------------------+
bool CNet::SeedOutputLayerBias(const double &biasValues[])
  {
   int outputs = ArraySize(biasValues);
   if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
      return false;
   CLayer *sourceLayer = layers.At(layers.Total() - 2);
   if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0)
      return false;
   CNeuronBaseOCL *sourceNeuron = sourceLayer.At(0);
   if(CheckPointer(sourceNeuron) == POINTER_INVALID || sourceNeuron.Type() != defNeuronBaseOCL)
      return false;
   int inputs = sourceNeuron.Neurons();
   double weights[];
   int count = sourceNeuron.getWeights(weights);
   if(count != (inputs + 1) * outputs)
      return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess
   for(int i = 0; i < outputs; i++)
      weights[(inputs + 1) * i + inputs] = biasValues[i];
   return sourceNeuron.setWeights(weights);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Save(const int file_handle)
  {
   if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle))
      return false;
   if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronPool::Load(const int file_handle)
  {
   if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle))
      return false;
   iWindow = FileReadInteger(file_handle, INT_VALUE);
   iStep = FileReadInteger(file_handle, INT_VALUE);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::Save(const int file_handle)
  {
   if(!CNeuronPool::Save(file_handle))
      return false;
   if(FileWriteDouble(file_handle, param) < 8)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronConv::Load(const int file_handle)
  {
   if(!CNeuronPool::Load(file_handle))
      return false;
   param = FileReadDouble(file_handle);
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronLSTM    :  public CNeuronPool
  {
protected:
   CLayer            *ForgetGate;
   CLayer            *InputGate;
   CLayer            *OutputGate;
   CLayer            *NewContent;
   CArrayDouble      *Memory;
   CArrayDouble      *PrevMemory;
   CArrayDouble      *Input;
   CArrayDouble      *InputGradient;
   //---
   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);
   virtual bool      updateInputWeights(CLayer *&prevLayer);
   virtual bool      updateInputWeights(CLayer *gate, CArrayDouble *input_data);
   virtual bool      InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type);
   virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence);

public:
                     CNeuronLSTM(void);
                    ~CNeuronLSTM(void);
   virtual bool      Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
   //---
   virtual CLayer    *getOutputLayer(void)  { return OutputLayer;  }
   virtual bool      calcInputGradients(CLayer *prevLayer) ;
   virtual bool      calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
   //--- methods for working with files
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronLSTM;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTM::CNeuronLSTM(void)
  {
   ForgetGate     =  new CLayer();
   InputGate      =  new CLayer();
   OutputGate     =  new CLayer();
   NewContent     =  new CLayer();
   Memory         =  new CArrayDouble();
   PrevMemory     =  new CArrayDouble();
   Input          =  new CArrayDouble();
   InputGradient  =  new CArrayDouble();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTM::~CNeuronLSTM(void)
  {
   if(CheckPointer(ForgetGate) != POINTER_INVALID)
      delete ForgetGate;
   if(CheckPointer(InputGate) != POINTER_INVALID)
      delete InputGate;
   if(CheckPointer(OutputGate) != POINTER_INVALID)
      delete OutputGate;
   if(CheckPointer(NewContent) != POINTER_INVALID)
      delete NewContent;
   if(CheckPointer(Memory) != POINTER_INVALID)
      delete Memory;
   if(CheckPointer(PrevMemory) != POINTER_INVALID)
      delete PrevMemory;
   if(CheckPointer(Input) != POINTER_INVALID)
      delete Input;
   if(CheckPointer(InputGradient) != POINTER_INVALID)
      delete InputGradient;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
  {
   if(units_count <= 0)
      return false;
//--- Init Layers
   if(!CNeuronPool::Init(numOutputs, myIndex, window, step, units_count, optimization_type))
      return false;
   if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(InputGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type))
      return false;
   if(!InitLayer(NewContent, units_count, window + units_count, optimization_type))
      return false;
   if(!Memory.Reserve(units_count))
      return false;
   if(!PrevMemory.Reserve(units_count))
      return false;
   CNeuron *temp;
   for(int i = 0; i < units_count; i++)
     {
      if(!Memory.Add(0))
         return false;
      if(!PrevMemory.Add(0))
         return false;
      temp = OutputLayer.At(i);
      temp.setOutputVal(0);
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(layer) == POINTER_INVALID)
     {
      layer = new CLayer(numOutputs);
      if(CheckPointer(layer) == POINTER_INVALID)
         return false;
     }
   else
      layer.Clear();
   if(!layer.Reserve(numUnits))
      return false;
//---
   CNeuron *temp;
   for(int i = 0; i < numUnits; i++)
     {
      temp = new CNeuron();
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      if(!temp.Init(numOutputs + 1, i, optimization_type))
         return false;
      if(!layer.Add(temp))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::feedForward(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0)
      return false;
   CNeuronBase *temp;
   CConnection *temp_con;
   if(CheckPointer(Input) == POINTER_INVALID)
     {
      Input = new CArrayDouble();
      if(CheckPointer(Input) == POINTER_INVALID)
         return false;
     }
   else
      Input.Clear();
//--- Concatenate input sequence
   int total = prevLayer.Total();
   if(!Input.Reserve(total + OutputLayer.Total()))
      return false;
   for(int i = 0; i < total; i++)
     {
      temp = prevLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
         return false;
     }
   total = OutputLayer.Total();
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
         return false;
     }
   int total_data = Input.Total();
//--- Calculated forget gate
   CArrayDouble *forget_gate = CalculateGate(ForgetGate, Input);
   if(CheckPointer(forget_gate) == POINTER_INVALID)
      return false;
//--- Calculated input gate
   CArrayDouble *input_gate = CalculateGate(InputGate, Input);
   if(CheckPointer(input_gate) == POINTER_INVALID)
      return false;
//--- Calculated output gate
   CArrayDouble *output_gate = CalculateGate(OutputGate, Input);
   if(CheckPointer(output_gate) == POINTER_INVALID)
      return false;
//--- Calculated new content
   CArrayDouble *new_content = new CArrayDouble();
   if(CheckPointer(new_content) == POINTER_INVALID)
      return false;
   total = NewContent.Total();
   for(int i = 0; i < total; i++)
     {
      temp = NewContent.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      double val = 0;
      for(int c = 0; c < total_data; c++)
        {
         temp_con = temp.Connections.At(c);
         if(CheckPointer(temp_con) == POINTER_INVALID)
            return false;
         val += temp_con.weight * Input.At(c);
        }
      val = TanhFunction(val);
      temp.setOutputVal(val);
      if(!new_content.Add(val))
         return false;
     }
//--- Calculated output sequences
   for(int i = 0; i < total; i++)
     {
      if(PrevMemory.Total() <= i)
         PrevMemory.Add(Memory.At(i));
      else
         PrevMemory.Update(i, Memory.At(i));
      double value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i);
      if(!Memory.Update(i, value))
         return false;
      temp = OutputLayer.At(i);
      value = TanhFunction(value) * output_gate.At(i);
      temp.setOutputVal(value);
     }
//---
   delete forget_gate;
   delete input_gate;
   delete new_content;
   delete output_gate;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CArrayDouble *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayDouble *sequence)
  {
   CNeuronBase *temp;
   CConnection *temp_con;
   CArrayDouble *result = new CArrayDouble();
   if(CheckPointer(gate) == POINTER_INVALID)
      return NULL;
   int total = gate.Total();
   int total_data = sequence.Total();
   for(int i = 0; i < total; i++)
     {
      temp = gate.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
        {
         delete result;
         return NULL;
        }
      double val = 0;
      for(int c = 0; c < total_data; c++)
        {
         temp_con = temp.Connections.At(c);
         if(CheckPointer(temp_con) == POINTER_INVALID)
           {
            delete result;
            return NULL;
           }
         val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c));
        }
      val = SigmoidFunction(val);
      temp.setOutputVal(val);
      if(!result.Add(val))
        {
         delete result;
         return NULL;
        }
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer)
  {
   if(CheckPointer(InputGradient) == POINTER_INVALID)
     {
      InputGradient = new CArrayDouble();
      if(CheckPointer(InputGradient) == POINTER_INVALID)
         return false;
     }
   else
      InputGradient.Clear();
//---
   int total = OutputLayer.Total();
   CNeuron *temp;
   CArrayDouble *MemoryGradient = new CArrayDouble();
   CNeuron *gate;
   CConnection *con;
//---
   if(nextLayer != OutputLayer)
      for(int i = 0; i < total; i++)
        {
         temp = OutputLayer.At(i);
         if(CheckPointer(temp) == POINTER_INVALID)
            return false;
         temp.setGradient(temp.sumDOW(nextLayer));
        }
//--- Calculated memory  and output gate gradients
   if(CheckPointer(MemoryGradient) == POINTER_INVALID)
      return false;
   if(!MemoryGradient.Reserve(total))
      return false;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      gate = OutputGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      double value = temp.getGradient() * gate.getOutputVal();
      value = TanhFunctionDerivative(Memory.At(i)) * value;
      if(i >= MemoryGradient.Total())
        {
         if(!MemoryGradient.Add(value))
            return false;
        }
      else
        {
         value = MemoryGradient.At(i) + value;
         if(!MemoryGradient.Update(i, value))
            return false;
        }
      gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0);
      //--- Calcculated gates and new content gradients
      gate = ForgetGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
      gate = InputGate.At(i);
      temp = NewContent.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
      temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0);
     }
//--- Calculated input gradients
   int total_inp = temp.getConnections().Total();
   for(int n = 0; n < total_inp; n++)
     {
      double value = 0;
      for(int i = 0; i < total; i++)
        {
         temp = ForgetGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = InputGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = OutputGate.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
         //---
         temp = NewContent.At(i);
         con = temp.getConnections().At(n);
         value += temp.getGradient() * con.weight;
        }
      if(InputGradient.Total() >= n)
        {
         if(!InputGradient.Add(value))
            return false;
        }
      else
         if(!InputGradient.Update(n, value))
            return false;
     }
//--- Calculated gradients for prev. state
   int shift = total_inp - total;
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      if(CheckPointer(temp) == POINTER_INVALID)
         return false;
      temp.setGradient(InputGradient.At(shift + i));
     }
//--- Calculated memory  and output gate gradients
   for(int i = 0; i < total; i++)
     {
      temp = OutputLayer.At(i);
      gate = OutputGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      double value = temp.getGradient() * gate.getPrevVal();
      value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value;
      if(!MemoryGradient.Update(i, value))
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0));
      //--- Calcculated gates and new content gradients
      gate = ForgetGate.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
      gate = InputGate.At(i);
      temp = NewContent.At(i);
      if(CheckPointer(gate) == POINTER_INVALID)
         return false;
      gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
      temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0));
     }
//---
   delete MemoryGradient;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *&prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID)
      return false;
//---
   if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input)
      || !updateInputWeights(NewContent, Input))
     {
      return false;
     }
   if(optimization == ADAM)
      t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayDouble *input_data)
  {
   if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID)
      return false;
   CNeuronBase *neuron;
   CConnection *con;
   int total_n = gate.Total();
   int total_data = input_data.Total();
   double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
   for(int n = 0; n < total_n; n++)
     {
      neuron = gate.At(n);
      if(CheckPointer(neuron) == POINTER_INVALID)
         return false;
      double g = neuron.getGradient();
      double g2 = g * g;
      for(int i = 0; i < total_data; i++)
        {
         con = neuron.getConnections().At(i);
         if(CheckPointer(con) == POINTER_INVALID)
            return false;
         double data = input_data.At(i);
         if(optimization == SGD)
            con.weight += con.deltaWeight = (g != 0 && data != 0 ? eta * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight;
         else
           {
            con.mt = b1 * con.mt + (1 - b1) * g;
            con.vt = b2 * con.vt + (1 - b2) * g2 + 0.00000001;
            con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
            // Sign-agreement gate removed - see CNeuron::updateInputWeights' comment for why.
            con.weight += con.deltaWeight;
           }
         // See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
         // AI\Network.cl's LSTM_UpdateWeightsAdam MAX_WEIGHT clamp.
         con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index)
  {
   if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index)
      return false;
//---
   prevNeuron.setGradient(InputGradient.At(index));
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return false;
//---
   int total = prevLayer.Total();
   if(total <= 0)
      return false;
   CNeuronBase *neuron;
   bool result = true;
   for(int i = 0; (i < total && result); i++)
     {
      neuron = prevLayer.At(i);
      if(CheckPointer(neuron) == POINTER_INVALID)
        {
         result = false;
         break;
        }
      result = calcInputGradients(neuron, i);
     }
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Save(const int file_handle)
  {
   if(!CNeuronPool::Save(file_handle))
      return false;
   if(!ForgetGate.Save(file_handle))
      return false;
   if(!InputGate.Save(file_handle))
      return false;
   if(!OutputGate.Save(file_handle))
      return false;
   if(!NewContent.Save(file_handle))
      return false;
   if(!Memory.Save(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Load(const int file_handle)
  {
   if(!CNeuronPool::Load(file_handle))
      return false;
   if(!ForgetGate.Load(file_handle))
      return false;
   if(!InputGate.Load(file_handle))
      return false;
   if(!OutputGate.Load(file_handle))
      return false;
   if(!NewContent.Load(file_handle))
      return false;
   if(!Memory.Load(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunction(double x)
  {
   switch(activation)
     {
      case NONE:
         return(x);
         break;
      case TANH:
         return TanhFunction(x);
         break;
      case SIGMOID:
         return SigmoidFunction(x);
         break;
      case PRELU:
         // fixed 0.01 slope, matches CNeuronConv::activationFunction's `param` default -
         // this case was previously missing here, so any generic Dense (defNeuron) layer
         // given PRELU silently fell through to the `return x` below (pure linear identity,
         // no nonlinearity at all) instead of leaky-ReLU.
         return(x >= 0 ? x : 0.01 * x);
         break;
     }
//---
   return x;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunctionDerivative(double x)
  {
   switch(activation)
     {
      case NONE:
         return(1);
         break;
      case TANH:
         return TanhFunctionDerivative(x);
         break;
      case SIGMOID:
         return SigmoidFunctionDerivative(x);
         break;
      case PRELU:
         // See activationFunction() above - was previously missing here too, defaulting
         // to a derivative of 1 (which happens to be right for x>=0, but wrong for x<0,
         // where it must be the 0.01 leak slope, not 1).
         return(x >= 0 ? 1.0 : 0.01);
         break;
     }
//---
   return 1;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
template<typename T>
int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags)
  {
   int result = -1;
   for(int i = 0; i < m_buffers_total; i++)
     {
      if(m_buffers[i] != INVALID_HANDLE)
         continue;
      result = i;
      break;
     }
//---
   if(result < 0)
     {
      if(ArrayResize(m_buffers, m_buffers_total + 1) > 0)
        {
         m_buffers_total = ArraySize(m_buffers);
         result = m_buffers_total - 1;
         m_buffers[result] = INVALID_HANDLE;
        }
      else
         return result;
     }
//---
   if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags))
      return -1;
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL, CDirectMLMy *directml = NULL) :  hWeights(-1),
   hDeltaWeights(-1),
   hOutput(-1),
   hGradient(-1)
  {
   iOutputs = outputs;
   iFileHandle = handle;
   OpenCL = opencl;
   DirectML = directml;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CLayer::Load(const int file_handle)
  {
   iFileHandle = file_handle;
   if(!CArrayObj::Load(file_handle))
      return false;
   if(CheckPointer(m_data[0]) == POINTER_INVALID)
      return false;
//---
   switch(m_data[0].Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronPoolOCL:
      case defNeuronLSTMOCL:
        {
         CNeuronBaseOCL *temp = m_data[0];
         iOutputs = temp.getConnections();
         break;
        }
      default:
        {
         CNeuronBase *temp = m_data[0];
         iOutputs = temp.getConnections().Total();
         break;
        }
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNet::~CNet(void)
  {
   if(CheckPointer(layers) != POINTER_INVALID)
      delete layers;
   if(CheckPointer(opencl) != POINTER_INVALID)
     {
      opencl.Shutdown();
      delete opencl;
     }
   if(CheckPointer(directml) != POINTER_INVALID)
      delete directml;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "BufferDouble.mqh"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
class CNeuronBaseOCL    :  public CObject
  {
protected:
   COpenCLMy         *OpenCL;
   CDirectMLMy       *DirectML;
   CBufferDouble     *Output;
   CBufferDouble     *PrevOutput;
   CBufferDouble     *Weights;
   CBufferDouble     *DeltaWeights;
   CBufferDouble     *Gradient;
   CBufferDouble     *FirstMomentum;
   CBufferDouble     *SecondMomentum;
   //---
   const double      alpha;
   int               t;
   //---
   int               m_myIndex;
   ENUM_ACTIVATION   activation;
   ENUM_OPTIMIZATION optimization;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);

public:
                     CNeuronBaseOCL(void);
                    ~CNeuronBaseOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual void      SetActivationFunction(ENUM_ACTIVATION value) {  activation = value; }
   //---
   virtual int       getOutputIndex(void)          {  return Output.GetIndex();        }
   virtual int       getPrevOutIndex(void)         {  return PrevOutput.GetIndex();    }
   virtual int       getGradientIndex(void)        {  return Gradient.GetIndex();      }
   virtual int       getWeightsIndex(void)         {  return Weights.GetIndex();       }
   virtual int       getDeltaWeightsIndex(void)    {  return DeltaWeights.GetIndex();  }
   virtual int       getFirstMomentumIndex(void)   {  return FirstMomentum.GetIndex(); }
   virtual int       getSecondMomentumIndex(void)  {  return SecondMomentum.GetIndex();}
   //---
   virtual int       getOutputVal(double &values[])   {  return Output.GetData(values);      }
   virtual int       getOutputVal(CArrayDouble *values)   {  return Output.GetData(values);  }
   virtual int       getPrevVal(double &values[])     {  return PrevOutput.GetData(values);  }
   virtual int       getGradient(double &values[])    {  return Gradient.GetData(values);    }
   //--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by
   //--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call
   //--- (which only computes the raw, unweighted delta) and before the backward pass reads this same
   //--- buffer to propagate into the hidden layers.
   virtual bool      setGradient(const double &values[])
     {
      int count = ArraySize(values);
      for(int i = 0; i < count; i++)
         if(!Gradient.Update(i, values[i]))
            return false;
      return Gradient.BufferWrite();
     }
   // Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created,
   // so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here.
   virtual int       getWeights(double &values[])     {  return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values));     }
   // Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment
   // (see that method's declaration comment) - writes a full replacement weight array back to this
   // buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element,
   // used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment
   // then pushes it to the device - the shape callers use when blending two already-read-out arrays.
   virtual bool      setWeights(double &values[])
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
         return false;
      if(!Weights.AssignArray(values))
         return false;
      return Weights.BufferWrite();
     }
   virtual int       Neurons(void)                    {  return Output.Total();              }
   virtual ENUM_ACTIVATION Activation(void)           {  return activation;                  }
   virtual int       getConnections(void)             {  return (CheckPointer(Weights) != POINTER_INVALID ? Weights.Total() / (Gradient.Total()) : 0);   }
   //---
   virtual bool      feedForward(CObject *SourceObject);
   virtual bool      calcHiddenGradients(CObject *TargetObject);
   virtual bool      calcOutputGradients(CArrayDouble *Target);
   virtual bool      updateInputWeights(CObject *SourceObject);
   //---
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   //---
   virtual int       Type(void)        const                      {  return defNeuronBaseOCL;                  }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBaseOCL::CNeuronBaseOCL(void)   :  alpha(momentum),
   activation(TANH),
   optimization(SGD),
   t(1)
  {
   OpenCL = NULL;
   DirectML = NULL;
   Output = new CBufferDouble();
   PrevOutput = new CBufferDouble();
   Weights = new CBufferDouble();
   DeltaWeights = new CBufferDouble();
   Gradient = new CBufferDouble();
   FirstMomentum = new CBufferDouble();
   SecondMomentum = new CBufferDouble();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronBaseOCL::~CNeuronBaseOCL(void)
  {
   if(CheckPointer(Output) != POINTER_INVALID)
      delete Output;
   if(CheckPointer(PrevOutput) != POINTER_INVALID)
      delete PrevOutput;
   if(CheckPointer(Weights) != POINTER_INVALID)
      delete Weights;
   if(CheckPointer(DeltaWeights) != POINTER_INVALID)
      delete DeltaWeights;
   if(CheckPointer(Gradient) != POINTER_INVALID)
      delete Gradient;
   if(CheckPointer(FirstMomentum) != POINTER_INVALID)
      delete FirstMomentum;
   if(CheckPointer(SecondMomentum) != POINTER_INVALID)
      delete SecondMomentum;
   OpenCL = NULL;
   DirectML = NULL;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
      return false;
   OpenCL = open_cl;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(OpenCL))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(OpenCL))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
      // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
      // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
      // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
      // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
      // for the fact that they zero out roughly half their input distribution, whereas the
      // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
      // was ~2x too conservative here. Applied to every layer through this one shared Init()
      // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
      // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
      // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
      // extremes on every backend, so the imprecision there is not worth the much larger, riskier
      // change of threading activation awareness through every neuron subtype's Init() overload.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(OpenCL))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(OpenCL))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(OpenCL))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(OpenCL))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same  |
//| buffer layout, buffers just get created on the DML backend.      |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0)
      return false;
   DirectML = direct_ml;
   optimization = optimization_type;
//---
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(!Output.BufferInit(numNeurons, 1.0))
      return false;
   if(!Output.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(!PrevOutput.BufferInit(numNeurons, 1.0))
      return false;
   if(!PrevOutput.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(!Gradient.BufferInit(numNeurons + 1, 0.0))
      return false;
   if(!Gradient.BufferCreate(DirectML))
      return false;
//---
   if(numOutputs > 0)
     {
      if(CheckPointer(Weights) == POINTER_INVALID)
        {
         Weights = new CBufferDouble();
         if(CheckPointer(Weights) == POINTER_INVALID)
            return false;
        }
      int count = (int)((numNeurons + 1) * numOutputs);
      if(!Weights.Reserve(count))
         return false;
      // He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
      double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
      for(int i = 0; i < count; i++)
        {
         double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
         if(weigh == 0)
            weigh = 0.001;
         if(!Weights.Add(weigh))
            return false;
        }
      if(!Weights.BufferCreate(DirectML))
         return false;
      //---
      if(optimization == SGD)
        {
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
           {
            DeltaWeights = new CBufferDouble();
            if(CheckPointer(DeltaWeights) == POINTER_INVALID)
               return false;
           }
         if(!DeltaWeights.BufferInit(count, 0))
            return false;
         if(!DeltaWeights.BufferCreate(DirectML))
            return false;
         if(CheckPointer(FirstMomentum) != POINTER_INVALID)
           {
            delete FirstMomentum;
            FirstMomentum = NULL;
           }
         if(CheckPointer(SecondMomentum) != POINTER_INVALID)
           {
            delete SecondMomentum;
            SecondMomentum = NULL;
           }
        }
      else
        {
         if(CheckPointer(DeltaWeights) != POINTER_INVALID)
           {
            delete DeltaWeights;
            DeltaWeights = NULL;
           }
         //---
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
           {
            FirstMomentum = new CBufferDouble();
            if(CheckPointer(FirstMomentum) == POINTER_INVALID)
               return false;
           }
         if(!FirstMomentum.BufferInit(count, 0))
            return false;
         if(!FirstMomentum.BufferCreate(DirectML))
            return false;
         //---
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
           {
            SecondMomentum = new CBufferDouble();
            if(CheckPointer(SecondMomentum) == POINTER_INVALID)
               return false;
           }
         if(!SecondMomentum.BufferInit(count, 0))
            return false;
         if(!SecondMomentum.BufferCreate(DirectML))
            return false;
        }
     }
   else
     {
      if(CheckPointer(Weights) != POINTER_INVALID)
         delete Weights;
      if(CheckPointer(DeltaWeights) != POINTER_INVALID)
         delete DeltaWeights;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
#include "NeuronOCLConvPool.mqh"
//+------------------------------------------------------------------+
//| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from      |
//| scratch from the standard LSTM equations - NOT ported from the    |
//| NeuroNet_DNG reference (see the note above the LSTM kernels in    |
//| Network.cl for why). Single-timestep-truncated BPTT: gradient     |
//| does not flow back into h_prev/c_prev from an earlier step.       |
//| Adam-only - Init fails for any other optimization type.           |
//+------------------------------------------------------------------+
class CNeuronLSTMOCL : public CNeuronBaseOCL
  {
protected:
   int               m_iInputs;
   CBufferDouble     *WeightsLSTM;
   CBufferDouble     *FirstMomentumLSTM;
   CBufferDouble     *SecondMomentumLSTM;
   CBufferDouble     *DeltaWeightsLSTM;
   CBufferDouble     *WeightsGradient;
   CBufferDouble     *Concatenated;
   CBufferDouble     *ConcatenatedGradient;
   CBufferDouble     *Memory;
   CBufferDouble     *HiddenCache;
   //---
   virtual bool      feedForward(CNeuronBaseOCL *NeuronOCL);
   virtual bool      updateInputWeights(CNeuronBaseOCL *NeuronOCL);
   virtual bool      SetInputs(int count);

public:
                     CNeuronLSTMOCL(void) : m_iInputs(-1)
     {
      WeightsLSTM = NULL;
      FirstMomentumLSTM = NULL;
      SecondMomentumLSTM = NULL;
      DeltaWeightsLSTM = NULL;
      WeightsGradient = NULL;
      Concatenated = NULL;
      ConcatenatedGradient = NULL;
      Memory = NULL;
      HiddenCache = NULL;
     }
                    ~CNeuronLSTMOCL(void);
   virtual bool      Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
   virtual bool      calcInputGradients(CNeuronBaseOCL *NeuronOCL);
   virtual bool      Save(int const file_handle);
   virtual bool      Load(int const file_handle);
   virtual int       Type(void)   const   {  return defNeuronLSTMOCL;  }
   // See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the
   // base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
   virtual int       getWeightsLSTM(double &values[])     {  return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values));  }
   virtual bool      setWeightsLSTM(double &values[])
     {
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
      if(!WeightsLSTM.AssignArray(values))
         return false;
      return WeightsLSTM.BufferWrite();
     }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CNeuronLSTMOCL::~CNeuronLSTMOCL(void)
  {
   if(CheckPointer(WeightsLSTM) != POINTER_INVALID)
      delete WeightsLSTM;
   if(CheckPointer(FirstMomentumLSTM) != POINTER_INVALID)
      delete FirstMomentumLSTM;
   if(CheckPointer(SecondMomentumLSTM) != POINTER_INVALID)
      delete SecondMomentumLSTM;
   if(CheckPointer(DeltaWeightsLSTM) != POINTER_INVALID)
      delete DeltaWeightsLSTM;
   if(CheckPointer(WeightsGradient) != POINTER_INVALID)
      delete WeightsGradient;
   if(CheckPointer(Concatenated) != POINTER_INVALID)
      delete Concatenated;
   if(CheckPointer(ConcatenatedGradient) != POINTER_INVALID)
      delete ConcatenatedGradient;
   if(CheckPointer(Memory) != POINTER_INVALID)
      delete Memory;
   if(CheckPointer(HiddenCache) != POINTER_INVALID)
      delete HiddenCache;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type))
      return false;
   uint H = numNeurons;
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(OpenCL))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(OpenCL))
      return false;
   m_iInputs = -1;
   return true;
  }
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above.        |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
  {
   if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type))
      return false;
   uint H = numNeurons;
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(DirectML))
      return false;
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(DirectML))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(DirectML))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(DirectML))
      return false;
   m_iInputs = -1;
   return true;
  }
//+------------------------------------------------------------------+
//| Lazily sized on the first feedForward call, once the previous     |
//| layer's neuron count (the input size) is known.                   |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::SetInputs(int count)
  {
   m_iInputs = count;
   uint H = (uint)Neurons();
   int total = (int)(4 * H * (H + count + 1));
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
     {
      WeightsLSTM = new CBufferDouble();
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(!WeightsLSTM.Reserve(total))
      return false;
   // Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full
   // rationale; fan-in here is hidden units + input width (each gate reads both).
   double weighScale = 1.0 / MathSqrt((double)(H + count) + 1.0);
   for(int i = 0; i < total; i++)
     {
      double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
      if(weigh == 0)
         weigh = 0.001;
      if(!WeightsLSTM.Add(weigh))
         return false;
     }
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
     {
      FirstMomentumLSTM = new CBufferDouble();
      if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(!FirstMomentumLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
     {
      SecondMomentumLSTM = new CBufferDouble();
      if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(!SecondMomentumLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   // Momentum accumulator - only meaningfully used when optimization==SGD
   // (see updateInputWeights()), but allocated unconditionally regardless of
   // the configured optimizer, matching CNeuronBaseOCL::Init's pattern for
   // the dense-layer DeltaWeights/FirstMomentum/SecondMomentum trio above.
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
     {
      DeltaWeightsLSTM = new CBufferDouble();
      if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(!DeltaWeightsLSTM.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(WeightsGradient) == POINTER_INVALID)
     {
      WeightsGradient = new CBufferDouble();
      if(CheckPointer(WeightsGradient) == POINTER_INVALID)
         return false;
     }
   if(!WeightsGradient.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(m_iInputs <= 0)
     {
      if(!SetInputs(NeuronOCL.Neurons()))
         return false;
     }
   else
      if(m_iInputs != NeuronOCL.Neurons())
         return false;
   int H = Neurons();
   int I = m_iInputs;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.LSTMGates(WeightsLSTM.GetIndex(), getOutputIndex(), NeuronOCL.getOutputIndex(), Concatenated.GetIndex(), H, I) ||
         !DirectML.LSTMState(Concatenated.GetIndex(), Memory.GetIndex(), getOutputIndex(), HiddenCache.GetIndex(), getOutputIndex(), H))
        {
         printf("Error of execution DirectML LSTM feedForward");
         return false;
        }
      return Output.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset2[2] = {0, 0};
   uint size2[2] = {(uint)H, 4};
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_matrix_w, WeightsLSTM.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_hidden_prev, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_inputs, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_Gates, 2, offset2, size2))
     {
      printf("Error of execution kernel LSTM_Gates: %d", GetLastError());
      return false;
     }
   uint offset1[1] = {0};
   uint size1[1] = {(uint)H};
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_memory, Memory.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_prev, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_cache, HiddenCache.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_output, getOutputIndex());
   OpenCL.SetArgument(def_k_LSTM_State, def_k_lstms_hidden_size, H);
   if(!OpenCL.Execute(def_k_LSTM_State, 1, offset1, size1))
     {
      printf("Error of execution kernel LSTM_State: %d", GetLastError());
      return false;
     }
   //--- Output (== hidden_prev for the next timestep) stays GPU-resident; see the note in
   //--- CNeuronBaseOCL::feedForward().
   return true;
  }
//+------------------------------------------------------------------+
//| Writes the gradient into NeuronOCL (the earlier/input-side layer) |
//| - same inverted-call convention as CNeuronConvOCL::calcInputGradients.|
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID || m_iInputs <= 0)
      return false;
   int H = Neurons();
   int I = m_iInputs;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.LSTMGateGradient(getGradientIndex(), Memory.GetIndex(), Concatenated.GetIndex(), ConcatenatedGradient.GetIndex(), H) ||
         !DirectML.LSTMWeightsGradient(ConcatenatedGradient.GetIndex(), HiddenCache.GetIndex(), NeuronOCL.getOutputIndex(), WeightsGradient.GetIndex(), H, I) ||
         !DirectML.LSTMInputsGradient(ConcatenatedGradient.GetIndex(), WeightsLSTM.GetIndex(), NeuronOCL.getGradientIndex(), H, I))
        {
         printf("Error of execution DirectML LSTM calcInputGradients");
         return false;
        }
      double temp[];
      return NeuronOCL.getGradient(temp) > 0;
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset1[1] = {0};
   uint size1[1] = {(uint)H};
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_gradient, getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_memory, Memory.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated, Concatenated.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_GateGradient, def_k_lstmgg_hidden_size, H);
   if(!OpenCL.Execute(def_k_LSTM_GateGradient, 1, offset1, size1))
     {
      printf("Error of execution kernel LSTM_GateGradient: %d", GetLastError());
      return false;
     }
   uint sizeW[1] = {(uint)WeightsLSTM.Total()};
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_cache, HiddenCache.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_inputs, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_weights_gradient, WeightsGradient.GetIndex());
   OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_WeightsGradient, 1, offset1, sizeW))
     {
      printf("Error of execution kernel LSTM_WeightsGradient: %d", GetLastError());
      return false;
     }
   uint sizeI[1] = {(uint)I};
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_concatenated_gradient, ConcatenatedGradient.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_matrix_w, WeightsLSTM.GetIndex());
   OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_inputs_gradient, NeuronOCL.getGradientIndex());
   OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_hidden_size, H);
   OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_input_size, I);
   if(!OpenCL.Execute(def_k_LSTM_InputsGradient, 1, offset1, sizeI))
     {
      printf("Error of execution kernel LSTM_InputsGradient: %d", GetLastError());
      return false;
     }
   //--- NeuronOCL's Gradient stays GPU-resident; see the note in
   //--- CNeuronConvOCL::calcInputGradients().
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
      return false;
   int total = WeightsLSTM.Total();
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(optimization == SGD)
        {
         if(!DirectML.LSTMUpdateWeightsMomentum(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), DeltaWeightsLSTM.GetIndex(), eta, alpha, total))
           {
            printf("Error of execution DirectML LSTM_UpdateWeightsMomentum");
            return false;
           }
        }
      else
        {
         double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
         if(!DirectML.LSTMUpdateWeightsAdam(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), FirstMomentumLSTM.GetIndex(), SecondMomentumLSTM.GetIndex(), lt, b1, b2, total))
           {
            printf("Error of execution DirectML LSTM_UpdateWeightsAdam");
            return false;
           }
         t++;
        }
      return WeightsLSTM.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint offset1[1] = {0};
   uint size1[1] = {(uint)total};
   if(optimization == SGD)
     {
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_w, WeightsLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_weights_gradient, WeightsGradient.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_dw, DeltaWeightsLSTM.GetIndex());
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_learning_rates, (float)eta);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_momentum, (float)alpha);
      ResetLastError();
      if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsMomentum, 1, offset1, size1))
        {
         printf("Error of execution kernel LSTM_UpdateWeightsMomentum: %d", GetLastError());
         return false;
        }
     }
   else
     {
      double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_w, WeightsLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_weights_gradient, WeightsGradient.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_m, FirstMomentumLSTM.GetIndex());
      OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_v, SecondMomentumLSTM.GetIndex());
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_l, (float)lt);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b1, (float)b1);
      OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b2, (float)b2);
      ResetLastError();
      if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsAdam, 1, offset1, size1))
        {
         printf("Error of execution kernel LSTM_UpdateWeightsAdam: %d", GetLastError());
         return false;
        }
      t++;
     }
   //--- WeightsLSTM stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights().
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Save(const int file_handle)
  {
   if(!CNeuronBaseOCL::Save(file_handle))
      return false;
   if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < INT_VALUE)
      return false;
   if(m_iInputs <= 0)
      return true;
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID || !WeightsLSTM.BufferRead() || !WeightsLSTM.Save(file_handle))
      return false;
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID || !FirstMomentumLSTM.BufferRead() || !FirstMomentumLSTM.Save(file_handle))
      return false;
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID || !SecondMomentumLSTM.BufferRead() || !SecondMomentumLSTM.Save(file_handle))
      return false;
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID || !DeltaWeightsLSTM.BufferRead() || !DeltaWeightsLSTM.Save(file_handle))
      return false;
   if(CheckPointer(Memory) == POINTER_INVALID || !Memory.BufferRead() || !Memory.Save(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Load(const int file_handle)
  {
   if(!CNeuronBaseOCL::Load(file_handle))
      return false;
   m_iInputs = FileReadInteger(file_handle, INT_VALUE);
   uint H = (uint)Neurons();
   //--- scratch buffers (not persisted) were sized for the placeholder Init() unit
   //--- count; resize them now that the real neuron count is known.
   if(CheckPointer(Concatenated) == POINTER_INVALID)
     {
      Concatenated = new CBufferDouble();
      if(CheckPointer(Concatenated) == POINTER_INVALID)
         return false;
     }
   Concatenated.BufferFree();
   if(!Concatenated.BufferInit(4 * H, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !Concatenated.BufferCreate(OpenCL) : !Concatenated.BufferCreate(DirectML))
      return false;
   if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
     {
      ConcatenatedGradient = new CBufferDouble();
      if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
         return false;
     }
   ConcatenatedGradient.BufferFree();
   if(!ConcatenatedGradient.BufferInit(4 * H, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !ConcatenatedGradient.BufferCreate(OpenCL) : !ConcatenatedGradient.BufferCreate(DirectML))
      return false;
   if(CheckPointer(HiddenCache) == POINTER_INVALID)
     {
      HiddenCache = new CBufferDouble();
      if(CheckPointer(HiddenCache) == POINTER_INVALID)
         return false;
     }
   HiddenCache.BufferFree();
   if(!HiddenCache.BufferInit(H, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !HiddenCache.BufferCreate(OpenCL) : !HiddenCache.BufferCreate(DirectML))
      return false;
//---
   if(m_iInputs <= 0)
      return true;
   int total = (int)(4 * H * (H + m_iInputs + 1));
//---
   if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
     {
      WeightsLSTM = new CBufferDouble();
      if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(WeightsLSTM.GetIndex() >= 0)
      WeightsLSTM.BufferFree();
   if(!WeightsLSTM.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
     {
      FirstMomentumLSTM = new CBufferDouble();
      if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(FirstMomentumLSTM.GetIndex() >= 0)
      FirstMomentumLSTM.BufferFree();
   if(!FirstMomentumLSTM.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
     {
      SecondMomentumLSTM = new CBufferDouble();
      if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
         return false;
     }
   if(SecondMomentumLSTM.GetIndex() >= 0)
      SecondMomentumLSTM.BufferFree();
   if(!SecondMomentumLSTM.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
     {
      DeltaWeightsLSTM = new CBufferDouble();
      if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
         return false;
     }
   if(DeltaWeightsLSTM.GetIndex() >= 0)
      DeltaWeightsLSTM.BufferFree();
   if(!DeltaWeightsLSTM.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(Memory) == POINTER_INVALID)
     {
      Memory = new CBufferDouble();
      if(CheckPointer(Memory) == POINTER_INVALID)
         return false;
     }
   if(Memory.GetIndex() >= 0)
      Memory.BufferFree();
   if(!Memory.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !Memory.BufferCreate(OpenCL) : !Memory.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(WeightsGradient) == POINTER_INVALID)
     {
      WeightsGradient = new CBufferDouble();
      if(CheckPointer(WeightsGradient) == POINTER_INVALID)
         return false;
     }
   if(!WeightsGradient.BufferInit(total, 0))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
         temp = SourceObject;
         return feedForward(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
                                NeuronOCL.Neurons(), NativeActivationCode(activation)))
        {
         printf("Error of execution DirectML FeedForward");
         return false;
        }
      return Output.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Output.Total();
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel FeedForward: %d", GetLastError());
      return false;
     }
   //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
   //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
   //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
   //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
   //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
                                       NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1))
        {
         printf("Error of execution DirectML CalcHiddenGradient");
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = Neurons() + 1;
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
   OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
   if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
      return false;
     }
   //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
   //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
  {
   if(CheckPointer(Target) == POINTER_INVALID)
      return false;
   int count = Target.Total();
   for(int i = 0; i < count; i++)
      if(!Gradient.Update(i, Target.At(i)))
         return false;
   Gradient.BufferWrite();
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
        {
         printf("Error of execution DirectML CalcOutputGradient");
         return false;
        }
      return Gradient.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[1] = {0};
   uint global_work_size[1];
   global_work_size[0] = count;
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
   OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
   OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
   ResetLastError();
   if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
     {
      printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
      return false;
     }
   //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
   //--- BufferRead()s itself - see the note in feedForward() above.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
  {
   if(CheckPointer(NeuronOCL) == POINTER_INVALID)
      return false;
   if(CheckPointer(DirectML) != POINTER_INVALID)
     {
      int inputs = NeuronOCL.Neurons();
      int neurons = Neurons();
      if(optimization == SGD)
        {
         if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                             NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons))
           {
            printf("Error of execution DirectML UpdateWeightsMomentum");
            return false;
           }
        }
      else
        {
         double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
         if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
                                         NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
                                         inputs, lt, b1, b2, neurons))
           {
            printf("Error of execution DirectML UpdateWeightsAdam");
            return false;
           }
         t++;
        }
      return NeuronOCL.Weights.BufferRead();
     }
   if(CheckPointer(OpenCL) == POINTER_INVALID)
      return false;
   uint global_work_offset[2] = {0, 0};
   uint global_work_size[2];
   global_work_size[0] = Neurons();
   global_work_size[1] = NeuronOCL.Neurons();
   if(optimization == SGD)
     {
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
      OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta);
      OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
         return false;
        }
     }
   else
     {
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
         return false;
      if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
         return false;
      double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1))
         return false;
      if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2))
         return false;
      uint rest = global_work_size[1] % 4;
      global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
      ResetLastError();
      if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
        {
         printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
         return false;
        }
      t++;
     }
   //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
   //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
  {
   if(CheckPointer(TargetObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   CNeuronConvOCL *tempConv = NULL;
   CNeuronLSTMOCL *tempLstm = NULL;
   CNeuronPoolOCL *tempPool = NULL;
   switch(TargetObject.Type())
     {
      case defNeuronBaseOCL:
         temp = TargetObject;
         return calcHiddenGradients(temp);
         break;
      case defNeuronConvOCL:
         //--- Conv owns the backward step (calcInputGradients), called on itself with
         //--- "this" (the earlier layer) passed in so it writes into this->Gradient.
         tempConv = TargetObject;
         return tempConv.calcInputGradients(GetPointer(this));
         break;
      case defNeuronLSTMOCL:
         tempLstm = TargetObject;
         return tempLstm.calcInputGradients(GetPointer(this));
         break;
      case defNeuronPoolOCL:
         tempPool = TargetObject;
         return tempPool.calcInputGradients(GetPointer(this));
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
  {
   if(CheckPointer(SourceObject) == POINTER_INVALID)
      return false;
//---
   CNeuronBaseOCL *temp = NULL;
   switch(SourceObject.Type())
     {
      case defNeuronBaseOCL:
      case defNeuronConvOCL:
      case defNeuronLSTMOCL:
      case defNeuronPoolOCL:
         temp = SourceObject;
         return updateInputWeights(temp);
         break;
     }
//---
   return false;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Save(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
   if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
      return false;
//---
   if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
      return false;
   if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
      return false;
//---
   if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
      return false;
   if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
      return false;
   if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
      return false;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      FileWriteInteger(file_handle, 0);
      return true;
     }
   else
      FileWriteInteger(file_handle, 1);
//---
   if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
      return false;
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
         return false;
      if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Load(const int file_handle)
  {
   if(file_handle == INVALID_HANDLE)
      return false;
//---
   activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
   optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
   t = FileReadInteger(file_handle, INT_VALUE);
   if(CheckPointer(Output) == POINTER_INVALID)
     {
      Output = new CBufferDouble();
      if(CheckPointer(Output) == POINTER_INVALID)
         return false;
     }
   if(Output.GetIndex() >= 0)
      Output.BufferFree();
   if(!Output.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !Output.BufferCreate(OpenCL) : !Output.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(PrevOutput) == POINTER_INVALID)
     {
      PrevOutput = new CBufferDouble();
      if(CheckPointer(PrevOutput) == POINTER_INVALID)
         return false;
     }
   if(PrevOutput.GetIndex() >= 0)
      PrevOutput.BufferFree();
   if(!PrevOutput.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !PrevOutput.BufferCreate(OpenCL) : !PrevOutput.BufferCreate(DirectML))
      return false;
//---
   if(CheckPointer(Gradient) == POINTER_INVALID)
     {
      Gradient = new CBufferDouble();
      if(CheckPointer(Gradient) == POINTER_INVALID)
         return false;
     }
   if(Gradient.GetIndex() >= 0)
      Gradient.BufferFree();
   if(!Gradient.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !Gradient.BufferCreate(OpenCL) : !Gradient.BufferCreate(DirectML))
      return false;
//---
   if(FileReadInteger(file_handle) == 0)
      return true;
//---
   if(CheckPointer(Weights) == POINTER_INVALID)
     {
      Weights = new CBufferDouble();
      if(CheckPointer(Weights) == POINTER_INVALID)
         return false;
     }
   if(Weights.GetIndex() >= 0)
      Weights.BufferFree();
   if(!Weights.Load(file_handle))
      return false;
   if(CheckPointer(OpenCL) != POINTER_INVALID ? !Weights.BufferCreate(OpenCL) : !Weights.BufferCreate(DirectML))
      return false;
//---
   if(optimization == SGD)
     {
      if(CheckPointer(DeltaWeights) == POINTER_INVALID)
        {
         DeltaWeights = new CBufferDouble();
         if(CheckPointer(DeltaWeights) == POINTER_INVALID)
            return false;
        }
      if(DeltaWeights.GetIndex() >= 0)
         DeltaWeights.BufferFree();
      if(!DeltaWeights.Load(file_handle))
         return false;
      if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeights.BufferCreate(OpenCL) : !DeltaWeights.BufferCreate(DirectML))
         return false;
     }
   else
     {
      if(CheckPointer(FirstMomentum) == POINTER_INVALID)
        {
         FirstMomentum = new CBufferDouble();
         if(CheckPointer(FirstMomentum) == POINTER_INVALID)
            return false;
        }
      if(FirstMomentum.GetIndex() >= 0)
         FirstMomentum.BufferFree();
      if(!FirstMomentum.Load(file_handle))
         return false;
      if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentum.BufferCreate(OpenCL) : !FirstMomentum.BufferCreate(DirectML))
         return false;
      //---
      if(CheckPointer(SecondMomentum) == POINTER_INVALID)
        {
         SecondMomentum = new CBufferDouble();
         if(CheckPointer(SecondMomentum) == POINTER_INVALID)
            return false;
        }
      if(SecondMomentum.GetIndex() >= 0)
         SecondMomentum.BufferFree();
      if(!SecondMomentum.Load(file_handle))
         return false;
      if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentum.BufferCreate(OpenCL) : !SecondMomentum.BufferCreate(DirectML))
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+