koros0111/Warrior_EA
1
0
Fork 0
forked from animatedread/Warrior_EA
Code Pull requests Activity
Warrior_EA/AI/Network.mqh
AnimateDread 3fbe16127e docs: simplify comments and enum descriptions across AI and Enums 
Remove overly verbose explanations in Network.mqh comments and InputEnums.mqh enum values. Shorten descriptions to improve readability without losing essential information.
2026-07-18 23:59:40 -04:00

4489 lines
179 KiB
MQL5
Raw Permalink Blame History

//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include <Arrays\ArrayDouble.mqh>
#include <Arrays\ArrayInt.mqh>
#include <Arrays\ArrayObj.mqh>
#include <OpenCL\OpenCL.mqh>
//--- kept local to this file (rather than Enumerations\InputEnums.mqh) since AI\Network.mqh is
//--- included well before Variables\Inputs.mqh in Warrior_EA.mq5's include chain - this input has
//--- to be self-contained here regardless.
enum CPU_LOAD_PRESET
{
CPU_LOAD_10 = 10, // 10% of detected cores
CPU_LOAD_20 = 20, // 20% of detected cores
CPU_LOAD_30 = 30, // 30% of detected cores
CPU_LOAD_40 = 40, // 40% of detected cores
CPU_LOAD_50 = 50, // 50% of detected cores
CPU_LOAD_60 = 60, // 60% of detected cores
CPU_LOAD_70 = 70, // 70% of detected cores
CPU_LOAD_80 = 80, // 80% of detected cores
CPU_LOAD_90 = 90, // 90% of detected cores
CPU_LOAD_MAX = 100 // Max - all detected cores
};
//--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available,
//--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed). Caps how
//--- many of the auto-detected CPU cores WarriorCPU.dll's worker thread pool actually uses - has
//--- no effect at all when a GPU tier (OpenCL or DirectML) is active, since neither one calls into
//--- WarriorCPU.dll.
//--- Applied directly, undivided, to every CNet's own WarriorCPU.dll worker pool (live Net, shadow
//--- Net, and - under AIType=HYBRID - each of PAI/CONV/LSTM's own pair) even though several such
//--- pools can be alive at once. An earlier version divided this by how many CNet instances were
//--- alive (g_netPeerCount/EffectiveCpuLoadPercent()), reasoning that N pools each sized at 100%
//--- would N-way oversubscribe the CPU - but MQL5 gives one chart's EA a single execution thread,
//--- and every WarriorCPU.dll entry point (CPU_FeedForward/CPU_CalcOutputGradient/etc.) blocks that
//--- thread until its own ParallelFor() completes, so only ONE pool is EVER actively computing at a
//--- time - the rest sit idle (blocked on a condvar, ~0% CPU) regardless of how many exist. Dividing
//--- was pure waste: it capped whichever pool happened to be running at a fraction of the cores the
//--- user asked for, without preventing any contention that was never actually possible.
input CPU_LOAD_PRESET TargetCPULoad = CPU_LOAD_MAX; // CPU worker thread cap when no GPU accel (% cores)
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
//--- kept local to this file for the same include-order reason as CPU_LOAD_PRESET/TargetCPULoad
//--- above - AI\Network.mqh is parsed well before Variables\Inputs.mqh.
//--- Adam (Kingma & Ba, 2014) hyperparameters. Defaults are neuronetworksbook.pdf's own reference
//--- library defaults (defLearningRate/defBeta1/defBeta2 = 3.0e-4/0.9/0.999) - not the paper's
//--- abstract "0.001" mention, which the book's own worked examples don't actually use either.
//--- 3.0e-4 also happens to sit inside the noisy/non-stationary-trading-data range (0.0003-0.0005)
//--- this project had separately tuned lr to before this input existed, so no behavior conflict.
//--- Beta1 was previously hand-lowered to 0.8 as an experiment to fight a multi-era same-class-streak
//--- bug - that symptom's likely root cause (independent-sigmoid+BCE output gradient, since fixed to
//--- a joint softmax+CCE gradient in backProp()/backPropOCL()) is addressed elsewhere now, so this
//--- reverts to the literature/book default.
input double AdamLearningRate = 0.0003; // Adam learning rate (eta ceiling) - book default 3.0e-4
input double AdamBeta1 = 0.9; // Adam beta1 (1st-moment decay) - book default 0.9
input double AdamBeta2 = 0.999; // Adam beta2 (2nd-moment decay) - book default 0.999
//--- SGD+momentum hyperparameters. The book states no distinct default learning rate for this method
//--- (its own reference library reuses the same defLearningRate for every optimizer), so this reuses
//--- Adam's book-default rate as its starting point too. The book also states no numeric default for
//--- the momentum decay coefficient itself (just "in the range 0 to 1, exclusive") - 0.9 reuses
//--- Adam's beta1, the only concrete "momentum decay" value the book ever commits to a number for.
input double SgdLearningRate = 0.0003; // SGD+momentum learning rate - book default 3.0e-4
input double SgdMomentum = 0.9; // SGD+momentum decay - reuses Adam's beta1 default
#define lr AdamLearningRate
#define b1 AdamBeta1
#define b2 AdamBeta2
#define momentum SgdMomentum
double eta = lr;
#define defConnect 0x7781
#define defArrayConnects 0x7782
#define defNeuronBase 0x7783
#define defNeuron 0x7784
#define defNeuronConv 0x7785
#define defNeuronPool 0x7786
#define defLayer 0x7787
#define defArrayLayer 0x7788
#define defNet 0x7789
#define defNeuronLSTM 0x7791
//---
#define defBufferDouble 0x7882
#define defNeuronBaseOCL 0x7883
#define defNeuronLSTMOCL 0x7884
#define defNeuronConvOCL 0x7885
#define defNeuronPoolOCL 0x7886
//---
#define def_k_FeedForward 0
#define def_k_ff_matrix_w 0
#define def_k_ff_matrix_i 1
#define def_k_ff_matrix_o 2
#define def_k_ff_inputs 3
#define def_k_ff_activation 4
//---
#define def_k_CaclOutputGradient 1
#define def_k_cog_matrix_t 0
#define def_k_cog_matrix_o 1
#define def_k_cog_matrix_ig 2
#define def_k_cog_activation 3
//---
#define def_k_CaclHiddenGradient 2
#define def_k_chg_matrix_w 0
#define def_k_chg_matrix_g 1
#define def_k_chg_matrix_o 2
#define def_k_chg_matrix_ig 3
#define def_k_chg_outputs 4
#define def_k_chg_activation 5
//---
#define def_k_UpdateWeightsMomentum 3
#define def_k_uwm_matrix_w 0
#define def_k_uwm_matrix_g 1
#define def_k_uwm_matrix_i 2
#define def_k_uwm_matrix_dw 3
#define def_k_uwm_inputs 4
#define def_k_uwm_learning_rates 5
#define def_k_uwm_momentum 6
//---
#define def_k_UpdateWeightsAdam 4
#define def_k_uwa_matrix_w 0
#define def_k_uwa_matrix_g 1
#define def_k_uwa_matrix_i 2
#define def_k_uwa_matrix_m 3
#define def_k_uwa_matrix_v 4
#define def_k_uwa_inputs 5
#define def_k_uwa_l 6
#define def_k_uwa_b1 7
#define def_k_uwa_b2 8
//---
#define def_k_FeedForwardProof 15
#define def_k_ffp_matrix_i 0
#define def_k_ffp_matrix_o 1
#define def_k_ffp_inputs 2
#define def_k_ffp_window 3
#define def_k_ffp_step 4
//---
#define def_k_CalcInputGradientProof 16
#define def_k_cigp_matrix_i 0
#define def_k_cigp_matrix_g 1
#define def_k_cigp_matrix_o 2
#define def_k_cigp_matrix_ig 3
#define def_k_cigp_outputs 4
#define def_k_cigp_window 5
#define def_k_cigp_step 6
//---
#define def_k_FeedForwardConv 5
#define def_k_ffc_matrix_w 0
#define def_k_ffc_matrix_i 1
#define def_k_ffc_matrix_o 2
#define def_k_ffc_inputs 3
#define def_k_ffc_step 4
#define def_k_ffc_window_in 5
#define def_k_ffc_window_out 6
#define def_k_ffc_activation 7
//---
#define def_k_CalcHiddenGradientConv 6
#define def_k_chgc_matrix_w 0
#define def_k_chgc_matrix_g 1
#define def_k_chgc_matrix_o 2
#define def_k_chgc_matrix_ig 3
#define def_k_chgc_outputs 4
#define def_k_chgc_step 5
#define def_k_chgc_window_in 6
#define def_k_chgc_window_out 7
#define def_k_chgc_activation 8
//---
#define def_k_UpdateWeightsConvMomentum 7
#define def_k_uwcm_matrix_w 0
#define def_k_uwcm_matrix_g 1
#define def_k_uwcm_matrix_i 2
#define def_k_uwcm_matrix_dw 3
#define def_k_uwcm_inputs 4
#define def_k_uwcm_learning_rates 5
#define def_k_uwcm_momentum 6
#define def_k_uwcm_window_in 7
#define def_k_uwcm_window_out 8
#define def_k_uwcm_step 9
//---
#define def_k_UpdateWeightsConvAdam 8
#define def_k_uwca_matrix_w 0
#define def_k_uwca_matrix_g 1
#define def_k_uwca_matrix_i 2
#define def_k_uwca_matrix_m 3
#define def_k_uwca_matrix_v 4
#define def_k_uwca_inputs 5
#define def_k_uwca_l 6
#define def_k_uwca_b1 7
#define def_k_uwca_b2 8
#define def_k_uwca_window_in 9
#define def_k_uwca_window_out 10
#define def_k_uwca_step 11
//---
// LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient
// flows back into h_prev/c_prev from a prior step). Supports both Adam and
// SGD+momentum (LSTM_UpdateWeightsAdam/LSTM_UpdateWeightsMomentum below) -
// see CNeuronLSTMOCL::updateInputWeights for the optimizer dispatch.
// Derived from scratch from the standard LSTM equations - NOT ported from
// the NeuroNet_DNG reference, whose LSTM_HiddenGradient kernel overwrites
// the live weights buffer instead of writing to weights_gradient.
#define def_k_LSTM_Gates 9
#define def_k_lstmg_matrix_w 0
#define def_k_lstmg_hidden_prev 1
#define def_k_lstmg_inputs 2
#define def_k_lstmg_concatenated 3
#define def_k_lstmg_hidden_size 4
#define def_k_lstmg_input_size 5
//---
#define def_k_LSTM_State 10
#define def_k_lstms_concatenated 0
#define def_k_lstms_memory 1
#define def_k_lstms_hidden_prev 2
#define def_k_lstms_hidden_cache 3
#define def_k_lstms_output 4
#define def_k_lstms_hidden_size 5
//---
#define def_k_LSTM_GateGradient 11
#define def_k_lstmgg_gradient 0
#define def_k_lstmgg_memory 1
#define def_k_lstmgg_concatenated 2
#define def_k_lstmgg_concatenated_gradient 3
#define def_k_lstmgg_hidden_size 4
//---
#define def_k_LSTM_WeightsGradient 12
#define def_k_lstmwg_concatenated_gradient 0
#define def_k_lstmwg_hidden_cache 1
#define def_k_lstmwg_inputs 2
#define def_k_lstmwg_weights_gradient 3
#define def_k_lstmwg_hidden_size 4
#define def_k_lstmwg_input_size 5
//---
#define def_k_LSTM_InputsGradient 13
#define def_k_lstmig_concatenated_gradient 0
#define def_k_lstmig_matrix_w 1
#define def_k_lstmig_inputs_gradient 2
#define def_k_lstmig_hidden_size 3
#define def_k_lstmig_input_size 4
//---
#define def_k_LSTM_UpdateWeightsAdam 14
#define def_k_lstmuwa_matrix_w 0
#define def_k_lstmuwa_weights_gradient 1
#define def_k_lstmuwa_matrix_m 2
#define def_k_lstmuwa_matrix_v 3
#define def_k_lstmuwa_l 4
#define def_k_lstmuwa_b1 5
#define def_k_lstmuwa_b2 6
//---
// SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see
// AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body.
#define def_k_LSTM_UpdateWeightsMomentum 17
#define def_k_lstmuwm_matrix_w 0
#define def_k_lstmuwm_weights_gradient 1
#define def_k_lstmuwm_matrix_dw 2
#define def_k_lstmuwm_learning_rates 3
#define def_k_lstmuwm_momentum 4
//---
// b1/b2 are now the AdamBeta1/AdamBeta2 inputs declared above (book defaults 0.9/0.999) - see
// AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era
// same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root
// cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE
// gradient in backProp()/backPropOCL(), which addresses it more directly than shortening b1 ever
// could. b1/b2 are passed as runtime parameters into every backend (not baked into compiled
// kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam
// signatures), so they're safe to expose as ordinary inputs.
// Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had
// already gone unstable (real collapses were happening at weight magnitudes several orders of
// magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh
// engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually
// reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below.
#define MAX_WEIGHT 100.0
// Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in
// DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see
// WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright
// +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of
// training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi-
// hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles. 0.01 is the standard AdamW
// default (Loshchilov & Hutter).
#define WEIGHT_DECAY 0.01
// Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full
// rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden
// Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras),
// most likely from 5x back-to-back oversampling replay building artificially correlated momentum.
// Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the
// post-update weight value and is far too loose (1e6) to prevent this.
#define MAX_WEIGHT_DELTA 0.1
// Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see
// SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near its
// activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes that
// neuron's entire backprop gradient contribution regardless of how wrong its output is. A saturated
// unit can then never receive a corrective signal to unstick it. 1e-4 matches the equivalent floor in
// Dmitriy Gizlyk's reference NeuroNet.mqh/NeuroNet.cl engine.
#define MIN_ACTIVATION_DERIVATIVE 1.0e-4
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#resource "Network.cl" as string cl_program
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
enum ENUM_ACTIVATION
{
NONE,
TANH,
SIGMOID,
PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction
};
//+------------------------------------------------------------------+
//| Translates ENUM_ACTIVATION to the "activation" int code every |
//| native compute backend actually understands (Network.cl's |
//| kernels, and the mirrored Activation()/inline switches in |
//| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, and |
//| deliberately anything else (incl. NONE) falls through every one |
//| of those switches unmatched, which is exactly linear passthrough -|
//| there's no case 3 anywhere on the native side, so PRELU relies on |
//| the FeedForwardConv-family kernels specifically, and NONE never |
//| needs a case at all. This is NOT the same numbering as |
//| ENUM_ACTIVATION itself (NONE=0, TANH=1, SIGMOID=2, PRELU=3) - a |
//| raw (int)activation cast at a kernel call site silently sends the |
//| WRONG activation to the GPU/DLL tier (e.g. MQL5 TANH -> native |
//| SIGMOID). Only use this at actual kernel-dispatch call sites - |
//| CNeuronBase::Save()/CNeuronBaseOCL::Save() persist the raw |
//| ENUM_ACTIVATION value instead, and must keep using (int)activation |
//| directly so saved topology files round-trip through Load() as-is. |
//+------------------------------------------------------------------+
int NativeActivationCode(ENUM_ACTIVATION value)
{
switch(value)
{
case TANH: return 0;
case SIGMOID: return 1;
case PRELU: return 2;
default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches
}
}
//---
enum ENUM_OPTIMIZATION
{
SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras)
ADAM // Adam (adaptive step, faster convergence, can overfit)
};
//---
enum ENUM_BUFFERS
{
WEIGHTS,
DELTA_WEIGHTS,
OUTPUT,
GRADIENT,
FIRST_MOMENTUM,
SECOND_MOMENTUM
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include "NeuronPrimitives.mqh"
class CLayer;
//---
class CNeuronBase : public CObject
{
protected:
double outputVal;
double prevVal;
uint m_myIndex;
double gradient;
CArrayCon *Connections;
ENUM_ACTIVATION activation;
ENUM_OPTIMIZATION optimization;
int t;
//---
virtual bool feedForward(CLayer *prevLayer) { return false; }
virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; }
virtual bool updateInputWeights(CLayer *&prevLayer) { return false; }
virtual double activationFunction(double x);
virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); }
virtual double TanhFunction(double x) { return tanh(x); }
virtual CLayer *getOutputLayer(void) { return NULL; }
public:
CNeuronBase(void);
~CNeuronBase(void);
virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type);
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
//---
static double alpha;
//---
virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; }
virtual double getOutputVal() { return outputVal; }
virtual double getPrevVal() { return prevVal; }
virtual void setGradient(double val) { gradient = val; }
virtual double getGradient() { return gradient; }
virtual CArrayCon *getConnections() { return Connections;}
virtual double activationFunctionDerivative(double x);
virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); }
virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); }
//---
virtual bool feedForward(CObject *&SourceObject);
virtual bool calcHiddenGradients(CObject *&TargetObject);
virtual bool updateInputWeights(CObject *&SourceObject);
//---
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle)
{
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
return(Connections.Load(file_handle));
}
//---
virtual int Type(void) const { return defNeuronBase; }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNeuronBase::alpha = momentum; // momentum
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronBase::CNeuronBase(void) :
outputVal(1),
gradient(0),
activation(TANH),
t(1),
optimization(SGD)
{
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronBase::~CNeuronBase(void)
{
if(CheckPointer(Connections) != POINTER_INVALID)
delete Connections;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type)
{
if(CheckPointer(Connections) == POINTER_INVALID)
{
Connections = new CArrayCon();
if(CheckPointer(Connections) == POINTER_INVALID)
return false;
}
//---
if(Connections.Reserve(fmax(numOutputs, 1)))
for(uint c = 0; c < numOutputs; c++)
{
if(!Connections.CreateElement(c))
return false;
Connections.IncreaseTotal();
}
//---
m_myIndex = myIndex;
optimization = optimization_type;
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include "NeuronCPU.mqh"
class COpenCLMy : public COpenCL
{
public:
COpenCLMy(void) {};
~COpenCLMy(void) {};
template<typename T>
int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
};
#include "NeuronDirectML.mqh"
class CLayer: public CArrayObj
{
private:
uint iOutputs;
int iFileHandle;
int hWeights;
int hDeltaWeights;
int hOutput;
int hGradient;
COpenCLMy *OpenCL;
CDirectMLMy *DirectML;
public:
CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL);
~CLayer(void) {};
//---
virtual bool CreateElement(int const index);
virtual void IncreaseTotal() { m_data_total++; }
virtual int Type(void) const { return defLayer; }
virtual bool Load(const int file_handle);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CLayer::CreateElement(int index)
{
if(index >= m_data_max)
return false;
//---
bool result = false;
CNeuronBase *temp = NULL;
CNeuronPool *temp_p = NULL;
CNeuronBaseOCL *temp_ocl = NULL;
if(iFileHandle <= 0)
{
temp = new CNeuron();
if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD))
return false;
result = true;
}
else
{
int type = FileReadInteger(iFileHandle);
switch(type)
{
case defNeuron:
temp = new CNeuron();
if(CheckPointer(temp) == POINTER_INVALID)
result = false;
result = temp.Init(iOutputs, index, ADAM);
break;
case defNeuronPool:
temp_p = new CNeuronPool();
if(CheckPointer(temp_p) == POINTER_INVALID)
result = false;
if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
{
temp = temp_p;
result = true;
}
break;
case defNeuronConv:
temp_p = new CNeuronConv();
if(CheckPointer(temp_p) == POINTER_INVALID)
result = false;
if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
{
temp = temp_p;
result = true;
}
break;
case defNeuronLSTM:
temp_p = new CNeuronLSTM();
if(CheckPointer(temp_p) == POINTER_INVALID)
result = false;
if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM))
{
temp = temp_p;
result = true;
}
break;
case defNeuronBaseOCL:
if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
return false;
temp_ocl = new CNeuronBaseOCL();
if(CheckPointer(temp_ocl) == POINTER_INVALID)
result = false;
if(CheckPointer(OpenCL) != POINTER_INVALID
? temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM)
: temp_ocl.Init(iOutputs, index, DirectML, 1, ADAM))
{
m_data[index] = temp_ocl;
return true;
}
break;
case defNeuronConvOCL:
{
if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
return false;
//--- placeholder dims; the real window/step/units are restored by Load() right after
CNeuronConvOCL *temp_conv = new CNeuronConvOCL();
if(CheckPointer(temp_conv) == POINTER_INVALID)
result = false;
if(CheckPointer(OpenCL) != POINTER_INVALID
? temp_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM)
: temp_conv.Init(iOutputs, index, DirectML, 1, 1, 1, 1, ADAM))
{
m_data[index] = temp_conv;
return true;
}
break;
}
case defNeuronPoolOCL:
{
if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
return false;
CNeuronPoolOCL *temp_pool = new CNeuronPoolOCL();
if(CheckPointer(temp_pool) == POINTER_INVALID)
result = false;
if(CheckPointer(OpenCL) != POINTER_INVALID
? temp_pool.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM)
: temp_pool.Init(iOutputs, index, DirectML, 1, 1, 1, ADAM))
{
m_data[index] = temp_pool;
return true;
}
break;
}
case defNeuronLSTMOCL:
{
if(CheckPointer(OpenCL) == POINTER_INVALID && CheckPointer(DirectML) == POINTER_INVALID)
return false;
CNeuronLSTMOCL *temp_lstm = new CNeuronLSTMOCL();
if(CheckPointer(temp_lstm) == POINTER_INVALID)
result = false;
if(CheckPointer(OpenCL) != POINTER_INVALID
? temp_lstm.Init(iOutputs, index, OpenCL, 1, ADAM)
: temp_lstm.Init(iOutputs, index, DirectML, 1, ADAM))
{
m_data[index] = temp_lstm;
return true;
}
break;
}
default:
result = false;
break;
}
}
if(result)
m_data[index] = temp;
//---
return (result);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include "ArrayLayer.mqh"
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CNeuronPool : public CNeuronBase
{
protected:
CLayer *OutputLayer;
int iWindow;
int iStep;
virtual bool feedForward(CLayer *prevLayer);
virtual bool calcHiddenGradients(CLayer *&nextLayer);
public:
CNeuronPool(void) {};
~CNeuronPool(void);
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
//---
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
virtual bool calcInputGradients(CLayer *prevLayer) ;
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
//--- methods for working with files
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle);
virtual int Type(void) const { return defNeuronPool; }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CNeuronConv : public CNeuronPool
{
protected:
double param; //PReLU param
virtual bool feedForward(CLayer *prevLayer);
virtual bool calcHiddenGradients(CLayer *&nextLayer);
virtual double activationFunction(double x);
virtual bool updateInputWeights(CLayer *&prevLayer);
public:
CNeuronConv() : param(0.01) { };
~CNeuronConv(void) { };
//---
virtual bool calcInputGradients(CLayer *prevLayer) ;
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
virtual double activationFunctionDerivative(double x);
virtual int Type(void) const { return defNeuronConv; }
//--- methods for working with files
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle);
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBase::feedForward(CObject *&SourceObject)
{
bool result = false;
//---
if(CheckPointer(SourceObject) == POINTER_INVALID)
return result;
//---
CLayer *temp_l;
CNeuronPool *temp_n;
switch(SourceObject.Type())
{
case defLayer:
temp_l = SourceObject;
result = feedForward(temp_l);
break;
case defNeuronConv:
case defNeuronPool:
case defNeuronLSTM:
temp_n = SourceObject;
result = feedForward(temp_n.getOutputLayer());
break;
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBase::updateInputWeights(CObject *&SourceObject)
{
bool result = false;
//---
if(CheckPointer(SourceObject) == POINTER_INVALID)
return result;
//---
CLayer *temp_l;
CNeuronPool *temp_n;
switch(SourceObject.Type())
{
case defLayer:
temp_l = SourceObject;
result = updateInputWeights(temp_l);
break;
case defNeuronConv:
case defNeuronPool:
case defNeuronLSTM:
temp_n = SourceObject;
temp_l = temp_n.getOutputLayer();
result = updateInputWeights(temp_l);
break;
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::feedForward(CLayer *prevLayer)
{
bool result = false;
//---
if(CheckPointer(prevLayer) == POINTER_INVALID)
return result;
//---
int total = prevLayer.Total() - iWindow + 1;
CNeuron *temp;
CConnection *con;
result = true;
for(int i = 0; (i < total && result); i += iStep)
{
double sum = 0;
for(int j = 0; (j < iWindow && result); j++)
{
temp = prevLayer.At(i + j);
con = Connections.At(j);
if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
return false;
double val = temp.getOutputVal();
sum += val * con.weight;
}
temp = OutputLayer.At(i / iStep);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setOutputVal(activationFunction(sum));
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunction(double x)
{
if(x >= 0)
return x;
return param * x;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject)
{
bool result = false;
//---
if(CheckPointer(TargetObject) == POINTER_INVALID)
return result;
//---
CLayer *temp_l;
CNeuronPool *temp_n;
switch(TargetObject.Type())
{
case defLayer:
temp_l = TargetObject;
result = calcHiddenGradients(temp_l);
break;
case defNeuronConv:
case defNeuronPool:
case defNeuronLSTM:
switch(Type())
{
case defNeuron:
temp_n = TargetObject;
result = temp_n.calcInputGradients(GetPointer(this), m_myIndex);
break;
case defNeuronLSTM:
temp_n = TargetObject;
temp_l = getOutputLayer();
if(!temp_n.calcInputGradients(temp_l))
{
result = false;
break;
}
result = calcHiddenGradients(temp_l);
break;
default:
temp_l =getOutputLayer();
temp_n = TargetObject;
result = temp_n.calcInputGradients(temp_l);
break;
}
break;
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer)
{
if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
return false;
//---
gradient = 0;
int total = OutputLayer.Total();
CNeuron *temp;
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal()));
}
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNeuronConv::activationFunctionDerivative(double x)
{
if(x >= 0)
return 1;
return param;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::updateInputWeights(CLayer *&prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
return false;
//---
CConnection *con;
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
for(int n = 0; n < iWindow && !IsStopped(); n++)
{
con = Connections.At(n);
if(CheckPointer(con) == POINTER_INVALID)
continue;
double delta = 0;
int total_i = OutputLayer.Total();
CNeuron *prev, *out;
for(int i = 0; i < total_i; i++)
{
prev = prevLayer.At(n * iStep + i);
out = OutputLayer.At(total_i - i - 1);
if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID)
continue;
delta += prev.getOutputVal() * out.getGradient();
}
if(optimization == SGD)
con.weight += con.deltaWeight = (delta != 0 ? eta*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0);
else
{
con.mt = b1 * con.mt + (1 - b1) * delta;
con.vt = b2 * con.vt + (1 - b2) * delta * delta + 0.00000001;
con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
// Sign-agreement gate - see CNeuron::updateInputWeights' matching comment.
if(con.deltaWeight * delta > 0)
con.weight += con.deltaWeight;
}
// See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
// AI\Network.cl's UpdateWeightsConvMomentum/UpdateWeightsConvAdam MAX_WEIGHT clamp.
con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
}
if(optimization == ADAM)
t++;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
{
iWindow = window;
iStep = step;
if(!CNeuronBase::Init(window, myIndex, optimization_type))
return false;
OutputLayer = new CLayer(numOutputs);
if(CheckPointer(OutputLayer) == POINTER_INVALID)
return false;
if(OutputLayer.Reserve(units_count))
for(int i = 0; i < units_count; i++)
{
if(!OutputLayer.CreateElement(i))
return false;
OutputLayer.IncreaseTotal();
}
//---
if(Type() == defNeuronPool)
{
if(CheckPointer(Connections) != POINTER_INVALID)
Connections.Clear();
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronPool::~CNeuronPool(void)
{
delete OutputLayer;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::feedForward(CLayer *prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID)
return false;
//---
int total = prevLayer.Total() - iWindow + 1;
CNeuron *temp;
for(int i = 0; i <= total; i += iStep)
{
double sum = 0;
for(int j = 0; j < iWindow; j++)
{
temp = prevLayer.At(i + j);
if(CheckPointer(temp) == POINTER_INVALID)
continue;
sum += temp.getOutputVal();
}
temp = OutputLayer.At(i / iStep);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setOutputVal(sum / iWindow);
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::calcHiddenGradients(CLayer *&nextLayer)
{
if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0)
return false;
//---
gradient = 0;
int total = OutputLayer.Total();
CNeuron *temp;
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setGradient(temp.sumDOW(nextLayer));
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CLayer *prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || CheckPointer(prevLayer.At(0)) == POINTER_INVALID)
return false;
//---
if(prevLayer.At(0).Type() != defNeuron)
{
CNeuronPool *temp = prevLayer.At(m_myIndex);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
prevLayer = temp.getOutputLayer();
if(CheckPointer(prevLayer) == POINTER_INVALID)
return false;
}
//---
CNeuronBase *prevNeuron, *outputNeuron;
int total = prevLayer.Total();
for(int i = 0; i < total; i++)
{
prevNeuron = prevLayer.At(i);
if(CheckPointer(prevNeuron) == POINTER_INVALID)
continue;
double prev_gradient = 0;
int start = i - iWindow + iStep;
start = (start - start % iStep) / iStep;
double stop = (i - i % iStep) / iStep + 1;
for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
{
outputNeuron = OutputLayer.At(out);
if(CheckPointer(outputNeuron) == POINTER_INVALID)
continue;
prev_gradient += outputNeuron.getGradient() / iWindow;
}
prevNeuron.setGradient(prev_gradient);
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::calcInputGradients(CNeuronBase *prevNeuron, uint index)
{
if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
return false;
//---
if(prevNeuron.Type() != defNeuron)
{
CNeuronPool *temp = prevNeuron;
return calcInputGradients(temp.getOutputLayer());
}
//---
CNeuronBase *outputNeuron;
double prev_gradient = 0;
int start = (int)index - iWindow + iStep;
start = (start - start % iStep) / iStep;
double stop = (index - index % iStep) / iStep + 1;
for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
{
outputNeuron = OutputLayer.At(out);
if(CheckPointer(outputNeuron) == POINTER_INVALID)
continue;
prev_gradient += outputNeuron.getGradient() / iWindow;
}
prevNeuron.setGradient(prev_gradient);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CLayer *prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
return false;
//---
if(prevLayer.At(0).Type() != defNeuron)
{
CNeuronPool *temp = prevLayer.At(m_myIndex);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
prevLayer = temp.getOutputLayer();
if(CheckPointer(prevLayer) == POINTER_INVALID)
return false;
}
//---
CNeuronBase *prevNeuron, *outputNeuron;
CConnection *con;
int total = prevLayer.Total();
for(int i = 0; i < total; i++)
{
prevNeuron = prevLayer.At(i);
if(CheckPointer(prevNeuron) == POINTER_INVALID)
continue;
double prev_gradient = 0;
int start = i - iWindow + iStep;
start = (start - start % iStep) / iStep;
double stop = (i - i % iStep) / iStep + 1;
for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
{
outputNeuron = OutputLayer.At(out);
int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep;
con = Connections.At(c);
if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
continue;
prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight;
}
prevNeuron.setGradient(prev_gradient);
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index)
{
if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID)
return false;
//---
if(prevNeuron.Type() != defNeuron)
{
CNeuronPool *temp = prevNeuron;
return calcInputGradients(temp.getOutputLayer());
}
//---
CNeuronBase *outputNeuron;
CConnection *con;
double prev_gradient = 0;
int start = (int)index - iWindow + iStep;
start = (start - start % iStep) / iStep;
double stop = (index - index % iStep) / iStep + 1;
for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++)
{
outputNeuron = OutputLayer.At(out);
int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep);
con = Connections.At(c);
if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID)
continue;
prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight;
}
prevNeuron.setGradient(prev_gradient);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBase::Save(int file_handle)
{
if(file_handle == INVALID_HANDLE)
return false;
if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
return false;
//---
if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
return false;
//---
if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
return false;
//---
if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE)
return false;
//---
return Connections.Save(file_handle);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include "LayerDescription.mqh"
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CNet
{
protected:
void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0);
bool InitOpenCL(void);
bool InitDirectML(void);
public:
CNet(CArrayObj *Description);
~CNet(void);
bool feedForward(CArrayDouble *inputVals);
//--- sampleWeight scales this example's output-layer gradient before it propagates back through the
//--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling
//--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same
//--- example multiple times).
void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0);
void getResults(CArrayDouble *&resultVals) ;
double getRecentAverageError() { return recentAverageError; }
//--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see
//--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array
//--- when there is nothing to persist/restore.
bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]);
bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[]);
//--- ephemeral in-run weight checkpoints (agent-local scratch file, never FILE_COMMON): unlike
//--- Save()/Load() these are NOT blocked in the tester/optimizer, since they exist only to let a
//--- single Train() call snapshot/restore weights mid-run (stability-based early stopping) and
//--- never persist a model across separate backtest or optimization passes.
bool SaveCheckpoint(string file_name);
bool LoadCheckpoint(string file_name);
//--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another
//--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight +
//--- tau*live.weight. Intended usage: `this` is a persistent "shadow" net that live trading/OOS
//--- checkpointing reads from, and `live` is the net Train()'s era loop actually backprops
//--- against. A single bad era's raw weights (e.g. an Adam overshoot) can only ever nudge the
//--- shadow by `tau`, so the deployed model can no longer whipsaw between 90%+ and single-digit
//--- OOS accuracy the way a directly-deployed live net can - the shadow is a running average over
//--- many eras, not a snapshot of whichever one happened to look best (or worst) in isolation.
//--- Requires `this` and `live` to share identical topology (same layer/neuron/window counts) -
//--- true whenever the shadow was cloned from live via Save()/Load() and never independently
//--- rebuilt. Silently skips (rather than fails) any layer/neuron pair that doesn't line up, so a
//--- topology mismatch degrades to a partial blend instead of corrupting unrelated layers.
bool BlendWeightsFrom(CNet &live, double tau);
//--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly
//--- initialized and carry the real learning signal) of each output neuron's incoming weight block,
//--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site
//--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to
//--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more
//--- non-majority classes than the true base rate warrants until backProp corrects it over many
//--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the
//--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns
//--- false (no-op) rather than corrupt anything if that assumption doesn't hold.
bool SeedOutputLayerBias(const double &biasValues[]);
//---
static double recentAverageSmoothingFactor;
private:
CArrayLayer *layers;
COpenCLMy *opencl;
CDirectMLMy *directml;
double recentAverageError;
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNet::CNet(CArrayObj *Description)
{
if(CheckPointer(Description) == POINTER_INVALID)
return;
//---
int total = Description.Total();
if(total <= 0)
return;
//---
layers = new CArrayLayer();
if(CheckPointer(layers) == POINTER_INVALID)
return;
//---
CLayer *temp;
CLayerDescription *desc = NULL, *next = NULL, *prev = NULL;
CNeuronBase *neuron = NULL;
CNeuronPool *neuron_p = NULL;
int output_count = 0;
int temp_count = 0;
//---
next = Description.At(1);
if(CheckPointer(next) != POINTER_INVALID &&
(next.type == defNeuron || next.type == defNeuronBaseOCL || next.type == defNeuronConv || next.type == defNeuronConvOCL ||
next.type == defNeuronLSTM))
{
//--- OpenCL first, DirectML/D3D12 next, plain CPU as the final fallback
if(!InitOpenCL())
InitDirectML();
}
//---
for(int i = 0; i < total; i++)
{
prev = desc;
desc = Description.At(i);
if((i + 1) < total)
{
next = Description.At(i + 1);
if(CheckPointer(next) == POINTER_INVALID)
return;
}
else
next = NULL;
int outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL) ? 0 : next.count);
temp = new CLayer(outputs);
int neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL ? 1 : 0));
if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
{
CNeuronBaseOCL *neuron_ocl = NULL;
switch(desc.type)
{
case defNeuron:
case defNeuronBaseOCL:
neuron_ocl = new CNeuronBaseOCL();
if(CheckPointer(neuron_ocl) == POINTER_INVALID)
{
delete temp;
return;
}
if(CheckPointer(opencl) != POINTER_INVALID
? !neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization)
: !neuron_ocl.Init(outputs, 0, directml, desc.count, desc.optimization))
{
delete temp;
return;
}
neuron_ocl.SetActivationFunction(desc.activation);
if(!temp.Add(neuron_ocl))
{
delete neuron_ocl;
delete temp;
return;
}
neuron_ocl = NULL;
break;
case defNeuronConv:
case defNeuronConvOCL:
{
CNeuronConvOCL *neuron_conv = new CNeuronConvOCL();
if(CheckPointer(neuron_conv) == POINTER_INVALID)
{
delete temp;
return;
}
//--- number of sliding positions - same formula the CPU CNeuronConv path uses
if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
{
int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
temp_count = (prevCount - desc.window) % desc.step;
output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
}
else
{
temp_count = (output_count - desc.window) % desc.step;
output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
}
bool convInit = (CheckPointer(opencl) != POINTER_INVALID
? neuron_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, output_count, desc.optimization)
: neuron_conv.Init(outputs, 0, directml, desc.window, desc.step, desc.count, output_count, desc.optimization));
if(!convInit)
{
delete neuron_conv;
delete temp;
return;
}
neuron_conv.SetActivationFunction(desc.activation);
if(!temp.Add(neuron_conv))
{
delete neuron_conv;
delete temp;
return;
}
neuron_conv = NULL;
break;
}
case defNeuronPool:
case defNeuronPoolOCL:
{
CNeuronPoolOCL *neuron_pool = new CNeuronPoolOCL();
if(CheckPointer(neuron_pool) == POINTER_INVALID)
{
delete temp;
return;
}
//--- number of sliding positions - same formula the CPU CNeuronPool path uses
if(CheckPointer(prev) == POINTER_INVALID || prev.type == defNeuron || prev.type == defNeuronBaseOCL)
{
int prevCount = (CheckPointer(prev) == POINTER_INVALID ? desc.count : prev.count);
temp_count = (prevCount - desc.window) % desc.step;
output_count = (prevCount - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
}
else
{
temp_count = (output_count - desc.window) % desc.step;
output_count = (output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2);
}
bool poolInit = (CheckPointer(opencl) != POINTER_INVALID
? neuron_pool.Init(outputs, 0, opencl, desc.window, desc.step, output_count, desc.optimization)
: neuron_pool.Init(outputs, 0, directml, desc.window, desc.step, output_count, desc.optimization));
if(!poolInit)
{
delete neuron_pool;
delete temp;
return;
}
if(!temp.Add(neuron_pool))
{
delete neuron_pool;
delete temp;
return;
}
neuron_pool = NULL;
break;
}
case defNeuronLSTM:
case defNeuronLSTMOCL:
{
CNeuronLSTMOCL *neuron_lstm = new CNeuronLSTMOCL();
if(CheckPointer(neuron_lstm) == POINTER_INVALID)
{
delete temp;
return;
}
bool lstmInit = (CheckPointer(opencl) != POINTER_INVALID
? neuron_lstm.Init(outputs, 0, opencl, desc.count, desc.optimization)
: neuron_lstm.Init(outputs, 0, directml, desc.count, desc.optimization));
if(!lstmInit)
{
delete neuron_lstm;
delete temp;
return;
}
if(!temp.Add(neuron_lstm))
{
delete neuron_lstm;
delete temp;
return;
}
neuron_lstm = NULL;
break;
}
default:
return;
break;
}
}
else
for(int n = 0; n < neurons; n++)
{
switch(desc.type)
{
case defNeuron:
neuron = new CNeuron();
if(CheckPointer(neuron) == POINTER_INVALID)
{
delete temp;
delete layers;
return;
}
neuron.Init(outputs, n, desc.optimization);
neuron.SetActivationFunction(desc.activation);
break;
case defNeuronConv:
neuron_p = new CNeuronConv();
if(CheckPointer(neuron_p) == POINTER_INVALID)
{
delete temp;
delete layers;
return;
}
if(CheckPointer(prev) != POINTER_INVALID)
{
if(prev.type == defNeuron)
{
temp_count = (int)((prev.count - desc.window) % desc.step);
output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
}
else
if(n == 0)
{
temp_count = (int)((output_count - desc.window) % desc.step);
output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
}
}
if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
neuron = neuron_p;
break;
case defNeuronPool:
neuron_p = new CNeuronPool();
if(CheckPointer(neuron_p) == POINTER_INVALID)
{
delete temp;
delete layers;
return;
}
if(CheckPointer(prev) != POINTER_INVALID)
{
if(prev.type == defNeuron)
{
temp_count = (int)((prev.count - desc.window) % desc.step);
output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
}
else
if(n == 0)
{
temp_count = (int)((output_count - desc.window) % desc.step);
output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2));
}
}
if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization))
neuron = neuron_p;
break;
case defNeuronLSTM:
neuron_p = new CNeuronLSTM();
if(CheckPointer(neuron_p) == POINTER_INVALID)
{
delete temp;
delete layers;
return;
}
output_count = (next != NULL ? next.window : desc.step);
if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization))
neuron = neuron_p;
break;
}
if(!temp.Add(neuron))
{
delete temp;
delete layers;
return;
}
neuron = NULL;
}
if(!layers.Add(temp))
{
delete temp;
delete layers;
return;
}
}
//---
}
//+------------------------------------------------------------------+
//| Tries to initialize OpenCL; on any failure (no GPU, driver |
//| missing, kernel build error) frees it and leaves opencl==NULL |
//| so the rest of CNet transparently runs its CPU code path. |
//+------------------------------------------------------------------+
bool CNet::InitOpenCL(void)
{
if(CheckPointer(opencl) != POINTER_INVALID)
return true;
//---
opencl = new COpenCLMy();
if(CheckPointer(opencl) == POINTER_INVALID || !opencl.Initialize(cl_program, true))
{
if(CheckPointer(opencl) != POINTER_INVALID)
delete opencl;
opencl = NULL;
PrintFormat("%s: OpenCL unavailable, falling back to CPU", __FUNCTION__);
return false;
}
//--- create kernels
opencl.SetKernelsCount(18);
opencl.KernelCreate(def_k_FeedForward, "FeedForward");
opencl.KernelCreate(def_k_CaclOutputGradient, "CaclOutputGradient");
opencl.KernelCreate(def_k_CaclHiddenGradient, "CaclHiddenGradient");
opencl.KernelCreate(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum");
opencl.KernelCreate(def_k_UpdateWeightsAdam, "UpdateWeightsAdam");
opencl.KernelCreate(def_k_FeedForwardConv, "FeedForwardConv");
opencl.KernelCreate(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv");
opencl.KernelCreate(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum");
opencl.KernelCreate(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam");
opencl.KernelCreate(def_k_LSTM_Gates, "LSTM_Gates");
opencl.KernelCreate(def_k_LSTM_State, "LSTM_State");
opencl.KernelCreate(def_k_LSTM_GateGradient, "LSTM_GateGradient");
opencl.KernelCreate(def_k_LSTM_WeightsGradient, "LSTM_WeightsGradient");
opencl.KernelCreate(def_k_LSTM_InputsGradient, "LSTM_InputsGradient");
opencl.KernelCreate(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam");
opencl.KernelCreate(def_k_LSTM_UpdateWeightsMomentum, "LSTM_UpdateWeightsMomentum");
opencl.KernelCreate(def_k_FeedForwardProof, "FeedForwardProof");
opencl.KernelCreate(def_k_CalcInputGradientProof, "CalcInputGradientProof");
return true;
}
//+------------------------------------------------------------------+
//| Second-tier GPU fallback: only tried when OpenCL init failed. |
//| Requires DirectML\WarriorDML.dll in the terminal's Libraries |
//| folder (build it with DirectML\build.bat); on any failure frees |
//| itself and leaves directml==NULL so CNet falls through to CPU. |
//+------------------------------------------------------------------+
bool CNet::InitDirectML(void)
{
if(CheckPointer(directml) != POINTER_INVALID)
return true;
//---
directml = new CDirectMLMy();
if(CheckPointer(directml) == POINTER_INVALID)
return false;
directml.SetCpuLoadPercent((int)TargetCPULoad);
if(!directml.Initialize())
{
int err = directml.LastError();
delete directml;
directml = NULL;
string reason;
switch(err)
{
case 1: reason = "CreateDXGIFactory1 failed"; break;
case 2: reason = "no DX12 hardware adapter found (feature level 11_0)"; break;
case 3: reason = "compute command queue creation failed"; break;
case 4: reason = "command allocator creation failed"; break;
case 5: reason = "command list creation failed"; break;
case 6: reason = "fence creation failed"; break;
case 7: reason = "fence event creation failed"; break;
case 8: reason = "HLSL kernel compile/PSO creation failed"; break;
default: reason = "neither WarriorDML.dll nor WarriorCPU.dll loaded (check Libraries folder / \"Allow DLL imports\")";
}
PrintFormat("%s: DirectML/D3D12 and CPU DLL both unavailable (%s), falling back to slow per-object CPU path", __FUNCTION__, reason);
return false;
}
if(directml.Tier() == COMPUTE_TIER_CPU)
PrintFormat("%s: DirectML/D3D12 unavailable, using multithreaded CPU DLL fallback (%d threads)", __FUNCTION__, directml.CpuThreadsUsed());
else
PrintFormat("%s: DirectML/D3D12 GPU tier active", __FUNCTION__);
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNet::feedForward(CArrayDouble *inputVals)
{
if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1)
return false;
//---
CLayer *previous = NULL;
CLayer *current = layers.At(0);
int total = MathMin(current.Total(), inputVals.Total());
CNeuronBase *neuron = NULL;
bool gpuActive = (CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID);
if(!gpuActive)
{
for(int i = 0; i < total; i++)
{
neuron = current.At(i);
if(CheckPointer(neuron) == POINTER_INVALID)
return false;
neuron.setOutputVal(inputVals.At(i));
}
}
else
{
CNeuronBaseOCL *neuron_ocl = current.At(0);
int total_data = inputVals.Total();
bool written;
if(CheckPointer(opencl) != POINTER_INVALID)
{
//--- OpenCL device buffers are float32 (see AI\Network.cl) - unlike CBufferDouble's own
//--- BufferWrite(), this call bypasses that class entirely (it writes straight into the
//--- first layer's Output buffer by index), so the narrow-to-float has to happen here too.
float array[];
if(ArrayResize(array, total_data) < 0)
return false;
for(int d = 0; d < total_data; d++)
array[d] = (float)inputVals.At(d);
written = opencl.BufferWrite(neuron_ocl.getOutputIndex(), array, 0, 0, total_data);
}
else
{
double array[];
if(ArrayResize(array, total_data) < 0)
return false;
for(int d = 0; d < total_data; d++)
array[d] = inputVals.At(d);
written = directml.BufferWrite(neuron_ocl.getOutputIndex(), array, total_data);
}
if(!written)
return false;
}
//---
CObject *temp = NULL;
for(int l = 1; l < layers.Total(); l++)
{
previous = current;
current = layers.At(l);
if(CheckPointer(current) == POINTER_INVALID)
return false;
//---
if(gpuActive)
{
CNeuronBaseOCL *current_ocl = current.At(0);
if(!current_ocl.feedForward(previous.At(0)))
return false;
continue;
}
//---
total = current.Total();
if(current.At(0).Type() == defNeuron)
total--;
//---
for(int n = 0; n < total; n++)
{
neuron = current.At(n);
if(CheckPointer(neuron) == POINTER_INVALID)
return false;
if(previous.At(0).Type() == defNeuron)
{
temp = previous;
if(!neuron.feedForward(temp))
return false;
continue;
}
if(neuron.Type() == defNeuron)
{
if(n == 0)
{
CLayer *temp_l = new CLayer(total);
if(CheckPointer(temp_l) == POINTER_INVALID)
return false;
CNeuronPool *Pool = NULL;
for(int p = 0; p < previous.Total(); p++)
{
Pool = previous.At(p);
if(CheckPointer(Pool) == POINTER_INVALID)
return false;
temp_l.AddArray(Pool.getOutputLayer());
}
temp = temp_l;
}
if(!neuron.feedForward(temp))
return false;
if(n == total - 1)
{
CLayer *temp_l = temp;
temp_l.FreeMode(false);
temp_l.Shutdown();
delete temp_l;
}
continue;
}
temp = previous.At(n);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
if(!neuron.feedForward(temp))
return false;
}
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNet::backProp(CArrayDouble *targetVals, double sampleWeight)
{
if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
return;
if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
{
backPropOCL(targetVals, sampleWeight);
return;
}
//---
CLayer *outputLayer = layers.At(layers.Total() - 1);
if(CheckPointer(outputLayer) == POINTER_INVALID)
return;
//---
double error = 0.0;
int total = outputLayer.Total() - 1;
//--- 3-output classification case: true softmax + categorical-cross-entropy gradient
//--- (dL/dz_i = softmax_i - target_i, the standard multi-class formula - see nnbook.txt section
//--- 1.4) instead of 3 independent per-neuron sigmoid deltas. Forward activation stays SIGMOID
//--- (bounded, avoids the historical logit-runaway collapse documented at
//--- BuildFreshTopology()'s desc.activation comment in ExpertSignalAIBase.mqh), but the BACKWARD
//--- delta is now computed from the softmax-normalized probability across all 3 outputs jointly,
//--- not each neuron's own raw sigmoid value in isolation. This is what actually ties Buy/Sell/
//--- Neutral together during training: raising one class's softmax probability now structurally
//--- lowers the other two's (via the shared normalizing sum), giving real competition instead of
//--- three independent binary regressions that can all drift toward "predict Neutral" together -
//--- root cause of the "overshoot to all-Neutral" convergence failure this replaces.
bool useSoftmaxGrad = (total == 3);
double smax[3];
if(useSoftmaxGrad)
{
double maxLogit = -DBL_MAX;
for(int n = 0; n < 3; n++)
{
CNeuron *nrn = outputLayer.At(n);
maxLogit = MathMax(maxLogit, nrn.getOutputVal());
}
double sum = 0.0;
for(int n = 0; n < 3; n++)
{
CNeuron *nrn = outputLayer.At(n);
smax[n] = exp(nrn.getOutputVal() - maxLogit);
sum += smax[n];
}
for(int n = 0; n < 3; n++)
smax[n] /= sum;
}
for(int n = 0; n < total && !IsStopped(); n++)
{
CNeuron *neuron = outputLayer.At(n);
double target = targetVals.At(n);
double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
double delta = clampedTarget - neuron.getOutputVal();
error += delta * delta;
if(useSoftmaxGrad)
neuron.setGradient(clampedTarget - smax[n]);
else
neuron.calcOutputGradients(targetVals.At(n));
//--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
//--- sampleWeight is derived) - scales the just-computed output gradient in place, before the
//--- hidden layers below read it via sumDOW(), so the whole backward chain sees the weighted
//--- signal without needing its own separate weighting logic.
if(sampleWeight != 1.0)
neuron.setGradient(neuron.getGradient() * sampleWeight);
}
error /= total;
error = sqrt(error);
recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
//---
CNeuronBase *neuron = NULL;
CObject *temp = NULL;
for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--)
{
CLayer *hiddenLayer = layers.At(layerNum);
CLayer *nextLayer = layers.At(layerNum + 1);
total = hiddenLayer.Total();
for(int n = 0; n < total && !IsStopped(); ++n)
{
neuron = hiddenLayer.At(n);
if(nextLayer.At(0).Type() == defNeuron)
{
temp = nextLayer;
neuron.calcHiddenGradients(temp);
continue;
}
if(neuron.Type() == defNeuron)
{
double g = 0;
for(int i = 0; i < nextLayer.Total(); i++)
{
temp = nextLayer.At(i);
neuron.calcHiddenGradients(temp);
g += neuron.getGradient();
}
neuron.setGradient(g);
continue;
}
temp = nextLayer.At(n);
neuron.calcHiddenGradients(temp);
}
}
//---
for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--)
{
CLayer *layer = layers.At(layerNum);
CLayer *prevLayer = layers.At(layerNum - 1);
total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0);
int n_conv = 0;
for(int n = 0; n < total && !IsStopped(); n++)
{
neuron = layer.At(n);
if(CheckPointer(neuron) == POINTER_INVALID)
return;
if(neuron.Type() == defNeuronPool)
continue;
switch(prevLayer.At(0).Type())
{
case defNeuron:
temp = prevLayer;
neuron.updateInputWeights(temp);
break;
case defNeuronConv:
case defNeuronPool:
case defNeuronLSTM:
if(neuron.Type() == defNeuron)
{
for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++)
{
temp = prevLayer.At(n_conv);
neuron.updateInputWeights(temp);
}
}
else
{
temp = prevLayer.At(n);
neuron.updateInputWeights(temp);
}
break;
default:
temp = NULL;
break;
}
}
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNet::backPropOCL(CArrayDouble *targetVals, double sampleWeight)
{
if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID ||
(CheckPointer(opencl) == POINTER_INVALID && CheckPointer(directml) == POINTER_INVALID))
return;
CLayer *currentLayer = layers.At(layers.Total() - 1);
if(CheckPointer(currentLayer) == POINTER_INVALID)
return;
//---
double error = 0.0;
int total = targetVals.Total();
double result[];
CNeuronBaseOCL *neuron = currentLayer.At(0);
if(neuron.getOutputVal(result) < total)
return;
for(int n = 0; n < total && !IsStopped(); n++)
{
double target = targetVals.At(n);
// Deliberately NOT special-cased on target==0 (an earlier version zeroed delta whenever
// target==0, so a one-hot classification target only ever counted the true class's own
// error - e.g. a Neutral-labeled bar's Buy/Sell neurons were invisible to this metric,
// which is what CNet::backProp()'s CPU-fallback path computes for every output
// unconditionally, and is what actually drives the dError<0.1 convergence gate in
// ExpertSignalAIBase::Train(). The real gradient (CPU_CalcOutputGradient in WarriorCPU.cpp
// / DirectML's equivalent) was never affected - only this diagnostic/convergence metric was.
double delta = (target > 1 ? 1 : target < -1 ? -1 : target) - result[n];
error += MathPow(delta, 2);
}
error /= total;
error = sqrt(error);
recentAverageError += (error - recentAverageError) / recentAverageSmoothingFactor;
if(!neuron.calcOutputGradients(targetVals))
return;
//--- 3-output classification case: overwrite the native per-neuron sigmoid delta with the true
//--- softmax + categorical-cross-entropy gradient (softmax_i - target_i), computed here in MQL5
//--- from the raw outputs already read back into result[] above - see the matching CNet::backProp()
//--- (CPU fallback) comment for the full rationale (ties Buy/Sell/Neutral together via the shared
//--- softmax normalizer instead of training 3 independent binary regressions). Backend DLLs/kernels
//--- (WarriorCPU.dll/WarriorDML.dll/Network.cl) still only ever compute the raw, unweighted
//--- per-neuron delta; this correction - like the sampleWeight scaling below - is applied entirely
//--- on the MQL5 side, so none of the 3 compute backends need to change.
if(total == 3)
{
double maxLogit = MathMax(result[0], MathMax(result[1], result[2]));
double smax[3];
double sm = 0.0;
for(int n = 0; n < 3; n++)
{
smax[n] = exp(result[n] - maxLogit);
sm += smax[n];
}
double gradOverwrite[3];
for(int n = 0; n < 3; n++)
{
smax[n] /= sm;
double target = targetVals.At(n);
double clampedTarget = (target > 1 ? 1 : target < -1 ? -1 : target);
gradOverwrite[n] = clampedTarget - smax[n];
}
neuron.setGradient(gradOverwrite);
}
//--- inverse-class-frequency loss weighting (see ExpertSignalAIBase.mqh's Train() for how
//--- sampleWeight is derived). CalcOutputGradient() above only computes the raw, unweighted delta
//--- (WarriorCPU.dll/WarriorDML.dll/Network.cl have no notion of per-sample weighting), so the
//--- gradient buffer is read back, scaled here in MQL5, and pushed back before the hidden layers
//--- below read it via CalcHiddenGradient/sumDOW - avoids touching any of the 3 compute backends.
if(sampleWeight != 1.0)
{
double gradVals[];
int gradCount = neuron.getGradient(gradVals);
if(gradCount > 0)
{
for(int g = 0; g < gradCount; g++)
gradVals[g] *= sampleWeight;
neuron.setGradient(gradVals);
}
}
//--- Calc Hidden Gradients
CObject *temp = NULL;
total = layers.Total();
for(int layerNum = total - 2; layerNum > 0; layerNum--)
{
CLayer *nextLayer = currentLayer;
currentLayer = layers.At(layerNum);
neuron = currentLayer.At(0);
neuron.calcHiddenGradients(nextLayer.At(0));
}
//---
CLayer *prevLayer = layers.At(total - 1);
for(int layerNum = total - 1; layerNum > 0; layerNum--)
{
currentLayer = prevLayer;
prevLayer = layers.At(layerNum - 1);
neuron = currentLayer.At(0);
neuron.updateInputWeights(prevLayer.At(0));
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
void CNet::getResults(CArrayDouble *&resultVals)
{
if(CheckPointer(resultVals) == POINTER_INVALID)
{
resultVals = new CArrayDouble();
if(CheckPointer(resultVals) == POINTER_INVALID)
return;
}
//---
resultVals.Clear();
if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0)
return;
//---
CLayer *output = layers.At(layers.Total() - 1);
if(CheckPointer(output) == POINTER_INVALID)
return;
//---
if(CheckPointer(opencl) != POINTER_INVALID || CheckPointer(directml) != POINTER_INVALID)
{
switch(output.At(0).Type())
{
case defNeuronBaseOCL:
case defNeuronConvOCL:
case defNeuronPoolOCL:
case defNeuronLSTMOCL:
{
CNeuronBaseOCL *temp = output.At(0);
temp.getOutputVal(resultVals);
return;
}
}
}
CNeuronBase *neuron = NULL;
CLayer *temp = NULL;
int total = output.Total();
if(output.At(0).Type() == defNeuron)
total--;
//---
for(int i = 0; i < total; i++)
{
neuron = output.At(i);
if(CheckPointer(neuron) == POINTER_INVALID)
continue;
if(neuron.Type() == defNeuron)
{
resultVals.Add(neuron.getOutputVal());
continue;
}
CNeuronPool *n = neuron;
temp = n.getOutputLayer();
for(int ii = 0; ii < temp.Total(); ii++)
{
neuron = temp.At(ii);
if(CheckPointer(neuron) == POINTER_INVALID)
continue;
resultVals.Add(neuron.getOutputVal());
}
}
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNet::Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[])
{
//--- only the shared FILE_COMMON production weights are protected from being overwritten by
//--- backtest/optimization noise - a LOCAL (common=false) file is exactly what the tester's
//--- per-agent cross-pass weight cache uses (see CExpertSignalAIBase::InitNeuralNetwork), and
//--- must be allowed to write even inside the tester/optimizer.
if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
return true;
if(file_name == NULL)
return false;
//---
int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_WRITE);
if(handle == INVALID_HANDLE)
return false;
//---
if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0 || FileWriteLong(handle, era) <= 0 ||
FileWriteInteger(handle, trainingComplete ? 1 : 0) <= 0)
{
FileClose(handle);
return false;
}
//--- AutoTuneIndicators "winning" AD indicator params, same append-only pattern as the era/
//--- trainingComplete fields above: count-prefixed so older readers can still stop before this block
int paramsCount = ArraySize(indicatorParams);
if(FileWriteInteger(handle, paramsCount) <= 0)
{
FileClose(handle);
return false;
}
for(int p = 0; p < paramsCount; p++)
if(FileWriteDouble(handle, indicatorParams[p]) <= 0)
{
FileClose(handle);
return false;
}
bool result = layers.Save(handle);
FileFlush(handle);
FileClose(handle);
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNet::Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[])
{
//--- see the matching comment in Save() - only shared FILE_COMMON production weights are blocked
if(common && (MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD)))
return false;
//---
if(file_name == NULL)
return false;
//---
Print(file_name);
int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ);
if(handle == INVALID_HANDLE)
return false;
//---
error = FileReadDouble(handle);
undefine = FileReadDouble(handle);
forecast = FileReadDouble(handle);
time = (datetime)FileReadLong(handle);
era = FileReadLong(handle);
// Older save files predate this field - FileReadInteger returns 0 past EOF,
// which correctly defaults to "not complete" so a resumed run keeps training
// instead of silently trusting an unfinished/unknown model as done.
trainingComplete = (FileReadInteger(handle) != 0);
// Older save files predate this block too - FileReadInteger returns 0 past EOF, which
// correctly yields an empty indicatorParams (nothing to restore) instead of misreading weights.
ArrayFree(indicatorParams);
int paramsCount = FileReadInteger(handle);
if(paramsCount > 0)
{
ArrayResize(indicatorParams, paramsCount);
for(int p = 0; p < paramsCount; p++)
indicatorParams[p] = FileReadDouble(handle);
}
//---
if(CheckPointer(layers) != POINTER_INVALID)
layers.Clear();
else
layers = new CArrayLayer();
int i = 0, num;
//---
if(!InitOpenCL())
InitDirectML();
//--- check
//--- read and check start marker - 0xFFFFFFFFFFFFFFFF
long temp = FileReadLong(handle);
if(temp == -1)
{
//--- read and check array type
if(FileReadInteger(handle, INT_VALUE) != layers.Type())
{
FileClose(handle);
return(false);
}
}
else
{
FileClose(handle);
return(false);
}
//--- read array length
num = FileReadInteger(handle, INT_VALUE);
//--- read array
if(num != 0)
{
for(i = 0; i < num; i++)
{
//--- create new element
CLayer *Layer = new CLayer(0, handle, opencl, directml);
if(!Layer.Load(handle))
break;
if(!layers.Add(Layer))
break;
}
}
FileClose(handle);
//--- result
return (layers.Total() == num);
}
//+------------------------------------------------------------------+
//| Ephemeral weight-only checkpoint, deliberately NOT gated by |
//| MQL_TESTER/MQL_OPTIMIZATION - see class declaration comment. |
//+------------------------------------------------------------------+
bool CNet::SaveCheckpoint(string file_name)
{
if(file_name == NULL || CheckPointer(layers) == POINTER_INVALID)
return false;
int handle = FileOpen(file_name, FILE_BIN | FILE_WRITE);
if(handle == INVALID_HANDLE)
return false;
bool result = layers.Save(handle);
FileFlush(handle);
FileClose(handle);
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNet::LoadCheckpoint(string file_name)
{
if(file_name == NULL)
return false;
int handle = FileOpen(file_name, FILE_BIN | FILE_READ);
if(handle == INVALID_HANDLE)
return false;
if(CheckPointer(layers) != POINTER_INVALID)
layers.Clear();
else
layers = new CArrayLayer();
int i = 0, num;
long temp = FileReadLong(handle);
if(temp == -1)
{
if(FileReadInteger(handle, INT_VALUE) != layers.Type())
{
FileClose(handle);
return(false);
}
}
else
{
FileClose(handle);
return(false);
}
num = FileReadInteger(handle, INT_VALUE);
if(num != 0)
{
for(i = 0; i < num; i++)
{
CLayer *Layer = new CLayer(0, handle, opencl, directml);
if(!Layer.Load(handle))
break;
if(!layers.Add(Layer))
break;
}
}
FileClose(handle);
return (layers.Total() == num);
}
//+------------------------------------------------------------------+
//| See this method's declaration comment for the EMA shadow-weight |
//| deployment rationale. |
//+------------------------------------------------------------------+
bool CNet::BlendWeightsFrom(CNet &live, double tau)
{
if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID)
return false;
int layerTotal = MathMin(layers.Total(), live.layers.Total());
for(int l = 0; l < layerTotal; l++)
{
CLayer *shadowLayer = layers.At(l);
CLayer *liveLayer = live.layers.At(l);
if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID)
continue;
int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total());
for(int n = 0; n < neuronTotal; n++)
{
CNeuronBaseOCL *shadowNeuron = shadowLayer.At(n);
CNeuronBaseOCL *liveNeuron = liveLayer.At(n);
if(CheckPointer(shadowNeuron) == POINTER_INVALID || CheckPointer(liveNeuron) == POINTER_INVALID)
continue;
if(shadowNeuron.Type() != liveNeuron.Type())
continue;
double shadowW[], liveW[];
switch(shadowNeuron.Type())
{
case defNeuronBaseOCL:
if(shadowNeuron.getWeights(shadowW) > 0 && liveNeuron.getWeights(liveW) > 0)
{
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
for(int wi = 0; wi < wt; wi++)
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
shadowNeuron.setWeights(shadowW);
}
break;
case defNeuronConvOCL:
{
CNeuronConvOCL *shadowConv = shadowNeuron;
CNeuronConvOCL *liveConv = liveNeuron;
if(shadowConv.getWeightsConv(shadowW) > 0 && liveConv.getWeightsConv(liveW) > 0)
{
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
for(int wi = 0; wi < wt; wi++)
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
shadowConv.setWeightsConv(shadowW);
}
}
break;
case defNeuronLSTMOCL:
{
CNeuronLSTMOCL *shadowLstm = shadowNeuron;
CNeuronLSTMOCL *liveLstm = liveNeuron;
if(shadowLstm.getWeightsLSTM(shadowW) > 0 && liveLstm.getWeightsLSTM(liveW) > 0)
{
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
for(int wi = 0; wi < wt; wi++)
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
shadowLstm.setWeightsLSTM(shadowW);
}
}
break;
}
}
}
return true;
}
//+------------------------------------------------------------------+
//| See this method's declaration comment for the cold-start bias |
//| rationale. The weight block that produces the output layer's |
//| values is stored on the layer BEFORE it (see CNeuronBaseOCL:: |
//| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE |
//| neuron, laid out as (sourceNeurons+1) values per destination |
//| neuron, the last of which is that neuron's bias term), so this |
//| reaches one layer back from the output layer to edit it. |
//+------------------------------------------------------------------+
bool CNet::SeedOutputLayerBias(const double &biasValues[])
{
int outputs = ArraySize(biasValues);
if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
return false;
CLayer *sourceLayer = layers.At(layers.Total() - 2);
if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0)
return false;
CNeuronBaseOCL *sourceNeuron = sourceLayer.At(0);
if(CheckPointer(sourceNeuron) == POINTER_INVALID || sourceNeuron.Type() != defNeuronBaseOCL)
return false;
int inputs = sourceNeuron.Neurons();
double weights[];
int count = sourceNeuron.getWeights(weights);
if(count != (inputs + 1) * outputs)
return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess
for(int i = 0; i < outputs; i++)
weights[(inputs + 1) * i + inputs] = biasValues[i];
return sourceNeuron.setWeights(weights);
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::Save(const int file_handle)
{
if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle))
return false;
if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE)
return false;
if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE)
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronPool::Load(const int file_handle)
{
if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle))
return false;
iWindow = FileReadInteger(file_handle, INT_VALUE);
iStep = FileReadInteger(file_handle, INT_VALUE);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::Save(const int file_handle)
{
if(!CNeuronPool::Save(file_handle))
return false;
if(FileWriteDouble(file_handle, param) < 8)
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronConv::Load(const int file_handle)
{
if(!CNeuronPool::Load(file_handle))
return false;
param = FileReadDouble(file_handle);
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CNeuronLSTM : public CNeuronPool
{
protected:
CLayer *ForgetGate;
CLayer *InputGate;
CLayer *OutputGate;
CLayer *NewContent;
CArrayDouble *Memory;
CArrayDouble *PrevMemory;
CArrayDouble *Input;
CArrayDouble *InputGradient;
//---
virtual bool feedForward(CLayer *prevLayer);
virtual bool calcHiddenGradients(CLayer *&nextLayer);
virtual bool updateInputWeights(CLayer *&prevLayer);
virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data);
virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type);
virtual CArrayDouble *CalculateGate(CLayer *gate, CArrayDouble *sequence);
public:
CNeuronLSTM(void);
~CNeuronLSTM(void);
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
//---
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
virtual bool calcInputGradients(CLayer *prevLayer) ;
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
//--- methods for working with files
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle);
virtual int Type(void) const { return defNeuronLSTM; }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronLSTM::CNeuronLSTM(void)
{
ForgetGate = new CLayer();
InputGate = new CLayer();
OutputGate = new CLayer();
NewContent = new CLayer();
Memory = new CArrayDouble();
PrevMemory = new CArrayDouble();
Input = new CArrayDouble();
InputGradient = new CArrayDouble();
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronLSTM::~CNeuronLSTM(void)
{
if(CheckPointer(ForgetGate) != POINTER_INVALID)
delete ForgetGate;
if(CheckPointer(InputGate) != POINTER_INVALID)
delete InputGate;
if(CheckPointer(OutputGate) != POINTER_INVALID)
delete OutputGate;
if(CheckPointer(NewContent) != POINTER_INVALID)
delete NewContent;
if(CheckPointer(Memory) != POINTER_INVALID)
delete Memory;
if(CheckPointer(PrevMemory) != POINTER_INVALID)
delete PrevMemory;
if(CheckPointer(Input) != POINTER_INVALID)
delete Input;
if(CheckPointer(InputGradient) != POINTER_INVALID)
delete InputGradient;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type)
{
if(units_count <= 0)
return false;
//--- Init Layers
if(!CNeuronPool::Init(numOutputs, myIndex, window, step, units_count, optimization_type))
return false;
if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type))
return false;
if(!InitLayer(InputGate, units_count, window + units_count, optimization_type))
return false;
if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type))
return false;
if(!InitLayer(NewContent, units_count, window + units_count, optimization_type))
return false;
if(!Memory.Reserve(units_count))
return false;
if(!PrevMemory.Reserve(units_count))
return false;
CNeuron *temp;
for(int i = 0; i < units_count; i++)
{
if(!Memory.Add(0))
return false;
if(!PrevMemory.Add(0))
return false;
temp = OutputLayer.At(i);
temp.setOutputVal(0);
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type)
{
if(CheckPointer(layer) == POINTER_INVALID)
{
layer = new CLayer(numOutputs);
if(CheckPointer(layer) == POINTER_INVALID)
return false;
}
else
layer.Clear();
if(!layer.Reserve(numUnits))
return false;
//---
CNeuron *temp;
for(int i = 0; i < numUnits; i++)
{
temp = new CNeuron();
if(CheckPointer(temp) == POINTER_INVALID)
return false;
if(!temp.Init(numOutputs + 1, i, optimization_type))
return false;
if(!layer.Add(temp))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::feedForward(CLayer *prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0)
return false;
CNeuronBase *temp;
CConnection *temp_con;
if(CheckPointer(Input) == POINTER_INVALID)
{
Input = new CArrayDouble();
if(CheckPointer(Input) == POINTER_INVALID)
return false;
}
else
Input.Clear();
//--- Concatenate input sequence
int total = prevLayer.Total();
if(!Input.Reserve(total + OutputLayer.Total()))
return false;
for(int i = 0; i < total; i++)
{
temp = prevLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
return false;
}
total = OutputLayer.Total();
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal()))
return false;
}
int total_data = Input.Total();
//--- Calculated forget gate
CArrayDouble *forget_gate = CalculateGate(ForgetGate, Input);
if(CheckPointer(forget_gate) == POINTER_INVALID)
return false;
//--- Calculated input gate
CArrayDouble *input_gate = CalculateGate(InputGate, Input);
if(CheckPointer(input_gate) == POINTER_INVALID)
return false;
//--- Calculated output gate
CArrayDouble *output_gate = CalculateGate(OutputGate, Input);
if(CheckPointer(output_gate) == POINTER_INVALID)
return false;
//--- Calculated new content
CArrayDouble *new_content = new CArrayDouble();
if(CheckPointer(new_content) == POINTER_INVALID)
return false;
total = NewContent.Total();
for(int i = 0; i < total; i++)
{
temp = NewContent.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
double val = 0;
for(int c = 0; c < total_data; c++)
{
temp_con = temp.Connections.At(c);
if(CheckPointer(temp_con) == POINTER_INVALID)
return false;
val += temp_con.weight * Input.At(c);
}
val = TanhFunction(val);
temp.setOutputVal(val);
if(!new_content.Add(val))
return false;
}
//--- Calculated output sequences
for(int i = 0; i < total; i++)
{
if(PrevMemory.Total() <= i)
PrevMemory.Add(Memory.At(i));
else
PrevMemory.Update(i, Memory.At(i));
double value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i);
if(!Memory.Update(i, value))
return false;
temp = OutputLayer.At(i);
value = TanhFunction(value) * output_gate.At(i);
temp.setOutputVal(value);
}
//---
delete forget_gate;
delete input_gate;
delete new_content;
delete output_gate;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CArrayDouble *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayDouble *sequence)
{
CNeuronBase *temp;
CConnection *temp_con;
CArrayDouble *result = new CArrayDouble();
if(CheckPointer(gate) == POINTER_INVALID)
return NULL;
int total = gate.Total();
int total_data = sequence.Total();
for(int i = 0; i < total; i++)
{
temp = gate.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
{
delete result;
return NULL;
}
double val = 0;
for(int c = 0; c < total_data; c++)
{
temp_con = temp.Connections.At(c);
if(CheckPointer(temp_con) == POINTER_INVALID)
{
delete result;
return NULL;
}
val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c));
}
val = SigmoidFunction(val);
temp.setOutputVal(val);
if(!result.Add(val))
{
delete result;
return NULL;
}
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer)
{
if(CheckPointer(InputGradient) == POINTER_INVALID)
{
InputGradient = new CArrayDouble();
if(CheckPointer(InputGradient) == POINTER_INVALID)
return false;
}
else
InputGradient.Clear();
//---
int total = OutputLayer.Total();
CNeuron *temp;
CArrayDouble *MemoryGradient = new CArrayDouble();
CNeuron *gate;
CConnection *con;
//---
if(nextLayer != OutputLayer)
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setGradient(temp.sumDOW(nextLayer));
}
//--- Calculated memory and output gate gradients
if(CheckPointer(MemoryGradient) == POINTER_INVALID)
return false;
if(!MemoryGradient.Reserve(total))
return false;
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
gate = OutputGate.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
double value = temp.getGradient() * gate.getOutputVal();
value = TanhFunctionDerivative(Memory.At(i)) * value;
if(i >= MemoryGradient.Total())
{
if(!MemoryGradient.Add(value))
return false;
}
else
{
value = MemoryGradient.At(i) + value;
if(!MemoryGradient.Update(i, value))
return false;
}
gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0);
//--- Calcculated gates and new content gradients
gate = ForgetGate.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
gate = InputGate.At(i);
temp = NewContent.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0);
temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0);
}
//--- Calculated input gradients
int total_inp = temp.getConnections().Total();
for(int n = 0; n < total_inp; n++)
{
double value = 0;
for(int i = 0; i < total; i++)
{
temp = ForgetGate.At(i);
con = temp.getConnections().At(n);
value += temp.getGradient() * con.weight;
//---
temp = InputGate.At(i);
con = temp.getConnections().At(n);
value += temp.getGradient() * con.weight;
//---
temp = OutputGate.At(i);
con = temp.getConnections().At(n);
value += temp.getGradient() * con.weight;
//---
temp = NewContent.At(i);
con = temp.getConnections().At(n);
value += temp.getGradient() * con.weight;
}
if(InputGradient.Total() >= n)
{
if(!InputGradient.Add(value))
return false;
}
else
if(!InputGradient.Update(n, value))
return false;
}
//--- Calculated gradients for prev. state
int shift = total_inp - total;
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
temp.setGradient(InputGradient.At(shift + i));
}
//--- Calculated memory and output gate gradients
for(int i = 0; i < total; i++)
{
temp = OutputLayer.At(i);
gate = OutputGate.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
double value = temp.getGradient() * gate.getPrevVal();
value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value;
if(!MemoryGradient.Update(i, value))
return false;
gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0));
//--- Calcculated gates and new content gradients
gate = ForgetGate.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
gate = InputGate.At(i);
temp = NewContent.At(i);
if(CheckPointer(gate) == POINTER_INVALID)
return false;
gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0));
temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0));
}
//---
delete MemoryGradient;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *&prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID)
return false;
//---
if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input)
|| !updateInputWeights(NewContent, Input))
{
return false;
}
if(optimization == ADAM)
t++;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayDouble *input_data)
{
if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID)
return false;
CNeuronBase *neuron;
CConnection *con;
int total_n = gate.Total();
int total_data = input_data.Total();
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
for(int n = 0; n < total_n; n++)
{
neuron = gate.At(n);
if(CheckPointer(neuron) == POINTER_INVALID)
return false;
double g = neuron.getGradient();
double g2 = g * g;
for(int i = 0; i < total_data; i++)
{
con = neuron.getConnections().At(i);
if(CheckPointer(con) == POINTER_INVALID)
return false;
double data = input_data.At(i);
if(optimization == SGD)
con.weight += con.deltaWeight = (g != 0 && data != 0 ? eta * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight;
else
{
con.mt = b1 * con.mt + (1 - b1) * g;
con.vt = b2 * con.vt + (1 - b2) * g2 + 0.00000001;
con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
// Sign-agreement gate - see CNeuron::updateInputWeights' matching comment.
if(con.deltaWeight * g > 0)
con.weight += con.deltaWeight;
}
// See CNeuron::updateInputWeights' matching clamp for why this is needed - matches
// AI\Network.cl's LSTM_UpdateWeightsAdam MAX_WEIGHT clamp.
con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
}
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index)
{
if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index)
return false;
//---
prevNeuron.setGradient(InputGradient.At(index));
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer)
{
if(CheckPointer(prevLayer) == POINTER_INVALID)
return false;
//---
int total = prevLayer.Total();
if(total <= 0)
return false;
CNeuronBase *neuron;
bool result = true;
for(int i = 0; (i < total && result); i++)
{
neuron = prevLayer.At(i);
if(CheckPointer(neuron) == POINTER_INVALID)
{
result = false;
break;
}
result = calcInputGradients(neuron, i);
}
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Save(const int file_handle)
{
if(!CNeuronPool::Save(file_handle))
return false;
if(!ForgetGate.Save(file_handle))
return false;
if(!InputGate.Save(file_handle))
return false;
if(!OutputGate.Save(file_handle))
return false;
if(!NewContent.Save(file_handle))
return false;
if(!Memory.Save(file_handle))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTM::Load(const int file_handle)
{
if(!CNeuronPool::Load(file_handle))
return false;
if(!ForgetGate.Load(file_handle))
return false;
if(!InputGate.Load(file_handle))
return false;
if(!OutputGate.Load(file_handle))
return false;
if(!NewContent.Load(file_handle))
return false;
if(!Memory.Load(file_handle))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunction(double x)
{
switch(activation)
{
case NONE:
return(x);
break;
case TANH:
return TanhFunction(x);
break;
case SIGMOID:
return SigmoidFunction(x);
break;
case PRELU:
// fixed 0.01 slope, matches CNeuronConv::activationFunction's `param` default -
// this case was previously missing here, so any generic Dense (defNeuron) layer
// given PRELU silently fell through to the `return x` below (pure linear identity,
// no nonlinearity at all) instead of leaky-ReLU.
return(x >= 0 ? x : 0.01 * x);
break;
}
//---
return x;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
double CNeuronBase::activationFunctionDerivative(double x)
{
switch(activation)
{
case NONE:
return(1);
break;
case TANH:
return TanhFunctionDerivative(x);
break;
case SIGMOID:
return SigmoidFunctionDerivative(x);
break;
case PRELU:
// See activationFunction() above - was previously missing here too, defaulting
// to a derivative of 1 (which happens to be right for x>=0, but wrong for x<0,
// where it must be the 0.01 leak slope, not 1).
return(x >= 0 ? 1.0 : 0.01);
break;
}
//---
return 1;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
template<typename T>
int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags)
{
int result = -1;
for(int i = 0; i < m_buffers_total; i++)
{
if(m_buffers[i] != INVALID_HANDLE)
continue;
result = i;
break;
}
//---
if(result < 0)
{
if(ArrayResize(m_buffers, m_buffers_total + 1) > 0)
{
m_buffers_total = ArraySize(m_buffers);
result = m_buffers_total - 1;
m_buffers[result] = INVALID_HANDLE;
}
else
return result;
}
//---
if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags))
return -1;
//---
return result;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL, CDirectMLMy *directml = NULL) : hWeights(-1),
hDeltaWeights(-1),
hOutput(-1),
hGradient(-1)
{
iOutputs = outputs;
iFileHandle = handle;
OpenCL = opencl;
DirectML = directml;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CLayer::Load(const int file_handle)
{
iFileHandle = file_handle;
if(!CArrayObj::Load(file_handle))
return false;
if(CheckPointer(m_data[0]) == POINTER_INVALID)
return false;
//---
switch(m_data[0].Type())
{
case defNeuronBaseOCL:
case defNeuronConvOCL:
case defNeuronPoolOCL:
case defNeuronLSTMOCL:
{
CNeuronBaseOCL *temp = m_data[0];
iOutputs = temp.getConnections();
break;
}
default:
{
CNeuronBase *temp = m_data[0];
iOutputs = temp.getConnections().Total();
break;
}
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNet::~CNet(void)
{
if(CheckPointer(layers) != POINTER_INVALID)
delete layers;
if(CheckPointer(opencl) != POINTER_INVALID)
{
opencl.Shutdown();
delete opencl;
}
if(CheckPointer(directml) != POINTER_INVALID)
delete directml;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
#include "BufferDouble.mqh"
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
class CNeuronBaseOCL : public CObject
{
protected:
COpenCLMy *OpenCL;
CDirectMLMy *DirectML;
CBufferDouble *Output;
CBufferDouble *PrevOutput;
CBufferDouble *Weights;
CBufferDouble *DeltaWeights;
CBufferDouble *Gradient;
CBufferDouble *FirstMomentum;
CBufferDouble *SecondMomentum;
//---
const double alpha;
int t;
//---
int m_myIndex;
ENUM_ACTIVATION activation;
ENUM_OPTIMIZATION optimization;
//---
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
public:
CNeuronBaseOCL(void);
~CNeuronBaseOCL(void);
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
//---
virtual int getOutputIndex(void) { return Output.GetIndex(); }
virtual int getPrevOutIndex(void) { return PrevOutput.GetIndex(); }
virtual int getGradientIndex(void) { return Gradient.GetIndex(); }
virtual int getWeightsIndex(void) { return Weights.GetIndex(); }
virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); }
virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); }
virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();}
//---
virtual int getOutputVal(double &values[]) { return Output.GetData(values); }
virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); }
virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); }
virtual int getGradient(double &values[]) { return Gradient.GetData(values); }
//--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by
//--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call
//--- (which only computes the raw, unweighted delta) and before the backward pass reads this same
//--- buffer to propagate into the hidden layers.
virtual bool setGradient(const double &values[])
{
int count = ArraySize(values);
for(int i = 0; i < count; i++)
if(!Gradient.Update(i, values[i]))
return false;
return Gradient.BufferWrite();
}
// Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created,
// so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here.
virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); }
// Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment
// (see that method's declaration comment) - writes a full replacement weight array back to this
// buffer's device-side (DLL/OpenCL/DirectML) storage. Unlike Weights.Update(i,...) (per-element,
// used by the Adam kernels' own writes), this replaces the whole buffer in one bulk assignment
// then pushes it to the device - the shape callers use when blending two already-read-out arrays.
virtual bool setWeights(double &values[])
{
if(CheckPointer(Weights) == POINTER_INVALID)
return false;
if(!Weights.AssignArray(values))
return false;
return Weights.BufferWrite();
}
virtual int Neurons(void) { return Output.Total(); }
virtual ENUM_ACTIVATION Activation(void) { return activation; }
virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID ? Weights.Total() / (Gradient.Total()) : 0); }
//---
virtual bool feedForward(CObject *SourceObject);
virtual bool calcHiddenGradients(CObject *TargetObject);
virtual bool calcOutputGradients(CArrayDouble *Target);
virtual bool updateInputWeights(CObject *SourceObject);
//---
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle);
//---
virtual int Type(void) const { return defNeuronBaseOCL; }
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum),
activation(TANH),
optimization(SGD),
t(1)
{
OpenCL = NULL;
DirectML = NULL;
Output = new CBufferDouble();
PrevOutput = new CBufferDouble();
Weights = new CBufferDouble();
DeltaWeights = new CBufferDouble();
Gradient = new CBufferDouble();
FirstMomentum = new CBufferDouble();
SecondMomentum = new CBufferDouble();
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronBaseOCL::~CNeuronBaseOCL(void)
{
if(CheckPointer(Output) != POINTER_INVALID)
delete Output;
if(CheckPointer(PrevOutput) != POINTER_INVALID)
delete PrevOutput;
if(CheckPointer(Weights) != POINTER_INVALID)
delete Weights;
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
delete DeltaWeights;
if(CheckPointer(Gradient) != POINTER_INVALID)
delete Gradient;
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
delete FirstMomentum;
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
delete SecondMomentum;
OpenCL = NULL;
DirectML = NULL;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
{
if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
return false;
OpenCL = open_cl;
optimization = optimization_type;
//---
if(CheckPointer(Output) == POINTER_INVALID)
{
Output = new CBufferDouble();
if(CheckPointer(Output) == POINTER_INVALID)
return false;
}
if(!Output.BufferInit(numNeurons, 1.0))
return false;
if(!Output.BufferCreate(OpenCL))
return false;
//---
if(CheckPointer(PrevOutput) == POINTER_INVALID)
{
PrevOutput = new CBufferDouble();
if(CheckPointer(PrevOutput) == POINTER_INVALID)
return false;
}
if(!PrevOutput.BufferInit(numNeurons, 1.0))
return false;
if(!PrevOutput.BufferCreate(OpenCL))
return false;
//---
if(CheckPointer(Gradient) == POINTER_INVALID)
{
Gradient = new CBufferDouble();
if(CheckPointer(Gradient) == POINTER_INVALID)
return false;
}
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
return false;
if(!Gradient.BufferCreate(OpenCL))
return false;
//---
if(numOutputs > 0)
{
if(CheckPointer(Weights) == POINTER_INVALID)
{
Weights = new CBufferDouble();
if(CheckPointer(Weights) == POINTER_INVALID)
return false;
}
int count = (int)((numNeurons + 1) * numOutputs);
if(!Weights.Reserve(count))
return false;
// He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
// et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
// LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
// 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
// ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
// for the fact that they zero out roughly half their input distribution, whereas the
// previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
// was ~2x too conservative here. Applied to every layer through this one shared Init()
// (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
// the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
// differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
// extremes on every backend, so the imprecision there is not worth the much larger, riskier
// change of threading activation awareness through every neuron subtype's Init() overload.
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
for(int i = 0; i < count; i++)
{
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
if(weigh == 0)
weigh = 0.001;
if(!Weights.Add(weigh))
return false;
}
if(!Weights.BufferCreate(OpenCL))
return false;
//---
if(optimization == SGD)
{
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
{
DeltaWeights = new CBufferDouble();
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
return false;
}
if(!DeltaWeights.BufferInit(count, 0))
return false;
if(!DeltaWeights.BufferCreate(OpenCL))
return false;
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
{
delete FirstMomentum;
FirstMomentum = NULL;
}
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
{
delete SecondMomentum;
SecondMomentum = NULL;
}
}
else
{
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
{
delete DeltaWeights;
DeltaWeights = NULL;
}
//---
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
{
FirstMomentum = new CBufferDouble();
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
return false;
}
if(!FirstMomentum.BufferInit(count, 0))
return false;
if(!FirstMomentum.BufferCreate(OpenCL))
return false;
//---
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
{
SecondMomentum = new CBufferDouble();
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
return false;
}
if(!SecondMomentum.BufferInit(count, 0))
return false;
if(!SecondMomentum.BufferCreate(OpenCL))
return false;
}
}
else
{
if(CheckPointer(Weights) != POINTER_INVALID)
delete Weights;
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
delete DeltaWeights;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same |
//| buffer layout, buffers just get created on the DML backend. |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
{
if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0)
return false;
DirectML = direct_ml;
optimization = optimization_type;
//---
if(CheckPointer(Output) == POINTER_INVALID)
{
Output = new CBufferDouble();
if(CheckPointer(Output) == POINTER_INVALID)
return false;
}
if(!Output.BufferInit(numNeurons, 1.0))
return false;
if(!Output.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(PrevOutput) == POINTER_INVALID)
{
PrevOutput = new CBufferDouble();
if(CheckPointer(PrevOutput) == POINTER_INVALID)
return false;
}
if(!PrevOutput.BufferInit(numNeurons, 1.0))
return false;
if(!PrevOutput.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(Gradient) == POINTER_INVALID)
{
Gradient = new CBufferDouble();
if(CheckPointer(Gradient) == POINTER_INVALID)
return false;
}
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
return false;
if(!Gradient.BufferCreate(DirectML))
return false;
//---
if(numOutputs > 0)
{
if(CheckPointer(Weights) == POINTER_INVALID)
{
Weights = new CBufferDouble();
if(CheckPointer(Weights) == POINTER_INVALID)
return false;
}
int count = (int)((numNeurons + 1) * numOutputs);
if(!Weights.Reserve(count))
return false;
// He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
for(int i = 0; i < count; i++)
{
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
if(weigh == 0)
weigh = 0.001;
if(!Weights.Add(weigh))
return false;
}
if(!Weights.BufferCreate(DirectML))
return false;
//---
if(optimization == SGD)
{
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
{
DeltaWeights = new CBufferDouble();
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
return false;
}
if(!DeltaWeights.BufferInit(count, 0))
return false;
if(!DeltaWeights.BufferCreate(DirectML))
return false;
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
{
delete FirstMomentum;
FirstMomentum = NULL;
}
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
{
delete SecondMomentum;
SecondMomentum = NULL;
}
}
else
{
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
{
delete DeltaWeights;
DeltaWeights = NULL;
}
//---
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
{
FirstMomentum = new CBufferDouble();
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
return false;
}
if(!FirstMomentum.BufferInit(count, 0))
return false;
if(!FirstMomentum.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
{
SecondMomentum = new CBufferDouble();
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
return false;
}
if(!SecondMomentum.BufferInit(count, 0))
return false;
if(!SecondMomentum.BufferCreate(DirectML))
return false;
}
}
else
{
if(CheckPointer(Weights) != POINTER_INVALID)
delete Weights;
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
delete DeltaWeights;
}
//---
return true;
}
//+------------------------------------------------------------------+
#include "NeuronOCLConvPool.mqh"
//+------------------------------------------------------------------+
//| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from |
//| scratch from the standard LSTM equations - NOT ported from the |
//| NeuroNet_DNG reference (see the note above the LSTM kernels in |
//| Network.cl for why). Single-timestep-truncated BPTT: gradient |
//| does not flow back into h_prev/c_prev from an earlier step. |
//| Adam-only - Init fails for any other optimization type. |
//+------------------------------------------------------------------+
class CNeuronLSTMOCL : public CNeuronBaseOCL
{
protected:
int m_iInputs;
CBufferDouble *WeightsLSTM;
CBufferDouble *FirstMomentumLSTM;
CBufferDouble *SecondMomentumLSTM;
CBufferDouble *DeltaWeightsLSTM;
CBufferDouble *WeightsGradient;
CBufferDouble *Concatenated;
CBufferDouble *ConcatenatedGradient;
CBufferDouble *Memory;
CBufferDouble *HiddenCache;
//---
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
virtual bool SetInputs(int count);
public:
CNeuronLSTMOCL(void) : m_iInputs(-1)
{
WeightsLSTM = NULL;
FirstMomentumLSTM = NULL;
SecondMomentumLSTM = NULL;
DeltaWeightsLSTM = NULL;
WeightsGradient = NULL;
Concatenated = NULL;
ConcatenatedGradient = NULL;
Memory = NULL;
HiddenCache = NULL;
}
~CNeuronLSTMOCL(void);
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
virtual bool Save(int const file_handle);
virtual bool Load(int const file_handle);
virtual int Type(void) const { return defNeuronLSTMOCL; }
// See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the
// base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); }
virtual bool setWeightsLSTM(double &values[])
{
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
return false;
if(!WeightsLSTM.AssignArray(values))
return false;
return WeightsLSTM.BufferWrite();
}
};
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
CNeuronLSTMOCL::~CNeuronLSTMOCL(void)
{
if(CheckPointer(WeightsLSTM) != POINTER_INVALID)
delete WeightsLSTM;
if(CheckPointer(FirstMomentumLSTM) != POINTER_INVALID)
delete FirstMomentumLSTM;
if(CheckPointer(SecondMomentumLSTM) != POINTER_INVALID)
delete SecondMomentumLSTM;
if(CheckPointer(DeltaWeightsLSTM) != POINTER_INVALID)
delete DeltaWeightsLSTM;
if(CheckPointer(WeightsGradient) != POINTER_INVALID)
delete WeightsGradient;
if(CheckPointer(Concatenated) != POINTER_INVALID)
delete Concatenated;
if(CheckPointer(ConcatenatedGradient) != POINTER_INVALID)
delete ConcatenatedGradient;
if(CheckPointer(Memory) != POINTER_INVALID)
delete Memory;
if(CheckPointer(HiddenCache) != POINTER_INVALID)
delete HiddenCache;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type))
return false;
uint H = numNeurons;
if(CheckPointer(Memory) == POINTER_INVALID)
{
Memory = new CBufferDouble();
if(CheckPointer(Memory) == POINTER_INVALID)
return false;
}
if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(OpenCL))
return false;
if(CheckPointer(Concatenated) == POINTER_INVALID)
{
Concatenated = new CBufferDouble();
if(CheckPointer(Concatenated) == POINTER_INVALID)
return false;
}
if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(OpenCL))
return false;
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
{
ConcatenatedGradient = new CBufferDouble();
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
return false;
}
if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(OpenCL))
return false;
if(CheckPointer(HiddenCache) == POINTER_INVALID)
{
HiddenCache = new CBufferDouble();
if(CheckPointer(HiddenCache) == POINTER_INVALID)
return false;
}
if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(OpenCL))
return false;
m_iInputs = -1;
return true;
}
//+------------------------------------------------------------------+
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above. |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
{
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type))
return false;
uint H = numNeurons;
if(CheckPointer(Memory) == POINTER_INVALID)
{
Memory = new CBufferDouble();
if(CheckPointer(Memory) == POINTER_INVALID)
return false;
}
if(!Memory.BufferInit(2 * H, 0) || !Memory.BufferCreate(DirectML))
return false;
if(CheckPointer(Concatenated) == POINTER_INVALID)
{
Concatenated = new CBufferDouble();
if(CheckPointer(Concatenated) == POINTER_INVALID)
return false;
}
if(!Concatenated.BufferInit(4 * H, 0) || !Concatenated.BufferCreate(DirectML))
return false;
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
{
ConcatenatedGradient = new CBufferDouble();
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
return false;
}
if(!ConcatenatedGradient.BufferInit(4 * H, 0) || !ConcatenatedGradient.BufferCreate(DirectML))
return false;
if(CheckPointer(HiddenCache) == POINTER_INVALID)
{
HiddenCache = new CBufferDouble();
if(CheckPointer(HiddenCache) == POINTER_INVALID)
return false;
}
if(!HiddenCache.BufferInit(H, 0) || !HiddenCache.BufferCreate(DirectML))
return false;
m_iInputs = -1;
return true;
}
//+------------------------------------------------------------------+
//| Lazily sized on the first feedForward call, once the previous |
//| layer's neuron count (the input size) is known. |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::SetInputs(int count)
{
m_iInputs = count;
uint H = (uint)Neurons();
int total = (int)(4 * H * (H + count + 1));
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
{
WeightsLSTM = new CBufferDouble();
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
return false;
}
if(!WeightsLSTM.Reserve(total))
return false;
// Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full
// rationale; fan-in here is hidden units + input width (each gate reads both).
double weighScale = 1.0 / MathSqrt((double)(H + count) + 1.0);
for(int i = 0; i < total; i++)
{
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
if(weigh == 0)
weigh = 0.001;
if(!WeightsLSTM.Add(weigh))
return false;
}
if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
{
FirstMomentumLSTM = new CBufferDouble();
if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
return false;
}
if(!FirstMomentumLSTM.BufferInit(total, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
{
SecondMomentumLSTM = new CBufferDouble();
if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
return false;
}
if(!SecondMomentumLSTM.BufferInit(total, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML))
return false;
//---
// Momentum accumulator - only meaningfully used when optimization==SGD
// (see updateInputWeights()), but allocated unconditionally regardless of
// the configured optimizer, matching CNeuronBaseOCL::Init's pattern for
// the dense-layer DeltaWeights/FirstMomentum/SecondMomentum trio above.
if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
{
DeltaWeightsLSTM = new CBufferDouble();
if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
return false;
}
if(!DeltaWeightsLSTM.BufferInit(total, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(WeightsGradient) == POINTER_INVALID)
{
WeightsGradient = new CBufferDouble();
if(CheckPointer(WeightsGradient) == POINTER_INVALID)
return false;
}
if(!WeightsGradient.BufferInit(total, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
return false;
if(m_iInputs <= 0)
{
if(!SetInputs(NeuronOCL.Neurons()))
return false;
}
else
if(m_iInputs != NeuronOCL.Neurons())
return false;
int H = Neurons();
int I = m_iInputs;
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(!DirectML.LSTMGates(WeightsLSTM.GetIndex(), getOutputIndex(), NeuronOCL.getOutputIndex(), Concatenated.GetIndex(), H, I) ||
!DirectML.LSTMState(Concatenated.GetIndex(), Memory.GetIndex(), getOutputIndex(), HiddenCache.GetIndex(), getOutputIndex(), H))
{
printf("Error of execution DirectML LSTM feedForward");
return false;
}
return Output.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint offset2[2] = {0, 0};
uint size2[2] = {(uint)H, 4};
OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_matrix_w, WeightsLSTM.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_hidden_prev, getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_inputs, NeuronOCL.getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_Gates, def_k_lstmg_concatenated, Concatenated.GetIndex());
OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_hidden_size, H);
OpenCL.SetArgument(def_k_LSTM_Gates, def_k_lstmg_input_size, I);
if(!OpenCL.Execute(def_k_LSTM_Gates, 2, offset2, size2))
{
printf("Error of execution kernel LSTM_Gates: %d", GetLastError());
return false;
}
uint offset1[1] = {0};
uint size1[1] = {(uint)H};
OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_concatenated, Concatenated.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_memory, Memory.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_prev, getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_hidden_cache, HiddenCache.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_State, def_k_lstms_output, getOutputIndex());
OpenCL.SetArgument(def_k_LSTM_State, def_k_lstms_hidden_size, H);
if(!OpenCL.Execute(def_k_LSTM_State, 1, offset1, size1))
{
printf("Error of execution kernel LSTM_State: %d", GetLastError());
return false;
}
//--- Output (== hidden_prev for the next timestep) stays GPU-resident; see the note in
//--- CNeuronBaseOCL::feedForward().
return true;
}
//+------------------------------------------------------------------+
//| Writes the gradient into NeuronOCL (the earlier/input-side layer) |
//| - same inverted-call convention as CNeuronConvOCL::calcInputGradients.|
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(NeuronOCL) == POINTER_INVALID || m_iInputs <= 0)
return false;
int H = Neurons();
int I = m_iInputs;
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(!DirectML.LSTMGateGradient(getGradientIndex(), Memory.GetIndex(), Concatenated.GetIndex(), ConcatenatedGradient.GetIndex(), H) ||
!DirectML.LSTMWeightsGradient(ConcatenatedGradient.GetIndex(), HiddenCache.GetIndex(), NeuronOCL.getOutputIndex(), WeightsGradient.GetIndex(), H, I) ||
!DirectML.LSTMInputsGradient(ConcatenatedGradient.GetIndex(), WeightsLSTM.GetIndex(), NeuronOCL.getGradientIndex(), H, I))
{
printf("Error of execution DirectML LSTM calcInputGradients");
return false;
}
double temp[];
return NeuronOCL.getGradient(temp) > 0;
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint offset1[1] = {0};
uint size1[1] = {(uint)H};
OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_gradient, getGradientIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_memory, Memory.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated, Concatenated.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_GateGradient, def_k_lstmgg_concatenated_gradient, ConcatenatedGradient.GetIndex());
OpenCL.SetArgument(def_k_LSTM_GateGradient, def_k_lstmgg_hidden_size, H);
if(!OpenCL.Execute(def_k_LSTM_GateGradient, 1, offset1, size1))
{
printf("Error of execution kernel LSTM_GateGradient: %d", GetLastError());
return false;
}
uint sizeW[1] = {(uint)WeightsLSTM.Total()};
OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_concatenated_gradient, ConcatenatedGradient.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_cache, HiddenCache.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_inputs, NeuronOCL.getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_WeightsGradient, def_k_lstmwg_weights_gradient, WeightsGradient.GetIndex());
OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_hidden_size, H);
OpenCL.SetArgument(def_k_LSTM_WeightsGradient, def_k_lstmwg_input_size, I);
if(!OpenCL.Execute(def_k_LSTM_WeightsGradient, 1, offset1, sizeW))
{
printf("Error of execution kernel LSTM_WeightsGradient: %d", GetLastError());
return false;
}
uint sizeI[1] = {(uint)I};
OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_concatenated_gradient, ConcatenatedGradient.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_matrix_w, WeightsLSTM.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_InputsGradient, def_k_lstmig_inputs_gradient, NeuronOCL.getGradientIndex());
OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_hidden_size, H);
OpenCL.SetArgument(def_k_LSTM_InputsGradient, def_k_lstmig_input_size, I);
if(!OpenCL.Execute(def_k_LSTM_InputsGradient, 1, offset1, sizeI))
{
printf("Error of execution kernel LSTM_InputsGradient: %d", GetLastError());
return false;
}
//--- NeuronOCL's Gradient stays GPU-resident; see the note in
//--- CNeuronConvOCL::calcInputGradients().
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
return false;
int total = WeightsLSTM.Total();
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(optimization == SGD)
{
if(!DirectML.LSTMUpdateWeightsMomentum(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), DeltaWeightsLSTM.GetIndex(), eta, alpha, total))
{
printf("Error of execution DirectML LSTM_UpdateWeightsMomentum");
return false;
}
}
else
{
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
if(!DirectML.LSTMUpdateWeightsAdam(WeightsLSTM.GetIndex(), WeightsGradient.GetIndex(), FirstMomentumLSTM.GetIndex(), SecondMomentumLSTM.GetIndex(), lt, b1, b2, total))
{
printf("Error of execution DirectML LSTM_UpdateWeightsAdam");
return false;
}
t++;
}
return WeightsLSTM.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint offset1[1] = {0};
uint size1[1] = {(uint)total};
if(optimization == SGD)
{
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_w, WeightsLSTM.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_weights_gradient, WeightsGradient.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_matrix_dw, DeltaWeightsLSTM.GetIndex());
OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_learning_rates, (float)eta);
OpenCL.SetArgument(def_k_LSTM_UpdateWeightsMomentum, def_k_lstmuwm_momentum, (float)alpha);
ResetLastError();
if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsMomentum, 1, offset1, size1))
{
printf("Error of execution kernel LSTM_UpdateWeightsMomentum: %d", GetLastError());
return false;
}
}
else
{
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_w, WeightsLSTM.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_weights_gradient, WeightsGradient.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_m, FirstMomentumLSTM.GetIndex());
OpenCL.SetArgumentBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_matrix_v, SecondMomentumLSTM.GetIndex());
OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_l, (float)lt);
OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b1, (float)b1);
OpenCL.SetArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuwa_b2, (float)b2);
ResetLastError();
if(!OpenCL.Execute(def_k_LSTM_UpdateWeightsAdam, 1, offset1, size1))
{
printf("Error of execution kernel LSTM_UpdateWeightsAdam: %d", GetLastError());
return false;
}
t++;
}
//--- WeightsLSTM stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights().
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Save(const int file_handle)
{
if(!CNeuronBaseOCL::Save(file_handle))
return false;
if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < INT_VALUE)
return false;
if(m_iInputs <= 0)
return true;
if(CheckPointer(WeightsLSTM) == POINTER_INVALID || !WeightsLSTM.BufferRead() || !WeightsLSTM.Save(file_handle))
return false;
if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID || !FirstMomentumLSTM.BufferRead() || !FirstMomentumLSTM.Save(file_handle))
return false;
if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID || !SecondMomentumLSTM.BufferRead() || !SecondMomentumLSTM.Save(file_handle))
return false;
if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID || !DeltaWeightsLSTM.BufferRead() || !DeltaWeightsLSTM.Save(file_handle))
return false;
if(CheckPointer(Memory) == POINTER_INVALID || !Memory.BufferRead() || !Memory.Save(file_handle))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronLSTMOCL::Load(const int file_handle)
{
if(!CNeuronBaseOCL::Load(file_handle))
return false;
m_iInputs = FileReadInteger(file_handle, INT_VALUE);
uint H = (uint)Neurons();
//--- scratch buffers (not persisted) were sized for the placeholder Init() unit
//--- count; resize them now that the real neuron count is known.
if(CheckPointer(Concatenated) == POINTER_INVALID)
{
Concatenated = new CBufferDouble();
if(CheckPointer(Concatenated) == POINTER_INVALID)
return false;
}
Concatenated.BufferFree();
if(!Concatenated.BufferInit(4 * H, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !Concatenated.BufferCreate(OpenCL) : !Concatenated.BufferCreate(DirectML))
return false;
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
{
ConcatenatedGradient = new CBufferDouble();
if(CheckPointer(ConcatenatedGradient) == POINTER_INVALID)
return false;
}
ConcatenatedGradient.BufferFree();
if(!ConcatenatedGradient.BufferInit(4 * H, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !ConcatenatedGradient.BufferCreate(OpenCL) : !ConcatenatedGradient.BufferCreate(DirectML))
return false;
if(CheckPointer(HiddenCache) == POINTER_INVALID)
{
HiddenCache = new CBufferDouble();
if(CheckPointer(HiddenCache) == POINTER_INVALID)
return false;
}
HiddenCache.BufferFree();
if(!HiddenCache.BufferInit(H, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !HiddenCache.BufferCreate(OpenCL) : !HiddenCache.BufferCreate(DirectML))
return false;
//---
if(m_iInputs <= 0)
return true;
int total = (int)(4 * H * (H + m_iInputs + 1));
//---
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
{
WeightsLSTM = new CBufferDouble();
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
return false;
}
if(WeightsLSTM.GetIndex() >= 0)
WeightsLSTM.BufferFree();
if(!WeightsLSTM.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsLSTM.BufferCreate(OpenCL) : !WeightsLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
{
FirstMomentumLSTM = new CBufferDouble();
if(CheckPointer(FirstMomentumLSTM) == POINTER_INVALID)
return false;
}
if(FirstMomentumLSTM.GetIndex() >= 0)
FirstMomentumLSTM.BufferFree();
if(!FirstMomentumLSTM.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumLSTM.BufferCreate(OpenCL) : !FirstMomentumLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
{
SecondMomentumLSTM = new CBufferDouble();
if(CheckPointer(SecondMomentumLSTM) == POINTER_INVALID)
return false;
}
if(SecondMomentumLSTM.GetIndex() >= 0)
SecondMomentumLSTM.BufferFree();
if(!SecondMomentumLSTM.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumLSTM.BufferCreate(OpenCL) : !SecondMomentumLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
{
DeltaWeightsLSTM = new CBufferDouble();
if(CheckPointer(DeltaWeightsLSTM) == POINTER_INVALID)
return false;
}
if(DeltaWeightsLSTM.GetIndex() >= 0)
DeltaWeightsLSTM.BufferFree();
if(!DeltaWeightsLSTM.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsLSTM.BufferCreate(OpenCL) : !DeltaWeightsLSTM.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(Memory) == POINTER_INVALID)
{
Memory = new CBufferDouble();
if(CheckPointer(Memory) == POINTER_INVALID)
return false;
}
if(Memory.GetIndex() >= 0)
Memory.BufferFree();
if(!Memory.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !Memory.BufferCreate(OpenCL) : !Memory.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(WeightsGradient) == POINTER_INVALID)
{
WeightsGradient = new CBufferDouble();
if(CheckPointer(WeightsGradient) == POINTER_INVALID)
return false;
}
if(!WeightsGradient.BufferInit(total, 0))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsGradient.BufferCreate(OpenCL) : !WeightsGradient.BufferCreate(DirectML))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
{
if(CheckPointer(SourceObject) == POINTER_INVALID)
return false;
//---
CNeuronBaseOCL *temp = NULL;
switch(SourceObject.Type())
{
case defNeuronBaseOCL:
case defNeuronConvOCL:
case defNeuronLSTMOCL:
case defNeuronPoolOCL:
temp = SourceObject;
return feedForward(temp);
break;
}
//---
return false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
return false;
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
NeuronOCL.Neurons(), NativeActivationCode(activation)))
{
printf("Error of execution DirectML FeedForward");
return false;
}
return Output.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint global_work_offset[1] = {0};
uint global_work_size[1];
global_work_size[0] = Output.Total();
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
{
printf("Error of execution kernel FeedForward: %d", GetLastError());
return false;
}
//--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
//--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
//--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
//--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
//--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
return false;
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1))
{
printf("Error of execution DirectML CalcHiddenGradient");
return false;
}
return Gradient.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint global_work_offset[1] = {0};
uint global_work_size[1];
global_work_size[0] = Neurons() + 1;
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
{
printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
return false;
}
//--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
//--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
{
if(CheckPointer(Target) == POINTER_INVALID)
return false;
int count = Target.Total();
for(int i = 0; i < count; i++)
if(!Gradient.Update(i, Target.At(i)))
return false;
Gradient.BufferWrite();
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
if(CheckPointer(DirectML) != POINTER_INVALID)
{
if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
{
printf("Error of execution DirectML CalcOutputGradient");
return false;
}
return Gradient.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint global_work_offset[1] = {0};
uint global_work_size[1];
global_work_size[0] = count;
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
ResetLastError();
if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
{
printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
return false;
}
//--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
//--- BufferRead()s itself - see the note in feedForward() above.
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
{
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
return false;
if(CheckPointer(DirectML) != POINTER_INVALID)
{
int inputs = NeuronOCL.Neurons();
int neurons = Neurons();
if(optimization == SGD)
{
if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons))
{
printf("Error of execution DirectML UpdateWeightsMomentum");
return false;
}
}
else
{
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
inputs, lt, b1, b2, neurons))
{
printf("Error of execution DirectML UpdateWeightsAdam");
return false;
}
t++;
}
return NeuronOCL.Weights.BufferRead();
}
if(CheckPointer(OpenCL) == POINTER_INVALID)
return false;
uint global_work_offset[2] = {0, 0};
uint global_work_size[2];
global_work_size[0] = Neurons();
global_work_size[1] = NeuronOCL.Neurons();
if(optimization == SGD)
{
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta);
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
ResetLastError();
if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
{
printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
return false;
}
}
else
{
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
return false;
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
return false;
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
return false;
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
return false;
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
return false;
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
return false;
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
return false;
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1))
return false;
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2))
return false;
uint rest = global_work_size[1] % 4;
global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
ResetLastError();
if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
{
printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
return false;
}
t++;
}
//--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
//--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
{
if(CheckPointer(TargetObject) == POINTER_INVALID)
return false;
//---
CNeuronBaseOCL *temp = NULL;
CNeuronConvOCL *tempConv = NULL;
CNeuronLSTMOCL *tempLstm = NULL;
CNeuronPoolOCL *tempPool = NULL;
switch(TargetObject.Type())
{
case defNeuronBaseOCL:
temp = TargetObject;
return calcHiddenGradients(temp);
break;
case defNeuronConvOCL:
//--- Conv owns the backward step (calcInputGradients), called on itself with
//--- "this" (the earlier layer) passed in so it writes into this->Gradient.
tempConv = TargetObject;
return tempConv.calcInputGradients(GetPointer(this));
break;
case defNeuronLSTMOCL:
tempLstm = TargetObject;
return tempLstm.calcInputGradients(GetPointer(this));
break;
case defNeuronPoolOCL:
tempPool = TargetObject;
return tempPool.calcInputGradients(GetPointer(this));
break;
}
//---
return false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
{
if(CheckPointer(SourceObject) == POINTER_INVALID)
return false;
//---
CNeuronBaseOCL *temp = NULL;
switch(SourceObject.Type())
{
case defNeuronBaseOCL:
case defNeuronConvOCL:
case defNeuronLSTMOCL:
case defNeuronPoolOCL:
temp = SourceObject;
return updateInputWeights(temp);
break;
}
//---
return false;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Save(const int file_handle)
{
if(file_handle == INVALID_HANDLE)
return false;
if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
return false;
//---
if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
return false;
if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
return false;
if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
return false;
//---
if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
return false;
if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
return false;
if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
return false;
//---
if(CheckPointer(Weights) == POINTER_INVALID)
{
FileWriteInteger(file_handle, 0);
return true;
}
else
FileWriteInteger(file_handle, 1);
//---
if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
return false;
if(optimization == SGD)
{
if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
return false;
}
else
{
if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
return false;
if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| |
//+------------------------------------------------------------------+
bool CNeuronBaseOCL::Load(const int file_handle)
{
if(file_handle == INVALID_HANDLE)
return false;
//---
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
t = FileReadInteger(file_handle, INT_VALUE);
if(CheckPointer(Output) == POINTER_INVALID)
{
Output = new CBufferDouble();
if(CheckPointer(Output) == POINTER_INVALID)
return false;
}
if(Output.GetIndex() >= 0)
Output.BufferFree();
if(!Output.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !Output.BufferCreate(OpenCL) : !Output.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(PrevOutput) == POINTER_INVALID)
{
PrevOutput = new CBufferDouble();
if(CheckPointer(PrevOutput) == POINTER_INVALID)
return false;
}
if(PrevOutput.GetIndex() >= 0)
PrevOutput.BufferFree();
if(!PrevOutput.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !PrevOutput.BufferCreate(OpenCL) : !PrevOutput.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(Gradient) == POINTER_INVALID)
{
Gradient = new CBufferDouble();
if(CheckPointer(Gradient) == POINTER_INVALID)
return false;
}
if(Gradient.GetIndex() >= 0)
Gradient.BufferFree();
if(!Gradient.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !Gradient.BufferCreate(OpenCL) : !Gradient.BufferCreate(DirectML))
return false;
//---
if(FileReadInteger(file_handle) == 0)
return true;
//---
if(CheckPointer(Weights) == POINTER_INVALID)
{
Weights = new CBufferDouble();
if(CheckPointer(Weights) == POINTER_INVALID)
return false;
}
if(Weights.GetIndex() >= 0)
Weights.BufferFree();
if(!Weights.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !Weights.BufferCreate(OpenCL) : !Weights.BufferCreate(DirectML))
return false;
//---
if(optimization == SGD)
{
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
{
DeltaWeights = new CBufferDouble();
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
return false;
}
if(DeltaWeights.GetIndex() >= 0)
DeltaWeights.BufferFree();
if(!DeltaWeights.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeights.BufferCreate(OpenCL) : !DeltaWeights.BufferCreate(DirectML))
return false;
}
else
{
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
{
FirstMomentum = new CBufferDouble();
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
return false;
}
if(FirstMomentum.GetIndex() >= 0)
FirstMomentum.BufferFree();
if(!FirstMomentum.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentum.BufferCreate(OpenCL) : !FirstMomentum.BufferCreate(DirectML))
return false;
//---
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
{
SecondMomentum = new CBufferDouble();
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
return false;
}
if(SecondMomentum.GetIndex() >= 0)
SecondMomentum.BufferFree();
if(!SecondMomentum.Load(file_handle))
return false;
if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentum.BufferCreate(OpenCL) : !SecondMomentum.BufferCreate(DirectML))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
View git blame Copy permalink