forked from animatedread/Warrior_EA
e4a154aa5e
AI/Network.mqh was 5,805 lines / 18 classes in one file. Investigation found method implementations for several classes (CNeuronBase/Pool/Conv, CNeuronBaseOCL) hand-interleaved across thousands of lines - not safe to split without risky manual reassembly. But 8 classes turned out to be genuinely self-contained (declaration + every method body physically contiguous, and only ever depended upon, never depending on anything declared later): CConnection/CArrayCon, CNeuron, CDirectMLMy (+ its WarriorDML.dll/WarriorCPU.dll #import blocks), CArrayLayer, CLayerDescription, CBufferDouble, and CNeuronConvOCL/CNeuronPoolOCL. Extracted each verbatim, via exact line-range extraction (not manual retyping) to eliminate transcription risk, into its own AI/*.mqh file, included from Network.mqh at the exact point each class used to sit - preserving original declaration order exactly. Mathematically verified byte-for-byte: reconstructing the original file from the 7 new files' bodies + Network.mqh's remaining segments is line-for-line identical to the pre-split git history. Compiled clean (MetaEditor, 0 errors/0 warnings) both before and after. The remaining tangled classes (CNeuronBase, CNeuronPool, CNeuronConv, CNet, CNeuronLSTM, CNeuronBaseOCL, CNeuronConvOCL/PoolOCL's shared base, CNeuronLSTMOCL, COpenCLMy) stay in Network.mqh (now ~4,450 lines) - splitting those safely needs deliberate per-method surgery, deferred to a future dedicated pass rather than rushed into this one. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
642 lines
29 KiB
MQL5
642 lines
29 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronOCLConvPool.mqh |
|
|
//| AnimateDread |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
//| CNeuronConvOCL/CNeuronPoolOCL - the GPU-accelerated (OpenCL + |
|
|
//| DirectML) convolution and max-pooling layers. Both derive from |
|
|
//| CNeuronBaseOCL (AI\Network.mqh, must already be declared) and use|
|
|
//| CBufferDouble (AI\BufferDouble.mqh). Included from Network.mqh at|
|
|
//| the exact point these classes used to sit (right after |
|
|
//| CNeuronBaseOCL's own declaration, before CNeuronLSTMOCL), so |
|
|
//| ordering matches the original file. Extracted verbatim (SOLID |
|
|
//| cleanup) - no logic changes. |
|
|
//+------------------------------------------------------------------+
|
|
#include "BufferDouble.mqh"
|
|
//| GPU-accelerated convolution layer (OpenCL + DirectML). Ported |
|
|
//| from the NeuroNet_DNG reference library's CNeuronConvOCL/ |
|
|
//| CNeuronProofOCL, adapted to this project's double-precision |
|
|
//| CBufferDouble/COpenCLMy/CDirectMLMy conventions. A single |
|
|
//| (window+1)*window_out weight block is shared across every |
|
|
//| sliding position - unlike CNeuronBaseOCL, where every output has |
|
|
//| its own private weight vector. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronConvOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
uint iWindow;
|
|
uint iStep;
|
|
uint iWindowOut;
|
|
CBufferDouble *WeightsConv;
|
|
CBufferDouble *DeltaWeightsConv;
|
|
CBufferDouble *FirstMomentumConv;
|
|
CBufferDouble *SecondMomentumConv;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
|
|
public:
|
|
CNeuronConvOCL(void) : iWindow(1), iStep(1), iWindowOut(1)
|
|
{
|
|
WeightsConv = NULL;
|
|
DeltaWeightsConv = NULL;
|
|
FirstMomentumConv = NULL;
|
|
SecondMomentumConv = NULL;
|
|
}
|
|
~CNeuronConvOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronConvOCL; }
|
|
// See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsConv instead of the
|
|
// base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
|
|
virtual int getWeightsConv(double &values[]) { return (CheckPointer(WeightsConv) == POINTER_INVALID ? 0 : WeightsConv.GetData(values)); }
|
|
virtual bool setWeightsConv(double &values[])
|
|
{
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
if(!WeightsConv.AssignArray(values))
|
|
return false;
|
|
return WeightsConv.BufferWrite();
|
|
}
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronConvOCL::~CNeuronConvOCL(void)
|
|
{
|
|
if(CheckPointer(WeightsConv) != POINTER_INVALID)
|
|
delete WeightsConv;
|
|
if(CheckPointer(DeltaWeightsConv) != POINTER_INVALID)
|
|
delete DeltaWeightsConv;
|
|
if(CheckPointer(FirstMomentumConv) != POINTER_INVALID)
|
|
delete FirstMomentumConv;
|
|
if(CheckPointer(SecondMomentumConv) != POINTER_INVALID)
|
|
delete SecondMomentumConv;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(window_out <= 0)
|
|
return false;
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window_out, optimization_type))
|
|
return false;
|
|
//---
|
|
iWindow = window_in;
|
|
iStep = step;
|
|
iWindowOut = (uint)fmax(window_out, 1);
|
|
//---
|
|
int count = (int)((iWindow + 1) * iWindowOut);
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
{
|
|
WeightsConv = new CBufferDouble();
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!WeightsConv.Reserve(count))
|
|
return false;
|
|
// Fan-in-scaled (LeCun-uniform) init - see CNeuronBaseOCL::Init's OpenCL overload for the full
|
|
// rationale; fan-in here is the conv window size.
|
|
double weighScale = 1.0 / MathSqrt((double)iWindow + 1.0);
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!WeightsConv.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!WeightsConv.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
{
|
|
DeltaWeightsConv = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeightsConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeightsConv.BufferCreate(OpenCL))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
{
|
|
FirstMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentumConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentumConv.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
{
|
|
SecondMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentumConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentumConv.BufferCreate(OpenCL))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(window_out <= 0)
|
|
return false;
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, units_count * window_out, optimization_type))
|
|
return false;
|
|
//---
|
|
iWindow = window_in;
|
|
iStep = step;
|
|
iWindowOut = (uint)fmax(window_out, 1);
|
|
//---
|
|
int count = (int)((iWindow + 1) * iWindowOut);
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
{
|
|
WeightsConv = new CBufferDouble();
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!WeightsConv.Reserve(count))
|
|
return false;
|
|
// Fan-in-scaled (LeCun-uniform) init - see the matching OpenCL Init() overload above.
|
|
double weighScale = 1.0 / MathSqrt((double)iWindow + 1.0);
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!WeightsConv.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!WeightsConv.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
{
|
|
DeltaWeightsConv = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeightsConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeightsConv.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
{
|
|
FirstMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentumConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentumConv.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
{
|
|
SecondMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentumConv.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentumConv.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
int positions = Output.Total() / (int)iWindowOut;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.FeedForwardConv(WeightsConv.GetIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
|
|
NeuronOCL.Neurons(), (int)iStep, (int)iWindow, (int)iWindowOut, NativeActivationCode(activation), positions))
|
|
{
|
|
printf("Error of execution DirectML FeedForwardConv");
|
|
return false;
|
|
}
|
|
return Output.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = (uint)positions;
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_w, WeightsConv.GetIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_o, Output.GetIndex());
|
|
OpenCL.SetArgument(def_k_FeedForwardConv, def_k_ffc_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_FeedForwardConv, def_k_ffc_step, (int)iStep);
|
|
OpenCL.SetArgument(def_k_FeedForwardConv, def_k_ffc_window_in, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_FeedForwardConv, def_k_ffc_window_out, (int)iWindowOut);
|
|
OpenCL.SetArgument(def_k_FeedForwardConv, def_k_ffc_activation, NativeActivationCode(activation));
|
|
if(!OpenCL.Execute(def_k_FeedForwardConv, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel FeedForwardConv: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Output stays GPU-resident; see the note in CNeuronBaseOCL::feedForward().
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Writes the gradient into NeuronOCL (the EARLIER/input-side layer) |
|
|
//| - opposite call direction from the dense calcHiddenGradients, but |
|
|
//| matches the reference library's own CNeuronConvOCL convention. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
int outputs = Neurons();
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.CalcHiddenGradientConv(WeightsConv.GetIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getGradientIndex(),
|
|
outputs, (int)iStep, (int)iWindow, (int)iWindowOut, (int)NeuronOCL.Activation(), NeuronOCL.Neurons()))
|
|
{
|
|
printf("Error of execution DirectML CalcHiddenGradientConv");
|
|
return false;
|
|
}
|
|
double temp[];
|
|
return NeuronOCL.getGradient(temp) > 0;
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = NeuronOCL.Neurons();
|
|
OpenCL.SetArgumentBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, WeightsConv.GetIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, outputs);
|
|
OpenCL.SetArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)iStep);
|
|
OpenCL.SetArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)iWindowOut);
|
|
OpenCL.SetArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)NeuronOCL.Activation());
|
|
if(!OpenCL.Execute(def_k_CalcHiddenGradientConv, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel CalcHiddenGradientConv: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- NeuronOCL's Gradient stays GPU-resident; its own calcHiddenGradients/calcInputGradients
|
|
//--- reads it via getGradientIndex(). The old getGradient(temp) call was a discarded-result
|
|
//--- sync (GetData() BufferRead()s internally) with no consumer of the read - pure overhead.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
int inputs = NeuronOCL.Neurons();
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(optimization == SGD)
|
|
{
|
|
if(!DirectML.UpdateWeightsConvMomentum(WeightsConv.GetIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), DeltaWeightsConv.GetIndex(),
|
|
inputs, eta, alpha, (int)iWindow, (int)iWindowOut, (int)iStep))
|
|
{
|
|
printf("Error of execution DirectML UpdateWeightsConvMomentum");
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!DirectML.UpdateWeightsConvAdam(WeightsConv.GetIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
|
|
FirstMomentumConv.GetIndex(), SecondMomentumConv.GetIndex(), inputs, lt, b1, b2, (int)iWindow, (int)iWindowOut, (int)iStep))
|
|
{
|
|
printf("Error of execution DirectML UpdateWeightsConvAdam");
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
return WeightsConv.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
if(optimization == SGD)
|
|
{
|
|
global_work_size[0] = WeightsConv.Total();
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_w, WeightsConv.GetIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_dw, DeltaWeightsConv.GetIndex());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_inputs, inputs);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_learning_rates, (float)eta);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_momentum, (float)alpha);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_in, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_out, (int)iWindowOut);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_step, (int)iStep);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsConvMomentum, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsConvMomentum: %d", GetLastError());
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
global_work_size[0] = iWindow + 1;
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_w, WeightsConv.GetIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_inputs, inputs);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_l, (float)lt);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b1, (float)b1);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b2, (float)b2);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_in, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_out, (int)iWindowOut);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_step, (int)iStep);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsConvAdam, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsConvAdam: %d", GetLastError());
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
//--- WeightsConv stays GPU-resident; see the note in CNeuronBaseOCL::updateInputWeights().
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)iWindow, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)iStep, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)iWindowOut, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID || !WeightsConv.BufferRead() || !WeightsConv.Save(file_handle))
|
|
return false;
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID || !DeltaWeightsConv.BufferRead() || !DeltaWeightsConv.Save(file_handle))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID || !FirstMomentumConv.BufferRead() || !FirstMomentumConv.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID || !SecondMomentumConv.BufferRead() || !SecondMomentumConv.Save(file_handle))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronConvOCL::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
return false;
|
|
iWindow = (uint)FileReadInteger(file_handle, INT_VALUE);
|
|
iStep = (uint)FileReadInteger(file_handle, INT_VALUE);
|
|
iWindowOut = (uint)FileReadInteger(file_handle, INT_VALUE);
|
|
//---
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
{
|
|
WeightsConv = new CBufferDouble();
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(WeightsConv.GetIndex() >= 0)
|
|
WeightsConv.BufferFree();
|
|
if(!WeightsConv.Load(file_handle))
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID ? !WeightsConv.BufferCreate(OpenCL) : !WeightsConv.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
{
|
|
DeltaWeightsConv = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(DeltaWeightsConv.GetIndex() >= 0)
|
|
DeltaWeightsConv.BufferFree();
|
|
if(!DeltaWeightsConv.Load(file_handle))
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID ? !DeltaWeightsConv.BufferCreate(OpenCL) : !DeltaWeightsConv.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
{
|
|
FirstMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(FirstMomentumConv.GetIndex() >= 0)
|
|
FirstMomentumConv.BufferFree();
|
|
if(!FirstMomentumConv.Load(file_handle))
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID ? !FirstMomentumConv.BufferCreate(OpenCL) : !FirstMomentumConv.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
{
|
|
SecondMomentumConv = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentumConv) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(SecondMomentumConv.GetIndex() >= 0)
|
|
SecondMomentumConv.BufferFree();
|
|
if(!SecondMomentumConv.Load(file_handle))
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID ? !SecondMomentumConv.BufferCreate(OpenCL) : !SecondMomentumConv.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| GPU-accelerated max-pooling layer (OpenCL + DirectML). No weights,|
|
|
//| so no updateInputWeights work - just a sliding max. Ported from |
|
|
//| the NeuroNet_DNG reference's CNeuronProofOCL kernels. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronPoolOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
uint iWindow;
|
|
uint iStep;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; }
|
|
|
|
public:
|
|
CNeuronPoolOCL(void) : iWindow(2), iStep(1) {}
|
|
~CNeuronPoolOCL(void) {}
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronPoolOCL; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count, optimization_type))
|
|
return false;
|
|
iWindow = window;
|
|
iStep = step;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, units_count, optimization_type))
|
|
return false;
|
|
iWindow = window;
|
|
iStep = step;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
int outputs = Output.Total();
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.FeedForwardProof(NeuronOCL.getOutputIndex(), Output.GetIndex(), NeuronOCL.Neurons(), (int)iWindow, (int)iStep, outputs))
|
|
{
|
|
printf("Error of execution DirectML FeedForwardProof");
|
|
return false;
|
|
}
|
|
return Output.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint offset1[1] = {0};
|
|
uint size1[1] = {(uint)outputs};
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForwardProof, def_k_ffp_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForwardProof, def_k_ffp_matrix_o, Output.GetIndex());
|
|
OpenCL.SetArgument(def_k_FeedForwardProof, def_k_ffp_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_FeedForwardProof, def_k_ffp_window, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_FeedForwardProof, def_k_ffp_step, (int)iStep);
|
|
if(!OpenCL.Execute(def_k_FeedForwardProof, 1, offset1, size1))
|
|
{
|
|
printf("Error of execution kernel FeedForwardProof: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Output stays GPU-resident; see the note in CNeuronBaseOCL::feedForward().
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Inverted-call convention, same as Conv/LSTM's calcInputGradients. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
int outputs = Neurons();
|
|
int inputs = NeuronOCL.Neurons();
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.CalcInputGradientProof(NeuronOCL.getOutputIndex(), getGradientIndex(), getOutputIndex(), NeuronOCL.getGradientIndex(),
|
|
outputs, (int)iWindow, (int)iStep, inputs))
|
|
{
|
|
printf("Error of execution DirectML CalcInputGradientProof");
|
|
return false;
|
|
}
|
|
double temp[];
|
|
return NeuronOCL.getGradient(temp) > 0;
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint offset1[1] = {0};
|
|
uint size1[1] = {(uint)inputs};
|
|
OpenCL.SetArgumentBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_o, getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_ig, NeuronOCL.getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CalcInputGradientProof, def_k_cigp_outputs, outputs);
|
|
OpenCL.SetArgument(def_k_CalcInputGradientProof, def_k_cigp_window, (int)iWindow);
|
|
OpenCL.SetArgument(def_k_CalcInputGradientProof, def_k_cigp_step, (int)iStep);
|
|
if(!OpenCL.Execute(def_k_CalcInputGradientProof, 1, offset1, size1))
|
|
{
|
|
printf("Error of execution kernel CalcInputGradientProof: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- NeuronOCL's Gradient stays GPU-resident; see the note in
|
|
//--- CNeuronConvOCL::calcInputGradients().
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::Save(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Save(file_handle))
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)iWindow, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)iStep, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronPoolOCL::Load(const int file_handle)
|
|
{
|
|
if(!CNeuronBaseOCL::Load(file_handle))
|
|
return false;
|
|
iWindow = (uint)FileReadInteger(file_handle, INT_VALUE);
|
|
iStep = (uint)FileReadInteger(file_handle, INT_VALUE);
|
|
return true;
|
|
}
|