958 lines
65 KiB
MQL5
958 lines
65 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronNet.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Connect libraries |
|
|
//+------------------------------------------------------------------+
|
|
#include "arraylayers.mqh"
|
|
#include "positionencoder.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| Class CNet |
|
|
//| Purpose: Basic dispatcher class for implementing the operation |
|
|
//| of the neural network |
|
|
//+------------------------------------------------------------------+
|
|
class CNet : public CObject
|
|
{
|
|
protected:
|
|
bool m_bTrainMode;
|
|
CArrayLayers* m_cLayers;
|
|
CMyOpenCL* m_cOpenCL;
|
|
bool m_bOpenCL;
|
|
TYPE m_dNNLoss;
|
|
int m_iLossSmoothFactor;
|
|
CPositionEncoder* m_cPositionEncoder;
|
|
bool m_bPositionEncoder;
|
|
ENUM_LOSS_FUNCTION m_eLossFunction;
|
|
VECTOR m_adLambda;
|
|
TYPE m_dLearningRate;
|
|
VECTOR m_adBeta;
|
|
|
|
public:
|
|
CNet(void);
|
|
~CNet(void);
|
|
//--- Methods for creating an object
|
|
bool Create(CArrayObj *descriptions);
|
|
bool Create(CArrayObj *descriptions, TYPE learning_rate,
|
|
TYPE beta1, TYPE beta2);
|
|
bool Create(CArrayObj *descriptions, ENUM_LOSS_FUNCTION loss_function,
|
|
TYPE lambda1, TYPE lambda2);
|
|
bool Create(CArrayObj *descriptions, TYPE learning_rate, TYPE beta1,
|
|
TYPE beta2, ENUM_LOSS_FUNCTION loss_function, TYPE lambda1,
|
|
TYPE lambda2);
|
|
//--- Implementing work with OpenCL
|
|
void UseOpenCL(bool value);
|
|
bool UseOpenCL(void) const { return(m_bOpenCL); }
|
|
bool InitOpenCL(void);
|
|
//--- Methods of working with positional encoding
|
|
void UsePositionEncoder(bool value);
|
|
bool UsePositionEncoder(void) const { return(m_bPositionEncoder); }
|
|
//--- Organization of the main algorithms for the model
|
|
bool FeedForward(const CBufferType *inputs);
|
|
bool Backpropagation(CBufferType *target);
|
|
bool UpdateWeights(uint batch_size = 1);
|
|
bool GetResults(CBufferType *&result);
|
|
void SetLearningRates(TYPE learning_rate, TYPE beta1 = defBeta1,
|
|
TYPE beta2 = defBeta2);
|
|
//--- Loss function methods
|
|
bool LossFunction(ENUM_LOSS_FUNCTION loss_function, TYPE lambda1 = defLambdaL1,
|
|
TYPE lambda2 = defLambdaL2);
|
|
ENUM_LOSS_FUNCTION LossFunction(void) const { return(m_eLossFunction);}
|
|
ENUM_LOSS_FUNCTION LossFunction(TYPE &lambda1, TYPE &lambda2);
|
|
TYPE GetRecentAverageLoss(void) const { return(m_dNNLoss); }
|
|
void LossSmoothFactor(int value) { m_iLossSmoothFactor = value;}
|
|
int LossSmoothFactor(void) const { return(m_iLossSmoothFactor);}
|
|
//--- Model operating mode control
|
|
bool TrainMode(void) const { return m_bTrainMode; }
|
|
void TrainMode(bool mode);
|
|
//--- File handling methods
|
|
virtual bool Save(string file_name = NULL);
|
|
virtual bool Save(const int file_handle);
|
|
virtual bool Load(string file_name = NULL, bool common = false);
|
|
virtual bool Load(const int file_handle);
|
|
//--- Object identification method
|
|
virtual int Type(void) const { return(defNeuronNet); }
|
|
//--- Get pointers to internal objects
|
|
virtual CBufferType *GetGradient(uint layer) const;
|
|
virtual CBufferType *GetWeights(uint layer) const;
|
|
virtual CBufferType *GetDeltaWeights(uint layer) const;
|
|
virtual int GetGPTUnits(void);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Class constructor |
|
|
//+------------------------------------------------------------------+
|
|
CNet::CNet(void) : m_bTrainMode(false),
|
|
m_bOpenCL(false),
|
|
m_bPositionEncoder(false),
|
|
m_dNNLoss(-1),
|
|
m_iLossSmoothFactor(defLossSmoothFactor),
|
|
m_dLearningRate(defLearningRate),
|
|
m_eLossFunction(LOSS_MSE)
|
|
{
|
|
m_adLambda.Init(2);
|
|
m_adBeta.Init(2);
|
|
m_adLambda[0] = defLambdaL1;
|
|
m_adLambda[1] = defLambdaL2;
|
|
m_adBeta[0] = defBeta1;
|
|
m_adBeta[1] = defBeta2;
|
|
m_cLayers = new CArrayLayers();
|
|
m_cOpenCL = new CMyOpenCL();
|
|
m_cPositionEncoder = new CPositionEncoder();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Class destructor |
|
|
//+------------------------------------------------------------------+
|
|
CNet::~CNet(void)
|
|
{
|
|
if(!!m_cLayers)
|
|
delete m_cLayers;
|
|
if(!!m_cPositionEncoder)
|
|
delete m_cPositionEncoder;
|
|
if(!!m_cOpenCL)
|
|
delete m_cOpenCL;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Class initialization method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Create(CArrayObj *descriptions)
|
|
{
|
|
//--- Control block
|
|
if(!descriptions)
|
|
return false;
|
|
//--- Check the number of layers being created
|
|
int total = descriptions.Total();
|
|
if(total < 2)
|
|
return false;
|
|
//--- Initialize OpenCL objects
|
|
if(m_bOpenCL)
|
|
m_bOpenCL = InitOpenCL();
|
|
if(!m_cLayers.SetOpencl(m_cOpenCL))
|
|
m_bOpenCL = false;
|
|
//--- Organize a loop to create neural layers
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
CLayerDescription *temp = descriptions.At(i);
|
|
if(!temp)
|
|
return false;
|
|
if(i == 0)
|
|
{
|
|
if(temp.type != defNeuronBase)
|
|
return false;
|
|
temp.window = 0;
|
|
}
|
|
else
|
|
{
|
|
CLayerDescription *prev = descriptions.At(i - 1);
|
|
if(temp.window <= 0 || temp.window > prev.count || temp.type == defNeuronBase)
|
|
{
|
|
switch(prev.type)
|
|
{
|
|
case defNeuronConv:
|
|
case defNeuronProof:
|
|
temp.window = prev.count * prev.window_out;
|
|
break;
|
|
case defNeuronAttention:
|
|
case defNeuronMHAttention:
|
|
temp.window = prev.count * prev.window;
|
|
break;
|
|
case defNeuronGPT:
|
|
temp.window = prev.window;
|
|
break;
|
|
default:
|
|
temp.window = prev.count;
|
|
break;
|
|
}
|
|
switch(temp.type)
|
|
{
|
|
case defNeuronAttention:
|
|
case defNeuronMHAttention:
|
|
case defNeuronGPT:
|
|
break;
|
|
default:
|
|
temp.step = 0;
|
|
}
|
|
}
|
|
}
|
|
if(!m_cLayers.CreateElement(i, temp))
|
|
return false;
|
|
}
|
|
//--- Initialize positional encoding objects
|
|
if(m_bPositionEncoder)
|
|
{
|
|
if(!m_cPositionEncoder)
|
|
{
|
|
m_cPositionEncoder = new CPositionEncoder();
|
|
if(!m_cPositionEncoder)
|
|
m_bPositionEncoder = false;
|
|
return true;
|
|
}
|
|
CLayerDescription *temp = descriptions.At(0);
|
|
if(!m_cPositionEncoder.InitEncoder(temp.count, temp.window))
|
|
UsePositionEncoder(false);
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Class initialization method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Create(CArrayObj *descriptions,
|
|
TYPE learning_rate,
|
|
TYPE beta1, TYPE beta2,
|
|
ENUM_LOSS_FUNCTION loss_function, TYPE lambda1, TYPE lambda2)
|
|
{
|
|
if(!Create(descriptions))
|
|
return false;
|
|
SetLearningRates(learning_rate, beta1, beta2);
|
|
if(!LossFunction(loss_function, lambda1, lambda2))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Class initialization method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Create(CArrayObj *descriptions, ENUM_LOSS_FUNCTION loss_function, TYPE lambda1, TYPE lambda2)
|
|
{
|
|
if(!Create(descriptions))
|
|
return false;
|
|
if(!LossFunction(loss_function, lambda1, lambda2))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Class initialization method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Create(CArrayObj *descriptions,
|
|
TYPE learning_rate,
|
|
TYPE beta1, TYPE beta2)
|
|
{
|
|
if(!Create(descriptions))
|
|
return false;
|
|
SetLearningRates(learning_rate, beta1, beta2);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Feed-forward method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::FeedForward(const CBufferType *inputs)
|
|
{
|
|
//--- Control block
|
|
if(!inputs)
|
|
return false;
|
|
CNeuronBase *InputLayer = m_cLayers.At(0);
|
|
if(!InputLayer)
|
|
return false;
|
|
CBufferType *Inputs = InputLayer.GetOutputs();
|
|
if(!Inputs)
|
|
return false;
|
|
if(Inputs.Total() != inputs.Total())
|
|
return false;
|
|
//--- Transfer the source data to the neural layer
|
|
Inputs.m_mMatrix = inputs.m_mMatrix;
|
|
//--- Apply positional encoding
|
|
if(m_bPositionEncoder && !m_cPositionEncoder.AddEncoder(Inputs))
|
|
return false;
|
|
if(m_bOpenCL)
|
|
Inputs.BufferCreate(m_cOpenCL);
|
|
//--- Organize a loop through all neural layers
|
|
//--- and calling the feed-forward method for each of them
|
|
CNeuronBase *PrevLayer = InputLayer;
|
|
int total = m_cLayers.Total();
|
|
for(int i = 1; i < total; i++)
|
|
{
|
|
CNeuronBase *Layer = m_cLayers.At(i);
|
|
if(!Layer)
|
|
{
|
|
PrintFormat("%s - %d Layer %d", __FUNCTION__, __LINE__, i);
|
|
return false;
|
|
}
|
|
if(!Layer.FeedForward(PrevLayer))
|
|
{
|
|
PrintFormat("%s - %d Layer %d", __FUNCTION__, __LINE__, i);
|
|
return false;
|
|
}
|
|
PrevLayer = Layer;
|
|
}
|
|
if(m_bOpenCL)
|
|
if(!PrevLayer.GetOutputs().BufferRead())
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Backpropagation method |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Backpropagation(CBufferType *target)
|
|
{
|
|
//--- Control block
|
|
if(!target)
|
|
return false;
|
|
int total = m_cLayers.Total();
|
|
CNeuronBase *Output = m_cLayers.At(total - 1);
|
|
if(!Output || Output.Total() != target.Total())
|
|
return false;
|
|
//--- Calculate the value of the loss function
|
|
TYPE loss = Output.GetOutputs().m_mMatrix.Loss(target.m_mMatrix, m_eLossFunction);
|
|
if(loss == FLT_MAX || !MathIsValidNumber(loss))
|
|
{
|
|
printf("%s -> %d", __FUNCTION__, __LINE__);
|
|
Print(target.m_mMatrix);
|
|
Print(Output.GetOutputs().m_mMatrix);
|
|
return false;
|
|
}
|
|
m_dNNLoss = (m_dNNLoss < 0 ? loss : m_dNNLoss + (loss - m_dNNLoss) / m_iLossSmoothFactor);
|
|
//--- Calculate error gradient at the output of the neural network
|
|
CBufferType* grad = Output.GetGradients();
|
|
grad.m_mMatrix = target.m_mMatrix;
|
|
if(m_cOpenCL)
|
|
{
|
|
if(!grad.BufferWrite())
|
|
return false;
|
|
}
|
|
if(!Output.CalcOutputGradient(grad, m_eLossFunction))
|
|
return false;
|
|
//--- Run a loop iterating through all neural layers in reverser order
|
|
for(int i = total - 2; i >= 0; i--)
|
|
{
|
|
CNeuronBase *temp = m_cLayers.At(i);
|
|
if(!temp)
|
|
return false;
|
|
//--- Call method propagating the error gradient through the hidden layer
|
|
if(!Output.CalcHiddenGradient(temp))
|
|
return false;
|
|
//--- Call method propagating the error gradient to the weight matrix
|
|
if(!Output.CalcDeltaWeights(temp, i == 0))
|
|
return false;
|
|
Output = temp;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for updating weight matrices |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::UpdateWeights(uint batch_size = 1)
|
|
{
|
|
//--- Control block
|
|
if(batch_size <= 0)
|
|
return false;
|
|
//--- Organize a loop through all hidden layers
|
|
int total = m_cLayers.Total();
|
|
for(int i = 1; i < total; i++)
|
|
{
|
|
//--- Check the validity of pointer to the neural layer object
|
|
CNeuronBase *temp = m_cLayers.At(i);
|
|
if(!temp)
|
|
return false;
|
|
//--- Call the method for updating the weight matrix of the inner layer
|
|
if(!temp.UpdateWeights(batch_size, m_dLearningRate, m_adBeta, m_adLambda))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for obtaining feed-forward results |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::GetResults(CBufferType *&result)
|
|
{
|
|
int total = m_cLayers.Total();
|
|
CNeuronBase *temp = m_cLayers.At(total - 1);
|
|
if(!temp)
|
|
return false;
|
|
CBufferType *output = temp.GetOutputs();
|
|
if(!output)
|
|
return false;
|
|
if(!result)
|
|
{
|
|
result = new CBufferType();
|
|
if(!result)
|
|
return false;
|
|
}
|
|
if(m_cOpenCL)
|
|
if(!output.BufferRead())
|
|
return false;
|
|
result.m_mMatrix = output.m_mMatrix;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for saving class elements to a file |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Save(string file_name = NULL)
|
|
{
|
|
//--- Control block
|
|
if(file_name == NULL || file_name == "")
|
|
file_name = defFileName;
|
|
//--- Open file for writing
|
|
int handle = FileOpen(file_name, FILE_WRITE | FILE_BIN);
|
|
//--- Call method for saving the class using the file handle
|
|
bool result = Save(handle);
|
|
//--- Close the open file
|
|
FileClose(handle);
|
|
//---
|
|
return result;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for saving class elements to a file |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Save(const int file_handle)
|
|
{
|
|
//--- Control block
|
|
if(file_handle == INVALID_HANDLE || !m_cLayers)
|
|
return false;
|
|
//--- Save constants
|
|
if(!FileWriteInteger(file_handle, (int)m_bOpenCL) ||
|
|
!FileWriteDouble(file_handle, m_dNNLoss) ||
|
|
!FileWriteInteger(file_handle, m_iLossSmoothFactor) ||
|
|
!FileWriteInteger(file_handle, (int)m_bPositionEncoder) ||
|
|
!FileWriteDouble(file_handle, (double)m_dLearningRate) ||
|
|
!FileWriteDouble(file_handle, (double)m_adBeta[0]) ||
|
|
!FileWriteDouble(file_handle, (double)m_adBeta[1]) ||
|
|
!FileWriteDouble(file_handle, (double)m_adLambda[0]) ||
|
|
!FileWriteDouble(file_handle, (double)m_adLambda[1]) ||
|
|
!FileWriteInteger(file_handle, (int)m_eLossFunction))
|
|
return false;
|
|
//--- Save the positional encoding object if necessary
|
|
if(m_bPositionEncoder)
|
|
{
|
|
if(!m_cPositionEncoder ||
|
|
!m_cPositionEncoder.Save(file_handle))
|
|
return false;
|
|
}
|
|
//--- Call the method for saving data of a dynamic array of neural layers
|
|
return m_cLayers.Save(file_handle);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for restoring the class from saved data |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Load(string file_name = NULL, bool common = false)
|
|
{
|
|
//--- Control block
|
|
if(!FileIsExist(file_name, (common ? FILE_COMMON : 0)))
|
|
file_name = defFileName;
|
|
//--- Open the file and call the method for loading data using the file handle
|
|
int handle = FileOpen(file_name, FILE_READ | FILE_BIN | FILE_SHARE_READ | (common ? FILE_COMMON : 0));
|
|
bool result = Load(handle);
|
|
FileClose(handle);
|
|
//---
|
|
return result;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for restoring the class from saved data |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::Load(const int file_handle)
|
|
{
|
|
//--- Control block
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
//--- Read constants
|
|
m_bOpenCL = (bool)FileReadInteger(file_handle);
|
|
m_dNNLoss = (TYPE)FileReadDouble(file_handle);
|
|
m_iLossSmoothFactor = FileReadInteger(file_handle);
|
|
m_bPositionEncoder = (bool)FileReadInteger(file_handle);
|
|
m_dLearningRate = (TYPE)FileReadDouble(file_handle);
|
|
m_adBeta[0] = (TYPE)FileReadDouble(file_handle);
|
|
m_adBeta[1] = (TYPE)FileReadDouble(file_handle);
|
|
m_adLambda[0] = (TYPE)FileReadDouble(file_handle);
|
|
m_adLambda[1] = (TYPE)FileReadDouble(file_handle);
|
|
ENUM_LOSS_FUNCTION loss = (ENUM_LOSS_FUNCTION) FileReadInteger(file_handle);
|
|
if(!LossFunction(loss, defLambdaL1, defLambdaL2))
|
|
{
|
|
Print("Error, unknown loss function: ", EnumToString(loss));
|
|
return false;
|
|
}
|
|
//--- Load a positional encoding object
|
|
if(m_bPositionEncoder)
|
|
{
|
|
if(!m_cPositionEncoder)
|
|
{
|
|
m_cPositionEncoder = new CPositionEncoder();
|
|
if(!m_cPositionEncoder)
|
|
return false;
|
|
}
|
|
if(!m_cPositionEncoder.Load(file_handle))
|
|
return false;
|
|
}
|
|
//--- Initialize object for working with OpenCL
|
|
if(m_bOpenCL)
|
|
{
|
|
if(!InitOpenCL())
|
|
m_bOpenCL = false;
|
|
}
|
|
else
|
|
if(!!m_cOpenCL)
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
}
|
|
//--- Initialize and load data from a dynamic array of neural layers
|
|
if(!m_cLayers)
|
|
{
|
|
m_cLayers = new CArrayLayers();
|
|
if(!m_cLayers)
|
|
return false;
|
|
}
|
|
if(m_bOpenCL)
|
|
m_cLayers.SetOpencl(m_cOpenCL);
|
|
//---
|
|
return m_cLayers.Load(file_handle);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for initializing objects for working with OpenCL |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::InitOpenCL(void)
|
|
{
|
|
//--- Delete previously created OpenCL objects
|
|
if(!!m_cOpenCL)
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
}
|
|
//--- Create a new object to work with OpenCL
|
|
m_cOpenCL = new CMyOpenCL();
|
|
if(!m_cOpenCL)
|
|
return false;
|
|
//--- Initialize object for working with OpenCL
|
|
if(!m_cOpenCL.Initialize(cl_program, true))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.SetKernelsCount(43))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.SetBuffersCount(10))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
//--- Initialize OpenCL kernels
|
|
if(!m_cOpenCL.KernelCreate(def_k_PerceptronFeedForward, "PerceptronFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_CalcOutputGradient, "CalcOutputGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_CalcHiddenGradient, "CalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_CalcDeltaWeights, "CalcDeltaWeights"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SGDUpdate, "SGDUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_MomentumUpdate, "MomentumUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AdaGradUpdate, "AdaGradUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_RMSPropUpdate, "RMSPropUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AdaDeltaUpdate, "AdaDeltaUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AdamUpdate, "AdamUpdate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_ProofFeedForward, "ProofFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_ProofHiddenGradients, "ProofCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_ConvolutionFeedForward, "ConvolutionFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_ConvolutionHiddenGradients, "ConvolutionCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_ConvolutionDeltaWeights, "ConvolutionCalcDeltaWeights"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LSTMFeedForward, "LSTMFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LSTMHiddenGradients, "LSTMCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AttentionFeedForward, "AttentionFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AttentionScoreGradients, "AttentionCalcScoreGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_AttentionHiddenGradients, "AttentionCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_GPTFeedForward, "GPTFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_GPTScoreGradients, "GPTCalcScoreGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_GPTHiddenGradients, "GPTCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_BatchNormFeedForward, "BatchNormFeedForward"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_BatchNormCalcHiddenGradient, "BatchNormCalcHiddenGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_BatchNormCalcDeltaWeights, "BatchNormCalcDeltaWeights"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_MaskMult, "MaskMult"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_Sum, "Sum"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LayerNormalize, "LayerNormalize"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LayerNormalizeGradient, "LayerNormalizeGradient"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LineActivation, "LineActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SigmoidActivation, "SigmoidActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SigmoidDerivative, "SigmoidDerivative"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_TANHActivation, "TanhActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_TANHDerivative, "TanhDerivative"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LReLuActivation, "LReLUActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_LReLuDerivative, "LReLUDerivative"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SoftMAXActivation, "SoftMaxActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SoftMAXDerivative, "SoftMaxDerivative"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SwishActivation, "SwishActivation"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_SwishDerivative, "SwishDerivative"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_Split, "Split"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
if(!m_cOpenCL.KernelCreate(def_k_Concatenate, "Concatenate"))
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for passing a pointer to the OpenCL object to all |
|
|
//| internal objects |
|
|
//+------------------------------------------------------------------+
|
|
void CNet::UseOpenCL(bool value)
|
|
{
|
|
if(!value)
|
|
{
|
|
if(!m_cOpenCL)
|
|
{
|
|
m_bOpenCL = value;
|
|
return;
|
|
}
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
if(!!m_cLayers)
|
|
m_cLayers.SetOpencl(m_cOpenCL);
|
|
m_bOpenCL = value;
|
|
return;
|
|
}
|
|
//---
|
|
if(!!m_cOpenCL)
|
|
{
|
|
m_cOpenCL.Shutdown();
|
|
delete m_cOpenCL;
|
|
}
|
|
m_bOpenCL = InitOpenCL();
|
|
if(!!m_cLayers)
|
|
m_cLayers.SetOpencl(m_cOpenCL);
|
|
return;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for setting learning parameters |
|
|
//+------------------------------------------------------------------+
|
|
void CNet::SetLearningRates(TYPE learning_rate, TYPE beta1 = defBeta1, TYPE beta2 = defBeta2)
|
|
{
|
|
m_dLearningRate = learning_rate;
|
|
m_adBeta[0] = beta1;
|
|
m_adBeta[1] = beta2;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for setting the loss function |
|
|
//+------------------------------------------------------------------+
|
|
bool CNet::LossFunction(ENUM_LOSS_FUNCTION loss_function, TYPE lambda1 = 0.000000, TYPE lambda2 = 0.000000)
|
|
{
|
|
//--- save the parameters of the loss function
|
|
m_eLossFunction = loss_function;
|
|
m_adLambda[0] = lambda1;
|
|
m_adLambda[1] = lambda2;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for getting a pointer to gradient buffer by layer number |
|
|
//+------------------------------------------------------------------+
|
|
CBufferType *CNet::GetGradient(uint layer) const
|
|
{
|
|
if(layer >= (uint)m_cLayers.Total())
|
|
return NULL;
|
|
//---
|
|
CNeuronBase *l = m_cLayers.At(layer);
|
|
return l.GetGradients();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for getting a pointer to weight matrix by layer number |
|
|
//+------------------------------------------------------------------+
|
|
CBufferType *CNet::GetWeights(uint layer) const
|
|
{
|
|
if(layer >= (uint)m_cLayers.Total())
|
|
return NULL;
|
|
//---
|
|
CNeuronBase *l = m_cLayers.At(layer);
|
|
return l.GetWeights();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for getting a pointer to buffer of accumulated error |
|
|
//| gradients at the weight matrix level by layer number |
|
|
//+------------------------------------------------------------------+
|
|
CBufferType *CNet::GetDeltaWeights(uint layer)const
|
|
{
|
|
if(layer >= (uint)m_cLayers.Total())
|
|
return NULL;
|
|
//---
|
|
CNeuronBase *l = m_cLayers.At(layer);
|
|
return l.GetDeltaWeights();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Set model running mode |
|
|
//+------------------------------------------------------------------+
|
|
void CNet::TrainMode(bool mode)
|
|
{
|
|
m_bTrainMode = mode;
|
|
int total = m_cLayers.Total();
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
if(!m_cLayers.At(i))
|
|
continue;
|
|
CNeuronBase *temp = m_cLayers.At(i);
|
|
temp.TrainMode(mode);
|
|
}
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Method for getting the depth of GPT blocks used |
|
|
//+------------------------------------------------------------------+
|
|
//int CNet::GetGPTUnits(void)
|
|
// {
|
|
// int result = 0;
|
|
// if(CheckPointer(m_cLayers) == POINTER_INVALID)
|
|
// return result;
|
|
// int total = m_cLayers.Total();
|
|
// for(int i = 0; i < total; i++)
|
|
// {
|
|
// if(CheckPointer(m_cLayers.At(i)) == POINTER_INVALID)
|
|
// continue;
|
|
// if(m_cLayers.At(i).Type() == defNeuronGPT)
|
|
// {
|
|
// CNeuronGPT *temp = m_cLayers.At(i);
|
|
// result += temp.GetUnits() * temp.GetLayers();
|
|
// }
|
|
// if(m_cLayers.At(i).Type() == defNeuronLSTM)
|
|
// {
|
|
// CNeuronLSTM *temp = m_cLayers.At(i);
|
|
// result += temp.GetDepth();
|
|
// }
|
|
// }
|
|
////---
|
|
// return result;
|
|
// }
|
|
//+------------------------------------------------------------------+
|
|
//| Method for setting the positional encoding usage flag |
|
|
//+------------------------------------------------------------------+
|
|
void CNet::UsePositionEncoder(bool value)
|
|
{
|
|
m_bPositionEncoder = value;
|
|
if(!m_bPositionEncoder)
|
|
{
|
|
if(!!m_cPositionEncoder)
|
|
delete m_cPositionEncoder;
|
|
return;
|
|
}
|
|
//---
|
|
if(!m_cPositionEncoder)
|
|
m_cPositionEncoder = new CPositionEncoder();
|
|
if(!m_cLayers || m_cLayers.Total() < 1)
|
|
return;
|
|
CNeuronBase *temp = m_cLayers.At(0);
|
|
if(!temp)
|
|
return;
|
|
if(!m_cPositionEncoder.InitEncoder(1, temp.GetOutputs().Total()))
|
|
UsePositionEncoder(false);
|
|
//---
|
|
return;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
|
|
//+------------------------------------------------------------------+
|