dng/NeuroNetworksBook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
NeuroNetworksBook/Include/realization/neuronbase.mqh

1043 lines
84 KiB
MQL5
Raw Permalink Normal View History

convert
2025-05-30 16:12:34 +02:00
<EFBFBD><EFBFBD>//+------------------------------------------------------------------+
//| NeuronBase.mqh |
//| Copyright 2021, MetaQuotes Ltd. |
//| https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link "https://www.mql5.com"
//+------------------------------------------------------------------+
//| >4:;NG05< 181;8>B5:8 |
//+------------------------------------------------------------------+
#include <Math\Stat\Normal.mqh>
#include "bufferdouble.mqh"
#include "layerdescription.mqh"
#include "activation.mqh"
//+------------------------------------------------------------------+
//| Class CNeuronBase |
//| 07=0G5=85: 07>2K9 :;0AA ?>;=>A2O7=>3> =59@>==>3> A;>O |
//+------------------------------------------------------------------+
class CNeuronBase : public CObject
{
protected:
bool m_bTrain;
CMyOpenCL *m_cOpenCL;
CActivation *m_cActivation;
ENUM_OPTIMIZATION m_eOptimization;
CBufferDouble *m_cOutputs;
CBufferDouble *m_cWeights;
CBufferDouble *m_cDeltaWeights;
CBufferDouble *m_cGradients;
CBufferDouble *m_cMomenum[2];
//---
virtual bool SGDUpdate(int batch_size, double learningRate,
double &Lambda[]);
virtual bool MomentumUpdate(int batch_size, double learningRate,
double &Beta[], double &Lambda[]);
virtual bool AdaGradUpdate(int batch_size, double learningRate,
double &Lambda[]);
virtual bool RMSPropUpdate(int batch_size, double learningRate,
double &Beta[], double &Lambda[]);
virtual bool AdaDeltaUpdate(int batch_size,
double &Beta[], double &Lambda[]);
virtual bool AdamUpdate(int batch_size, double learningRate,
double &Beta[], double &Lambda[]);
public:
CNeuronBase(void);
~CNeuronBase(void);
//---
virtual bool Init(CLayerDescription *description);
virtual bool SetOpenCL(CMyOpenCL *opencl);
virtual bool FeedForward(CNeuronBase *prevLayer);
virtual bool CalcOutputGradient(CBufferDouble *target);
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer);
virtual bool UpdateWeights(int batch_size, double learningRate,
double &Beta[], double &Lambda[]);
//---
virtual void TrainMode(bool flag) { m_bTrain = flag; }
virtual bool TrainMode(void) const { return m_bTrain; }
//---
CBufferDouble *GetOutputs(void) const { return(m_cOutputs); }
CBufferDouble *GetGradients(void) const { return(m_cGradients); }
CBufferDouble *GetWeights(void) const { return(m_cWeights); }
CBufferDouble *GetDeltaWeights(void) const { return(m_cDeltaWeights);}
//--- methods for working with files
virtual bool Save(const int file_handle);
virtual bool Load(const int file_handle);
//--- method of identifying the object
virtual int Type(void) const { return(defNeuronBase); }
};
//+------------------------------------------------------------------+
//| >=AB@C:B>@ :;0AA0 |
//+------------------------------------------------------------------+
CNeuronBase::CNeuronBase(void) : m_eOptimization(Adam)
{
m_cOpenCL = NULL;
m_cActivation = new CActivation();
m_cOutputs = new CBufferDouble();
m_cWeights = new CBufferDouble();
m_cDeltaWeights = new CBufferDouble();
m_cGradients = new CBufferDouble();
m_cMomenum[0] = new CBufferDouble();
m_cMomenum[1] = new CBufferDouble();
}
//+------------------------------------------------------------------+
//| 5AB@C:B>@ :;0AA0 |
//+------------------------------------------------------------------+
CNeuronBase::~CNeuronBase(void)
{
if(m_cActivation)
delete m_cActivation;
if(m_cOutputs)
delete m_cOutputs;
if(m_cWeights)
delete m_cWeights;
if(m_cDeltaWeights)
delete m_cDeltaWeights;
if(m_cGradients)
delete m_cGradients;
if(m_cMomenum[0])
delete m_cMomenum[0];
if(m_cMomenum[1])
delete m_cMomenum[1];
}
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O C:070B5;O =0 >1J5:B @01>BK A :>=B5:AB>< |
//| OpenCL |
//+------------------------------------------------------------------+
bool CNeuronBase::SetOpenCL(CMyOpenCL *opencl)
{
if(!opencl)
{
if(m_cOutputs)
m_cOutputs.BufferFree();
if(m_cGradients)
m_cGradients.BufferFree();
if(m_cWeights)
m_cWeights.BufferFree();
if(m_cDeltaWeights)
m_cDeltaWeights.BufferFree();
for(int i = 0; i < 2; i++)
{
if(m_cMomenum[i])
m_cMomenum[i].BufferFree();
}
}
if(m_cOpenCL != opencl)
{
if(m_cOpenCL)
delete m_cOpenCL;
m_cOpenCL = opencl;
if(m_cOutputs)
m_cOutputs.BufferCreate(opencl);
if(m_cGradients)
m_cGradients.BufferCreate(opencl);
if(m_cWeights)
m_cWeights.BufferCreate(opencl);
if(m_cDeltaWeights)
m_cDeltaWeights.BufferCreate(opencl);
for(int i = 0; i < 2; i++)
{
if(m_cMomenum[i])
m_cMomenum[i].BufferCreate(opencl);
}
if(m_cActivation)
m_cActivation.SetOpenCL(m_cOpenCL);
}
//---
return(!!m_cOpenCL);
}
//+------------------------------------------------------------------+
//| 5B>4 8=8F80;870F88 :;0AA0 |
//+------------------------------------------------------------------+
bool CNeuronBase::Init(CLayerDescription *desc)
{
//--- ;>: :>=B@>;O 8AE>4=KE 40==KE
if(!desc || desc.type != Type() || desc.count <= 0)
return false;
//--- !>740=85 1CD5@0 @57C;LB0B>2
if(!m_cOutputs)
if(!(m_cOutputs = new CBufferDouble()))
return false;
if(!m_cOutputs.BufferInit(1, desc.count, 0))
return false;
//--- !>740=85 1CD5@0 3@0485=B>2 >H81N:8
if(!m_cGradients)
if(!(m_cGradients = new CBufferDouble()))
return false;
if(!m_cGradients.BufferInit(1, desc.count, 0))
return false;
//--- #40;5=85 =5 8A?>;L7C5<KE >1J5:B>2 4;O A;>O 8AE>4=KE 40==KE
if(desc.window <= 0)
{
if(m_cActivation)
delete m_cActivation;
if(m_cWeights)
delete m_cWeights;
if(m_cDeltaWeights)
delete m_cDeltaWeights;
if(m_cMomenum[0])
delete m_cMomenum[0];
if(m_cMomenum[1])
delete m_cMomenum[1];
if(m_cOpenCL)
if(!m_cOutputs.BufferCreate(m_cOpenCL))
return false;
m_eOptimization = desc.optimization;
return true;
}
//--- =8F80;870F8O >1J5:B0 DC=:F88 0:B820F88
if(!m_cActivation)
if(!(m_cActivation = new CActivation()))
return false;
m_cActivation.SetFunction(desc.activation,
m_cOutputs.m_mMatrix,
desc.activation_params[0],
desc.activation_params[1]);
m_cActivation.SetOpenCL(m_cOpenCL);
//--- =8F80;870F8O >1J5:B0 <0B@8FK 25A>2
if(!m_cWeights)
if(!(m_cWeights = new CBufferDouble()))
return false;
if(!m_cWeights.BufferInit(desc.count, desc.window + 1, 0))
return false;
double weights[];
double sigma = desc.activation == ACT_LReLU ?
2.0 / (double)(MathPow(1 + desc.activation_params[0], 2) * desc.window) :
1.0 / (double)desc.window;
if(!MathRandomNormal(0, MathSqrt(sigma), m_cWeights.Total(), weights))
return false;
for(uint i = 0; i < m_cWeights.Total(); i++)
if(!m_cWeights.m_mMatrix.Flat(i, weights[i]))
return false;
//--- =8F80;870F8O >1J5:B0 =0:>?;5=8O 3@0485=B>2 =0 C@>2=5 <0B@8FK 25A>2
if(!m_cDeltaWeights)
if(!(m_cDeltaWeights = new CBufferDouble()))
return false;
if(!m_cDeltaWeights.BufferInit(desc.count, desc.window + 1, 0))
return false;
//--- =8F80;870F8O >1J5:B>2 <><5=B>2
switch(desc.optimization)
{
case None:
case SGD:
for(int i = 0; i < 2; i++)
if(m_cMomenum[i])
delete m_cMomenum[i];
break;
case MOMENTUM:
case AdaGrad:
case RMSProp:
if(!m_cMomenum[0])
if(!(m_cMomenum[0] = new CBufferDouble()))
return false;
if(!m_cMomenum[0].BufferInit(desc.count, desc.window + 1, 0))
return false;
if(m_cMomenum[1])
delete m_cMomenum[1];
break;
case AdaDelta:
case Adam:
for(int i = 0; i < 2; i++)
{
if(!m_cMomenum[i])
if(!(m_cMomenum[i] = new CBufferDouble()))
return(false);
if(!m_cMomenum[i].BufferInit(desc.count, desc.window + 1, 0))
return false;
}
break;
default:
return false;
break;
}
//--- !>E@0=5=85 <5B>40 >?B8<870F88 ?0@0<5B@>2
m_eOptimization = desc.optimization;
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 ?@O<>3> ?@>E>40 |
//+------------------------------------------------------------------+
bool CNeuronBase::FeedForward(CNeuronBase *prevLayer)
{
//--- ;>: :>=B@>;59
if(!prevLayer || !m_cOutputs || !m_cWeights || !prevLayer.GetOutputs() || !m_cActivation)
return false;
CBufferDouble *input_data = prevLayer.GetOutputs();
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
uint input_total = input_data.Total();
uint output_total = m_cOutputs.Total();
if(m_cWeights.Total() < (input_total + 1)*output_total)
return false;
//---
MATRIX m = input_data.m_mMatrix;
m.Resize(m.Rows(), m.Cols() + 1);
VECTOR v;
v.Init(m.Rows());
v.Fill(1);
if(!m.Col(v, input_data.m_mMatrix.Cols()))
return false;
m_cOutputs.m_mMatrix = m.MatMul(m_cWeights.m_mMatrix.Transpose());
return m_cActivation.Activation(m_cOutputs);
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(input_data.GetIndex() < 0)
return false;
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cActivation.GetIndex() < 0)
return false;
if(m_cOutputs.GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_inputs, input_data.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_sums, m_cActivation.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_outputs, m_cOutputs.GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_inputs_total, input_data.Total()))
return false;
double params[];
ENUM_ACTIVATION function = m_cActivation.GetFunction(params);
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_activation, (int)function))
return false;
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_act_param_a, params[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_act_param_b, params[1]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
uint off_set[] = {0};
uint NDRange[] = {m_cOutputs.Total()};
if(!m_cOpenCL.Execute(def_k_PerceptronFeedForward, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cOutputs.BufferRead())
return false;
if(function != ACT_SOFTMAX)
return true;
//--- ">;L:> 4;O SoftMax, =>@<0;870F8O @57C;LB0B>2
//--- 2KG8A;5=85 >I59 AC<<K 2A5E 7=0G5=89 1CD5@0 40==KE
double summ = 0;
double array[];
int total = m_cOutputs.GetData(array);
if(total <= 0)
return false;
for(int i = 0; i < total; i++)
summ += array[i];
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_Normalize, def_norm_inputs, m_cOutputs.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_Normalize, def_norm_outputs, m_cOutputs.GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_Normalize, def_norm_inputs_total, total))
return false;
if(!m_cOpenCL.SetArgument(def_k_Normalize, def_norm_const_value, summ))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
NDRange[0] = (total + 3) / 4;
if(!m_cOpenCL.Execute(def_k_Normalize, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cOutputs.BufferRead())
return false;
return true;
}
//---
return false;
}
//+------------------------------------------------------------------+
//| 5B>4 @0AGQB0 3@0485=B0 >H81:8 A;>O @57C;LB0B>2 |
//+------------------------------------------------------------------+
bool CNeuronBase::CalcOutputGradient(CBufferDouble *target)
{
//--- ;>: :>=B@>;59
if(!target || !m_cOutputs || !m_cGradients || target.Total() < m_cOutputs.Total() || m_cGradients.Total() < m_cOutputs.Total())
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
m_cGradients.m_mMatrix = target.m_mMatrix - m_cOutputs.m_mMatrix;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(target.GetIndex() < 0)
return false;
if(m_cOutputs.GetIndex() < 0)
return false;
if(m_cGradients.GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_target, target.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_outputs, m_cOutputs.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_gradients, m_cGradients.GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_CalcOutputGradient, def_outgr_outputs_total, m_cOutputs.Total()))
return false;
//--- >AB0=>2:0 :5@=5;0 2 ;G5@54L 2K?>;=5=8O
uint NDRange[] = { (m_cOutputs.Total() + 3) / 4 };
uint off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_CalcOutputGradient, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cGradients.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O 3@0485=B0 G5@57 A:@KBK9 A;>9 |
//+------------------------------------------------------------------+
bool CNeuronBase::CalcHiddenGradient(CNeuronBase *prevLayer)
{
//--- >@@5:B8@>2:0 2E>4OI53> 3@0485=B0 =0 ?@>872>4=CN DC=:F88 0:B820F88
if(!m_cActivation.Derivative(m_cOutputs, m_cGradients))
return false;
if(!prevLayer)
return false;
//--- @>25@:0 1CD5@>2 ?@54K4CI53> A;>O
CBufferDouble *input_data = prevLayer.GetOutputs();
CBufferDouble *input_gradient = prevLayer.GetGradients();
if(!input_data || !input_gradient || input_data.Total() > input_gradient.Total())
return false;
//--- @>25@:0 A>>B25BAB28O @07<5@0 1CD5@0 8AE>4=KE 40==KE 8 <0B@8FK 25A>2
uint input_total = input_data.Total();
if(!m_cWeights || m_cWeights.m_mMatrix.Cols() != (input_total + 1))
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
MATRIX grad = m_cGradients.m_mMatrix.MatMul(m_cWeights.m_mMatrix);
grad.Resize(input_data.m_mMatrix.Rows(), input_data.m_mMatrix.Cols());
input_gradient.m_mMatrix = grad;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(input_gradient.GetIndex() < 0)
return false;
if(m_cGradients.GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_gradient_inputs, input_gradient.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_gradients, m_cGradients.GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_CalcHiddenGradient, def_hidgr_outputs_total, m_cGradients.Total()))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
uint NDRange[] = {input_total};
uint off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_CalcHiddenGradient, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!input_gradient.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=85 3@0485=B0 >H81:8 4> <0B@8FK 25A>2 |
//+------------------------------------------------------------------+
bool CNeuronBase::CalcDeltaWeights(CNeuronBase *prevLayer)
{
//--- ;>: :>=B@>;59
if(!prevLayer || !m_cDeltaWeights || !m_cGradients)
return false;
CBufferDouble *Inputs = prevLayer.GetOutputs();
if(!Inputs)
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
MATRIX m = Inputs.m_mMatrix;
m.Resize(m.Rows(), m.Cols() + 1);
VECTOR v;
v.Init(m.Rows());
v.Fill(1);
if(!m.Col(v, Inputs.m_mMatrix.Cols()))
return false;
m = m_cGradients.m_mMatrix.Transpose().MatMul(m);
m_cDeltaWeights.m_mMatrix += m;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cGradients.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(Inputs.GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_inputs, Inputs.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_gradients, m_cGradients.GetIndex()))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
uint NDRange[] = {m_cGradients.Total(), Inputs.Total()};
uint off_set[] = {0, 0};
if(!m_cOpenCL.Execute(def_k_CalcDeltaWeights, 2, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cDeltaWeights.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 >1=>2;5=8O <0B@8FK 25A>2 |
//+------------------------------------------------------------------+
bool CNeuronBase::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
{
//--- ;>: :>=B@>;59
if(!m_cDeltaWeights || !m_cWeights || m_cWeights.Total() < m_cDeltaWeights.Total() || batch_size <= 0)
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B 8A?>;L7C5<>9 DC=:F88 0:B820F88
bool result = false;
switch(m_eOptimization)
{
case None:
result = true;
break;
case SGD:
result = SGDUpdate(batch_size, learningRate, Lambda);
break;
case MOMENTUM:
result = MomentumUpdate(batch_size, learningRate, Beta, Lambda);
break;
case AdaGrad:
result = AdaGradUpdate(batch_size, learningRate, Lambda);
break;
case RMSProp:
result = RMSPropUpdate(batch_size, learningRate, Beta, Lambda);
break;
case AdaDelta:
result = AdaDeltaUpdate(batch_size, Beta, Lambda);
break;
case Adam:
result = AdamUpdate(batch_size, learningRate, Beta, Lambda);
break;
}
//---
return result;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 AB>E0AB8G5A:8< 3@0485=B=K< A?CA:>< |
//+------------------------------------------------------------------+
bool CNeuronBase::SGDUpdate(int batch_size, double learningRate, double &Lambda[])
{
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
double lr = learningRate / ((double)batch_size);
m_cWeights.m_mMatrix -= Lambda[0] + Lambda[1] * m_cWeights.m_mMatrix;
m_cWeights.m_mMatrix += m_cDeltaWeights.m_mMatrix * lr;
m_cDeltaWeights.m_mMatrix.Fill(0);
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_SGDUpdate, def_sgd_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_SGDUpdate, def_sgd_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_learningRate, learningRate))
return false;
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_Lambda2, Lambda[1]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_SGDUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
return true;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 <5B>4>< <><5=B>2 |
//+------------------------------------------------------------------+
bool CNeuronBase::MomentumUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
{
if(Beta[0] == 0)
return SGDUpdate(batch_size, learningRate, Lambda);
//--- ;>: :>=B@>;59
if(!m_cMomenum[0])
return false;
if(m_cMomenum[0].Total() < m_cWeights.Total())
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
double lr = learningRate / ((double)batch_size);
m_cWeights.m_mMatrix -= Lambda[0] + m_cWeights.m_mMatrix * Lambda[1];
m_cMomenum[0].m_mMatrix = m_cDeltaWeights.m_mMatrix * lr + m_cMomenum[0].m_mMatrix * Beta[0] ;
m_cWeights.m_mMatrix += m_cMomenum[0].m_mMatrix;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(m_cMomenum[0].GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_momentum, m_cMomenum[0].GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_learningRate, learningRate))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_Lambda2, Lambda[1]))
return false;
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_beta, Beta[0]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_MomentumUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
return true;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 <5B>4>< AdaGrad |
//+------------------------------------------------------------------+
bool CNeuronBase::AdaGradUpdate(int batch_size, double learningRate, double &Lambda[])
{
//--- ;>: :>=B@>;59
if(!m_cMomenum[0])
return false;
if(m_cMomenum[0].Total() < m_cWeights.Total())
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
double lr = learningRate;
m_cWeights.m_mMatrix -= Lambda[0] + Lambda[1] * m_cWeights.m_mMatrix;
MATRIX delta = m_cDeltaWeights.m_mMatrix / ((double)batch_size);
MATRIX G = m_cMomenum[0].m_mMatrix = m_cMomenum[0].m_mMatrix + delta.Power(2);
for(ulong r = 0; r < G.Rows(); r++)
for(ulong c = 0; c < G.Cols(); c++)
G[r, c] = (G[r, c] > 0 ? lr / MathSqrt(G[r, c]) : 0);
m_cWeights.m_mMatrix += G * delta;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(m_cMomenum[0].GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_momentum, m_cMomenum[0].GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_learningRate, learningRate))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_Lambda2, Lambda[1]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_AdaGradUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
return true;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 <5B>4>< RMSProp |
//+------------------------------------------------------------------+
bool CNeuronBase::RMSPropUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
{
//--- ;>: :>=B@>;59
if(!m_cMomenum[0])
return false;
if(m_cMomenum[0].Total() < m_cWeights.Total())
return false;
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(!m_cOpenCL)
{
double lr = learningRate;
m_cWeights.m_mMatrix -= Lambda[0] + Lambda[1] * m_cWeights.m_mMatrix;
MATRIX delta = m_cDeltaWeights.m_mMatrix / ((double)batch_size);
MATRIX G = m_cMomenum[0].m_mMatrix = Beta[0] * m_cMomenum[0].m_mMatrix + (1 - Beta[0]) * delta.Power(2);
for(ulong r = 0; r < G.Rows(); r++)
for(ulong c = 0; c < G.Cols(); c++)
G[r, c] = (G[r, c] > 0 ? lr / MathSqrt(G[r, c]) : 0);
m_cWeights.m_mMatrix += G * delta;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(m_cMomenum[0].GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_momentum, m_cMomenum[0].GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_learningRate, learningRate))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_Lambda2, Lambda[1]))
return false;
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_beta, Beta[0]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_RMSPropUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 <5B>4>< AdaDelta |
//+------------------------------------------------------------------+
bool CNeuronBase::AdaDeltaUpdate(int batch_size, double &Beta[], double &Lambda[])
{
//--- ;>: :>=B@>;59
for(int i = 0; i < 2; i++)
{
if(!m_cMomenum[i])
return false;
if(m_cMomenum[i].Total() < m_cWeights.Total())
return false;
}
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
{
MATRIX delta = m_cDeltaWeights.m_mMatrix / ((double)batch_size);
MATRIX W = m_cMomenum[0].m_mMatrix = Beta[0] * m_cMomenum[0].m_mMatrix + (1 - Beta[0]) * m_cWeights.m_mMatrix.Power(2);
m_cMomenum[1].m_mMatrix = Beta[1] * m_cMomenum[1].m_mMatrix + (1 - Beta[1]) * delta.Power(2);
m_cWeights.m_mMatrix -= Lambda[0] + Lambda[1] * m_cWeights.m_mMatrix;
for(ulong r = 0; r < W.Rows(); r++)
for(ulong c = 0; c < W.Cols(); c++)
W[r, c] = (m_cMomenum[1].m_mMatrix[r, c] > 0 ? MathSqrt(W[r, c]) / MathSqrt(m_cMomenum[1].m_mMatrix[r, c]) : 0);
m_cWeights.m_mMatrix += W * delta;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(m_cMomenum[0].GetIndex() < 0)
return false;
if(m_cMomenum[1].GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_momentumW, m_cMomenum[0].GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_momentumG, m_cMomenum[1].GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_Lambda2, Lambda[1]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_beta1, Beta[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_beta2, Beta[1]))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_AdaDeltaUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 1=>2;5=85 <0B@8FK 25A>2 <5B>4>< Adam |
//+------------------------------------------------------------------+
bool CNeuronBase::AdamUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
{
//--- ;>: :>=B@>;59
for(int i = 0; i < 2; i++)
{
if(!m_cMomenum[i])
return false;
if(m_cMomenum[i].m_mMatrix.Rows() != m_cWeights.m_mMatrix.Rows() ||
m_cMomenum[i].m_mMatrix.Cols() != m_cWeights.m_mMatrix.Cols())
return false;
}
//--- 0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
{
MATRIX delta = m_cDeltaWeights.m_mMatrix / ((double)batch_size);
MATRIX M = m_cMomenum[0].m_mMatrix = Beta[0] * m_cMomenum[0].m_mMatrix + (1 - Beta[0]) * delta;
MATRIX V = m_cMomenum[1].m_mMatrix = Beta[1] * m_cMomenum[1].m_mMatrix + (1 - Beta[1]) * (delta * delta);
M /= (1 - Beta[0]);
V /= (1 - Beta[1]);
m_cWeights.m_mMatrix -= Lambda[0] + Lambda[1] * m_cWeights.m_mMatrix;
for(ulong r = 0; r < V.Rows(); r++)
for(ulong c = 0; c < V.Cols(); c++)
V[r, c] = (V[r, c] > 0 ? learningRate / MathSqrt(V[r, c]) : 0);
m_cWeights.m_mMatrix += M * V;
}
else
{
//--- !>740=85 1CD5@>2 40==KE
if(m_cWeights.GetIndex() < 0)
return false;
if(m_cDeltaWeights.GetIndex() < 0)
return false;
if(m_cMomenum[0].GetIndex() < 0)
return false;
if(m_cMomenum[1].GetIndex() < 0)
return false;
//--- 5@540G0 0@3C<5=B>2 :5@=5;C
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_delta_weights, m_cDeltaWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_weights, m_cWeights.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_momentumM, m_cMomenum[0].GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_momentumV, m_cMomenum[1].GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_total, m_cWeights.Total()))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_batch_size, batch_size))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_Lambda1, Lambda[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_Lambda2, Lambda[1]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_beta1, Beta[0]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_beta2, Beta[1]))
return false;
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_learningRate, learningRate))
return false;
//--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
int NDRange[] = { (int)((m_cOutputs.Total() + 3) / 4) };
int off_set[] = {0};
if(!m_cOpenCL.Execute(def_k_AdamUpdate, 1, off_set, NDRange))
return false;
//--- >;CG5=85 @57C;LB0B>2 @01>BK :5@=5;0
if(!m_cWeights.BufferRead())
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 A>E@0=5=8O M;5<5=B>2 :;0AA0 2 D09; |
//+------------------------------------------------------------------+
bool CNeuronBase::Save(const int file_handle)
{
//--- ;>: :>=B@>;59
if(file_handle == INVALID_HANDLE)
return false;
//--- 0?8AL 40==KE 1CD5@0 @57;LB0B>2
if(!m_cOutputs)
return false;
if(FileWriteInteger(file_handle, Type()) <= 0 ||
FileWriteInteger(file_handle, m_cOutputs.Total()) <= 0)
return false;
//--- @>25@:0 8 70?8AL D;030 A;>O 8AE>4=KE 40==KE
if(!m_cActivation || !m_cWeights)
{
if(FileWriteInteger(file_handle, 1) <= 0)
return false;
return true;
}
if(FileWriteInteger(file_handle, 0) <= 0)
return false;
int momentums = 0;
switch(m_eOptimization)
{
case SGD:
momentums = 0;
break;
case MOMENTUM:
case AdaGrad:
case RMSProp:
momentums = 1;
break;
case AdaDelta:
case Adam:
momentums = 2;
break;
default:
return false;
break;
}
for(int i = 0; i < momentums; i++)
if(!m_cMomenum[i])
return false;
//--- !>E@0=5=85 <0B@8FK 25A>2KE :>MDD8F85=B>2, <><5=B>2 8 DC=:F88 0:B820F88
if(FileWriteInteger(file_handle, (int)m_eOptimization) <= 0 ||
FileWriteInteger(file_handle, momentums) <= 0)
return false;
if(!m_cWeights.Save(file_handle) || !m_cActivation.Save(file_handle))
return false;
for(int i = 0; i < momentums; i++)
if(!m_cMomenum[i].Save(file_handle))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| 5B>4 2>AAB0=>2;:5=8O A>AB>O=8O :;0AA0 87 40==KE 2 D09;5 |
//+------------------------------------------------------------------+
bool CNeuronBase::Load(const int file_handle)
{
//--- ;>: :>=B@>;59
if(file_handle == INVALID_HANDLE)
return false;
//--- 03@C7:0 1CD5@0 @57C;LB0B>2
if(!m_cOutputs)
if(!(m_cOutputs = new CBufferDouble()))
return false;
int outputs = FileReadInteger(file_handle);
if(!m_cOutputs.BufferInit(1, outputs, 0))
return false;
//--- !>740=85 1CD5@0 3@0485=B>2 >H81:8
if(!m_cGradients)
if(!(m_cGradients = new CBufferDouble()))
return false;
if(!m_cGradients.BufferInit(1, outputs, 0))
return false;
//--- @>25@:0 D;030 A;>O 8AE>4=KE 40==KE
int input_layer = FileReadInteger(file_handle);
if(input_layer == 1)
{
if(m_cActivation)
delete m_cActivation;
if(m_cWeights)
delete m_cWeights;
if(m_cDeltaWeights)
delete m_cDeltaWeights;
if(m_cMomenum[0])
delete m_cMomenum[0];
if(m_cMomenum[1])
delete m_cMomenum[1];
if(m_cOpenCL)
if(!m_cOutputs.BufferCreate(m_cOpenCL))
return false;
m_eOptimization = None;
return true;
}
//--- !>740=85 >1J5:B>2 ?5@54 703@C7:>9 40==KE
if(!m_cActivation)
if(!(m_cActivation = new CActivation()))
return false;
if(!m_cWeights)
if(!(m_cWeights = new CBufferDouble()))
return false;
m_eOptimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle);
int momentums = FileReadInteger(file_handle);
//--- 03@C7:0 40==K9 87 D09;0
if(!m_cWeights.Load(file_handle) || !m_cActivation.Load(file_handle))
return false;
for(int i = 0; i < momentums; i++)
{
if(!m_cMomenum[i])
if(!(m_cMomenum[i] = new CBufferDouble()))
return false;
if(!m_cMomenum[i].Load(file_handle))
return false;
}
//--- =8F80;870F8O >AB02H8EAO 1CD5@>2
if(!m_cDeltaWeights)
if(!(m_cDeltaWeights = new CBufferDouble()))
return false;
//if(!m_cDeltaWeights.BufferInit(m_cWeights.m_mMatrix.Rows(),m_cWeights.m_mMatrix.Cols(), 0))
// return false;
//--- 5@540G0 C:070B5;O =0 >1J5:B> OpenCL 2 >1J5:B DC=:F88 0:B820F88
m_cActivation.SetOpenCL(m_cOpenCL);
//---
return true;
}
//+------------------------------------------------------------------+
Reference in a new issue Copy permalink