1253 lines
101 KiB
MQL5
1253 lines
101 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronBase.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Подключаем библиотеки |
|
|
//+------------------------------------------------------------------+
|
|
#include <Math\Stat\Normal.mqh>
|
|
#include "bufferdouble.mqh"
|
|
#include "layerdescription.mqh"
|
|
#include "activation.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| Class CNeuronBase |
|
|
//| Назначение: Базовый класс полносвязного нейронного слоя |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronBase : public CObject
|
|
{
|
|
protected:
|
|
bool m_bTrain;
|
|
CMyOpenCL *m_cOpenCL;
|
|
CActivation *m_cActivation;
|
|
ENUM_OPTIMIZATION m_eOptimization;
|
|
CBufferDouble *m_cOutputs;
|
|
CBufferDouble *m_cWeights;
|
|
CBufferDouble *m_cDeltaWeights;
|
|
CBufferDouble *m_cGradients;
|
|
CBufferDouble *m_cMomenum[2];
|
|
//---
|
|
virtual bool SGDUpdate(int batch_size, double learningRate,
|
|
double &Lambda[]);
|
|
virtual bool MomentumUpdate(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
virtual bool AdaGradUpdate(int batch_size, double learningRate,
|
|
double &Lambda[]);
|
|
virtual bool RMSPropUpdate(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
virtual bool AdaDeltaUpdate(int batch_size,
|
|
double &Beta[], double &Lambda[]);
|
|
virtual bool AdamUpdate(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
|
|
public:
|
|
CNeuronBase(void);
|
|
~CNeuronBase(void);
|
|
//---
|
|
virtual bool Init(CLayerDescription *description);
|
|
virtual bool SetOpenCL(CMyOpenCL *opencl);
|
|
virtual bool FeedForward(CNeuronBase *prevLayer);
|
|
virtual bool CalcOutputGradient(CBufferDouble *target);
|
|
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
|
|
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer);
|
|
virtual bool UpdateWeights(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
//---
|
|
virtual void TrainMode(bool flag) { m_bTrain = flag; }
|
|
virtual bool TrainMode(void) const { return m_bTrain; }
|
|
//---
|
|
CBufferDouble *GetOutputs(void) const { return(m_cOutputs); }
|
|
CBufferDouble *GetGradients(void) const { return(m_cGradients); }
|
|
CBufferDouble *GetWeights(void) const { return(m_cWeights); }
|
|
CBufferDouble *GetDeltaWeights(void) const { return(m_cDeltaWeights);}
|
|
//--- methods for working with files
|
|
virtual bool Save(const int file_handle);
|
|
virtual bool Load(const int file_handle);
|
|
//--- method of identifying the object
|
|
virtual int Type(void) const { return(defNeuronBase); }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Конструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBase::CNeuronBase(void) : m_eOptimization(Adam)
|
|
{
|
|
m_cOpenCL = NULL;
|
|
m_cActivation = new CActivation();
|
|
m_cOutputs = new CBufferDouble();
|
|
m_cWeights = new CBufferDouble();
|
|
m_cDeltaWeights = new CBufferDouble();
|
|
m_cGradients = new CBufferDouble();
|
|
m_cMomenum[0] = new CBufferDouble();
|
|
m_cMomenum[1] = new CBufferDouble();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Деструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBase::~CNeuronBase(void)
|
|
{
|
|
if(CheckPointer(m_cActivation) != POINTER_INVALID)
|
|
delete m_cActivation;
|
|
if(CheckPointer(m_cOutputs) != POINTER_INVALID)
|
|
delete m_cOutputs;
|
|
if(CheckPointer(m_cWeights) != POINTER_INVALID)
|
|
delete m_cWeights;
|
|
if(CheckPointer(m_cDeltaWeights) != POINTER_INVALID)
|
|
delete m_cDeltaWeights;
|
|
if(CheckPointer(m_cGradients) != POINTER_INVALID)
|
|
delete m_cGradients;
|
|
if(CheckPointer(m_cMomenum[0]) != POINTER_INVALID)
|
|
delete m_cMomenum[0];
|
|
if(CheckPointer(m_cMomenum[1]) != POINTER_INVALID)
|
|
delete m_cMomenum[1];
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения указателя на объект работы с контекстом |
|
|
//| OpenCL |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::SetOpenCL(CMyOpenCL *opencl)
|
|
{
|
|
if(CheckPointer(opencl) == POINTER_INVALID)
|
|
{
|
|
if(CheckPointer(m_cOutputs) != POINTER_INVALID)
|
|
m_cOutputs.BufferFree();
|
|
if(CheckPointer(m_cGradients) != POINTER_INVALID)
|
|
m_cGradients.BufferFree();
|
|
if(CheckPointer(m_cWeights) != POINTER_INVALID)
|
|
m_cWeights.BufferFree();
|
|
if(CheckPointer(m_cDeltaWeights) != POINTER_INVALID)
|
|
m_cDeltaWeights.BufferFree();
|
|
for(int i = 0; i < 2; i++)
|
|
{
|
|
if(CheckPointer(m_cMomenum[i]) != POINTER_INVALID)
|
|
m_cMomenum[i].BufferFree();
|
|
}
|
|
}
|
|
if(m_cOpenCL != opencl)
|
|
{
|
|
if(CheckPointer(m_cOpenCL) != POINTER_INVALID)
|
|
delete m_cOpenCL;
|
|
m_cOpenCL = opencl;
|
|
if(CheckPointer(m_cActivation) != POINTER_INVALID)
|
|
m_cActivation.SetOpenCL(m_cOpenCL);
|
|
}
|
|
//---
|
|
return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод инициализации класса |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::Init(CLayerDescription *description)
|
|
{
|
|
//--- Блок контроля исходных данных
|
|
if(CheckPointer(description) == POINTER_INVALID || description.type != Type() ||
|
|
description.count <= 0)
|
|
return false;
|
|
//--- Создание буфера результатов
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID)
|
|
{
|
|
m_cOutputs = new CBufferDouble();
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cOutputs.BufferInit(description.count, 0))
|
|
return false;
|
|
//--- Создание буфера градиентов ошибюки
|
|
if(CheckPointer(m_cGradients) == POINTER_INVALID)
|
|
{
|
|
m_cGradients = new CBufferDouble();
|
|
if(CheckPointer(m_cGradients) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cGradients.BufferInit(description.count, 0))
|
|
return false;
|
|
//--- Удаление не используемых объектов для слоя исходных данных
|
|
if(description.window <= 0)
|
|
{
|
|
if(CheckPointer(m_cActivation) != POINTER_INVALID)
|
|
delete m_cActivation;
|
|
if(CheckPointer(m_cWeights) != POINTER_INVALID)
|
|
delete m_cWeights;
|
|
if(CheckPointer(m_cDeltaWeights) != POINTER_INVALID)
|
|
delete m_cDeltaWeights;
|
|
if(CheckPointer(m_cMomenum[0]) != POINTER_INVALID)
|
|
delete m_cMomenum[0];
|
|
if(CheckPointer(m_cMomenum[1]) != POINTER_INVALID)
|
|
delete m_cMomenum[1];
|
|
if(CheckPointer(m_cOpenCL) != POINTER_INVALID)
|
|
if(!m_cOutputs.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
m_eOptimization = description.optimization;
|
|
return true;
|
|
}
|
|
//--- Инициализация объекта функции активации
|
|
if(CheckPointer(m_cActivation) == POINTER_INVALID)
|
|
{
|
|
m_cActivation = new CActivation();
|
|
if(CheckPointer(m_cActivation) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
m_cActivation.SetFunction(description.activation,
|
|
description.activation_params[0],
|
|
description.activation_params[1]);
|
|
if(description.activation == ACT_SWISH)
|
|
m_cActivation.BufferInit(description.count);
|
|
else
|
|
m_cActivation.BufferInit(1);
|
|
m_cActivation.SetOpenCL(m_cOpenCL);
|
|
//--- Инициализация объекта матрицы весов
|
|
if(CheckPointer(m_cWeights) == POINTER_INVALID)
|
|
{
|
|
m_cWeights = new CBufferDouble();
|
|
if(CheckPointer(m_cWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int total = description.count * (description.window + 1);
|
|
if(!m_cWeights.Reserve(total))
|
|
return false;
|
|
double weights[];
|
|
double sigma = description.activation == ACT_LReLU ?
|
|
2.0 / (double)(MathPow(1 + description.activation_params[0], 2) * description.window) :
|
|
1.0 / (double)description.window;
|
|
if(!MathRandomNormal(0, MathSqrt(sigma), total, weights))
|
|
return false;
|
|
if(!m_cWeights.AssignArray(weights))
|
|
return false;
|
|
//--- Инициализация объекта накопления градиентов на уровне матрицы весов
|
|
if(CheckPointer(m_cDeltaWeights) == POINTER_INVALID)
|
|
{
|
|
m_cDeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(m_cDeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cDeltaWeights.BufferInit(total, 0))
|
|
return false;
|
|
//--- Инициализация объектов моментов
|
|
switch(description.optimization)
|
|
{
|
|
case None:
|
|
case SGD:
|
|
for(int i = 0; i < 2; i++)
|
|
if(CheckPointer(m_cMomenum[i]) != POINTER_INVALID)
|
|
delete m_cMomenum[i];
|
|
break;
|
|
case MOMENTUM:
|
|
case AdaGrad:
|
|
case RMSProp:
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
m_cMomenum[0] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[0].BufferInit(total, 0))
|
|
return false;
|
|
if(CheckPointer(m_cMomenum[1]) != POINTER_INVALID)
|
|
delete m_cMomenum[1];
|
|
break;
|
|
case AdaDelta:
|
|
case Adam:
|
|
for(int i = 0; i < 2; i++)
|
|
{
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
m_cMomenum[i] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[i].BufferInit(total, 0))
|
|
return false;
|
|
}
|
|
break;
|
|
default:
|
|
return false;
|
|
break;
|
|
}
|
|
//--- Сохранение метода оптимизации параметров
|
|
m_eOptimization = description.optimization;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод прямого прохода |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::FeedForward(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputs) == POINTER_INVALID ||
|
|
CheckPointer(m_cWeights) == POINTER_INVALID ||
|
|
CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
|
|
CheckPointer(m_cActivation) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *input_data = prevLayer.GetOutputs();
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
int input_total = input_data.Total();
|
|
int output_total = m_cOutputs.Total();
|
|
if(m_cWeights.Total() < (input_total + 1)*output_total)
|
|
return false;
|
|
//---
|
|
double inputs[], weights[];
|
|
if(input_data.GetData(inputs, false) < input_total ||
|
|
m_cWeights.GetData(weights, false) < m_cWeights.Total())
|
|
return false;
|
|
for(int o = 0; o < output_total; o++)
|
|
{
|
|
int shift = o * (input_total + 1);
|
|
double output = 0;
|
|
for(int i = 0; i < input_total; i++)
|
|
output += inputs[i] * weights[shift + i];
|
|
output += m_cWeights.At(shift + input_total);
|
|
if(!m_cOutputs.Update(o, output))
|
|
return false;
|
|
}
|
|
return m_cActivation.Activation(m_cOutputs);
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(input_data.GetIndex() < 0)
|
|
if(!input_data.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(m_cWeights.GetIndex() < 0 && !m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
m_cActivation.SetOpenCL(m_cOpenCL);
|
|
if(m_cActivation.GetIndex() < 0 && (!m_cActivation.BufferInit(m_cOutputs.Total()) || !m_cActivation.BufferCreate()))
|
|
return false;
|
|
if(m_cOutputs.GetIndex() < 0 && !m_cOutputs.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_inputs, input_data.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_sums, m_cActivation.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_PerceptronFeedForward, def_pff_outputs, m_cOutputs.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_inputs_total, input_data.Total()))
|
|
return false;
|
|
double params[];
|
|
ENUM_ACTIVATION function = m_cActivation.GetFunction(params);
|
|
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_activation, (int)function))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_act_param_a, params[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_PerceptronFeedForward, def_pff_act_param_b, params[1]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int off_set[] = {0};
|
|
int NDRange[] = {m_cOutputs.Total()};
|
|
if(!m_cOpenCL.Execute(def_k_PerceptronFeedForward, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cOutputs.BufferRead())
|
|
return false;
|
|
if(function == ACT_SWISH && !m_cActivation.BufferRead())
|
|
return false;
|
|
input_data.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cActivation.BufferFree();
|
|
if(function != ACT_SOFTMAX)
|
|
return true;
|
|
//--- Только для SoftMax, нормализация результатов
|
|
//--- вычисление ощей суммы всех значений буфера данных
|
|
double summ = 0;
|
|
double array[];
|
|
int total = m_cOutputs.GetData(array);
|
|
if(total <= 0)
|
|
return false;
|
|
for(int i = 0; i < total; i++)
|
|
summ += array[i];
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_Normalize, def_norm_inputs, m_cOutputs.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_Normalize, def_norm_outputs, m_cOutputs.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_Normalize, def_norm_inputs_total, total))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_Normalize, def_norm_const_value, summ))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = total;
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
NDRange[0] = s;
|
|
if(!m_cOpenCL.Execute(def_k_Normalize, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cOutputs.BufferRead())
|
|
return false;
|
|
return true;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод расчёта градиента ошибки слоя результатов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::CalcOutputGradient(CBufferDouble *target)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(target) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputs) == POINTER_INVALID ||
|
|
CheckPointer(m_cGradients) == POINTER_INVALID ||
|
|
target.Total() < m_cOutputs.Total() ||
|
|
m_cGradients.Total() < m_cOutputs.Total())
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
for(int i = 0; i < m_cOutputs.Total(); i++)
|
|
if(!m_cGradients.Update(i, target.At(i) - m_cOutputs.At(i)))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(target.GetIndex() < 0)
|
|
if(!target.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(m_cOutputs.GetIndex() < 0 && !m_cOutputs.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(m_cGradients.GetIndex() < 0 && !m_cGradients.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_target, target.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_outputs, m_cOutputs.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcOutputGradient, def_outgr_gradients, m_cGradients.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_CalcOutputGradient, def_outgr_outputs_total, m_cOutputs.Total()))
|
|
return false;
|
|
//--- Постановка кернела в лчередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_CalcOutputGradient, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cGradients.BufferRead())
|
|
return false;
|
|
target.BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиента через скрытый слой |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::CalcHiddenGradient(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Корректировка входящего градиента на производную функции активации
|
|
m_cActivation.SetOpenCL(m_cOpenCL);
|
|
if(!m_cActivation.Derivative(m_cOutputs, m_cGradients))
|
|
return false;
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID)
|
|
return false;
|
|
//--- Проверка буферов предыдущего слоя
|
|
CBufferDouble *input_data = prevLayer.GetOutputs();
|
|
CBufferDouble *input_gradient = prevLayer.GetGradients();
|
|
if(CheckPointer(input_data) == POINTER_INVALID ||
|
|
CheckPointer(input_gradient) == POINTER_INVALID ||
|
|
input_data.Total() > input_gradient.Total())
|
|
return false;
|
|
//--- Проверка соответствия размера буфера исходных данных и матрицы весов
|
|
int input_total = input_data.Total();
|
|
if(CheckPointer(m_cWeights) == POINTER_INVALID ||
|
|
m_cWeights.Total() < (input_total + 1)*m_cOutputs.Total())
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double weights[], gradients[];
|
|
int grad_total = m_cGradients.Total();
|
|
if(m_cWeights.GetData(weights, false) <= 0 || m_cGradients.GetData(gradients, false) <= 0)
|
|
return false;
|
|
for(int i = 0; i < input_total; i++)
|
|
{
|
|
double grad = 0;
|
|
double inp = input_data.At(i);
|
|
for(int o = 0; o < grad_total; o++)
|
|
grad += gradients[o] * weights[o * (input_total + 1) + i];
|
|
if(!input_gradient.Update(i, grad))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(m_cWeights.GetIndex() < 0 && !m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(input_gradient.GetIndex() < 0 && !input_gradient.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(m_cGradients.GetIndex() < 0 && !m_cGradients.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_gradient_inputs, input_gradient.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcHiddenGradient, def_hidgr_gradients, m_cGradients.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_CalcHiddenGradient, def_hidgr_outputs_total, m_cGradients.Total()))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int NDRange[] = {input_total};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_CalcHiddenGradient, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!input_gradient.BufferRead())
|
|
return false;
|
|
m_cWeights.BufferFree();
|
|
m_cGradients.BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределение градиента ошибки до матрицы весов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::CalcDeltaWeights(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(m_cDeltaWeights) == POINTER_INVALID ||
|
|
CheckPointer(m_cGradients) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *Inputs = prevLayer.GetOutputs();
|
|
if(CheckPointer(Inputs) == POINTER_INVALID)
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double deltas[];
|
|
double inputs[];
|
|
int inputs_total = Inputs.GetData(inputs, false);
|
|
if(m_cDeltaWeights.GetData(deltas, false) <= 0 || inputs_total <= 0)
|
|
return false;
|
|
for(int g = 0; g < m_cGradients.Total(); g++)
|
|
{
|
|
int shift = g * (inputs_total + 1);
|
|
double gradient = m_cGradients.At(g);
|
|
if(gradient == 0)
|
|
continue;
|
|
for(int i = 0; i < inputs_total; i++)
|
|
deltas[shift + i] += inputs[i] * gradient;
|
|
deltas[shift + inputs_total] += gradient;
|
|
}
|
|
if(!m_cDeltaWeights.AssignArray(deltas))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(m_cGradients.GetIndex() < 0 && !m_cGradients.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(m_cDeltaWeights.GetIndex() < 0 && !m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(Inputs.GetIndex() < 0 && !Inputs.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_inputs, Inputs.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_CalcDeltaWeights, def_delt_gradients, m_cGradients.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_CalcDeltaWeights, def_delt_inputs_total, Inputs.Total()))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int NDRange[] = {m_cGradients.Total()};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_CalcDeltaWeights, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cGradients.BufferFree();
|
|
Inputs.BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод обновления матрицы весов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cDeltaWeights) == POINTER_INVALID ||
|
|
CheckPointer(m_cWeights) == POINTER_INVALID ||
|
|
m_cWeights.Total() < m_cDeltaWeights.Total() ||
|
|
batch_size <= 0)
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от используемой функции активации
|
|
bool result = false;
|
|
switch(m_eOptimization)
|
|
{
|
|
case None:
|
|
result = true;
|
|
break;
|
|
case SGD:
|
|
result = SGDUpdate(batch_size, learningRate, Lambda);
|
|
break;
|
|
case MOMENTUM:
|
|
result = MomentumUpdate(batch_size, learningRate, Beta, Lambda);
|
|
break;
|
|
case AdaGrad:
|
|
result = AdaGradUpdate(batch_size, learningRate, Lambda);
|
|
break;
|
|
case RMSProp:
|
|
result = RMSPropUpdate(batch_size, learningRate, Beta, Lambda);
|
|
break;
|
|
case AdaDelta:
|
|
result = AdaDeltaUpdate(batch_size, Beta, Lambda);
|
|
break;
|
|
case Adam:
|
|
result = AdamUpdate(batch_size, learningRate, Beta, Lambda);
|
|
break;
|
|
}
|
|
if(result)
|
|
m_cDeltaWeights.BufferInit(m_cDeltaWeights.Total(), 0);
|
|
//---
|
|
return result;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов стохастическим градиентным спуском |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::SGDUpdate(int batch_size, double learningRate, double &Lambda[])
|
|
{
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double lr = learningRate / ((double)batch_size);
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
w += lr * m_cDeltaWeights.At(i);
|
|
if(!m_cWeights.Update(i, w))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_SGDUpdate, def_sgd_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_SGDUpdate, def_sgd_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_learningRate, learningRate))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_SGDUpdate, def_sgd_Lambda2, Lambda[1]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_SGDUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов методом моментов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::MomentumUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
if(Beta[0] == 0)
|
|
return SGDUpdate(batch_size, learningRate, Lambda);
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[0] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[0].BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
}
|
|
if(m_cMomenum[0].Total() < m_cWeights.Total())
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double lr = learningRate / ((double)batch_size);
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
double delta = lr * m_cDeltaWeights.At(i) + Beta[0] * m_cMomenum[0].At(i);
|
|
w += delta;
|
|
if(!m_cWeights.Update(i, w) || !m_cMomenum[0].Update(i, delta))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[0].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_MomentumUpdate, def_moment_momentum, m_cMomenum[0].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_learningRate, learningRate))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_Lambda2, Lambda[1]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_MomentumUpdate, def_moment_beta, Beta[0]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_MomentumUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[0].BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cMomenum[0].BufferFree();
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов методом AdaGrad |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::AdaGradUpdate(int batch_size, double learningRate, double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[0] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[0].BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
}
|
|
if(m_cMomenum[0].Total() < m_cWeights.Total())
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double lr = learningRate;
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
double delta = m_cDeltaWeights.At(i) / ((double)batch_size);
|
|
double G = m_cMomenum[0].At(i) + MathPow(delta, 2);
|
|
w += lr / MathSqrt(G + 1.0e-10) * delta;
|
|
if(!m_cWeights.Update(i, w) || !m_cMomenum[0].Update(i, G))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[0].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaGradUpdate, def_adagrad_momentum, m_cMomenum[0].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_learningRate, learningRate))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaGradUpdate, def_adagrad_Lambda2, Lambda[1]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_AdaGradUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[0].BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cMomenum[0].BufferFree();
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов методом RMSProp |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::RMSPropUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[0] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[0]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[0].BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
}
|
|
if(m_cMomenum[0].Total() < m_cWeights.Total())
|
|
return false;
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
double lr = learningRate;
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
double delta = m_cDeltaWeights.At(i) / ((double)batch_size);
|
|
double G = Beta[0] * m_cMomenum[0].At(i) + (1 - Beta[0]) * MathPow(delta, 2);
|
|
w += lr / (MathSqrt(G) + 1.0e-10) * delta;
|
|
if(!m_cWeights.Update(i, w) || !m_cMomenum[0].Update(i, G))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[0].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_RMSPropUpdate, def_rms_momentum, m_cMomenum[0].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_learningRate, learningRate))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_Lambda2, Lambda[1]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_RMSPropUpdate, def_rms_beta, Beta[0]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_RMSPropUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[0].BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cMomenum[0].BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов методом AdaDelta |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::AdaDeltaUpdate(int batch_size, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
for(int i = 0; i < 2; i++)
|
|
{
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[i] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[i].BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
}
|
|
if(m_cMomenum[i].Total() < m_cWeights.Total())
|
|
return false;
|
|
}
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
double delta = m_cDeltaWeights.At(i) / ((double)batch_size);
|
|
double W = Beta[0] * m_cMomenum[0].At(i) + (1 - Beta[0]) * MathPow(w, 2);
|
|
double G = Beta[1] * m_cMomenum[1].At(i) + (1 - Beta[1]) * MathPow(delta, 2);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
w += MathSqrt(W) / (MathSqrt(G) + 1.0e-10) * delta;
|
|
if(!m_cWeights.Update(i, w) || !m_cMomenum[0].Update(i, W) ||
|
|
!m_cMomenum[1].Update(i, G))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[0].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[1].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_momentumW, m_cMomenum[0].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdaDeltaUpdate, def_adadelt_momentumG, m_cMomenum[1].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_Lambda2, Lambda[1]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_beta1, Beta[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdaDeltaUpdate, def_adadelt_beta2, Beta[1]))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_AdaDeltaUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[0].BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[1].BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cMomenum[0].BufferFree();
|
|
m_cMomenum[1].BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Обновление матрицы весов методом Adam |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::AdamUpdate(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
for(int i = 0; i < 2; i++)
|
|
{
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[i] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cMomenum[i].BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
}
|
|
if(m_cMomenum[i].Total() < m_cWeights.Total())
|
|
return false;
|
|
}
|
|
//--- Разветвление алгоритма в зависимости от устройста выполнениия операций
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
for(int i = 0; i < m_cWeights.Total(); i++)
|
|
{
|
|
double w = m_cWeights.At(i);
|
|
double delta = m_cDeltaWeights.At(i) / ((double)batch_size);
|
|
double M = Beta[0] * m_cMomenum[0].At(i) + (1 - Beta[0]) * delta;
|
|
double V = Beta[1] * m_cMomenum[1].At(i) + (1 - Beta[1]) * MathPow(delta, 2);
|
|
double m = M / (1 - Beta[0]);
|
|
double v = V / (1 - Beta[1]);
|
|
w -= Lambda[0] + Lambda[1] * w;
|
|
w += learningRate * m / (MathSqrt(v) + 1.0e-10);
|
|
if(!m_cWeights.Update(i, w) || !m_cMomenum[0].Update(i, M) ||
|
|
!m_cMomenum[1].Update(i, V))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создание буферов данных
|
|
if(!m_cWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[0].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cMomenum[1].BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_delta_weights, m_cDeltaWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_weights, m_cWeights.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_momentumM, m_cMomenum[0].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AdamUpdate, def_adam_momentumV, m_cMomenum[1].GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_total, m_cWeights.Total()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_batch_size, batch_size))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_Lambda1, Lambda[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_Lambda2, Lambda[1]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_beta1, Beta[0]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_beta2, Beta[1]))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AdamUpdate, def_adam_learningRate, learningRate))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_AdamUpdate, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получение результатов работы кернела
|
|
if(!m_cWeights.BufferRead())
|
|
return false;
|
|
if(!m_cDeltaWeights.BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[0].BufferRead())
|
|
return false;
|
|
if(!m_cMomenum[1].BufferRead())
|
|
return false;
|
|
m_cDeltaWeights.BufferFree();
|
|
m_cWeights.BufferFree();
|
|
m_cMomenum[0].BufferFree();
|
|
m_cMomenum[1].BufferFree();
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод сохранения элементов класса в файл |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::Save(const int file_handle)
|
|
{
|
|
//--- Блок контролей
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
//--- Запись данных буфера резльтатов
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, Type()) <= 0 ||
|
|
FileWriteInteger(file_handle, m_cOutputs.Total()) <= 0)
|
|
return false;
|
|
//--- Проверка и запись флага слоя исходных данных
|
|
if(CheckPointer(m_cActivation) == POINTER_INVALID ||
|
|
CheckPointer(m_cWeights) == POINTER_INVALID)
|
|
{
|
|
if(FileWriteInteger(file_handle, 1) <= 0)
|
|
return false;
|
|
return true;
|
|
}
|
|
if(FileWriteInteger(file_handle, 0) <= 0)
|
|
return false;
|
|
int momentums = 0;
|
|
switch(m_eOptimization)
|
|
{
|
|
case SGD:
|
|
momentums = 0;
|
|
break;
|
|
case MOMENTUM:
|
|
case AdaGrad:
|
|
case RMSProp:
|
|
momentums = 1;
|
|
break;
|
|
case AdaDelta:
|
|
case Adam:
|
|
momentums = 2;
|
|
break;
|
|
default:
|
|
return false;
|
|
break;
|
|
}
|
|
for(int i = 0; i < momentums; i++)
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
return false;
|
|
//--- Сохранение матрицы весовых коэффициентов, моментов и функции активации
|
|
if(FileWriteInteger(file_handle, (int)m_eOptimization) <= 0 ||
|
|
FileWriteInteger(file_handle, momentums) <= 0)
|
|
return false;
|
|
if(!m_cWeights.Save(file_handle) || !m_cActivation.Save(file_handle))
|
|
return false;
|
|
for(int i = 0; i < momentums; i++)
|
|
if(!m_cMomenum[i].Save(file_handle))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод восстановлкения состояния класса из данных в файле |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBase::Load(const int file_handle)
|
|
{
|
|
//--- Блок контролей
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
//--- Загрузка буфера результатов
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID)
|
|
{
|
|
m_cOutputs = new CBufferDouble();
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int outputs = FileReadInteger(file_handle);
|
|
if(!m_cOutputs.BufferInit(outputs, 0))
|
|
return false;
|
|
//--- Создание буфера градиентов ошибки
|
|
if(CheckPointer(m_cGradients) == POINTER_INVALID)
|
|
{
|
|
m_cGradients = new CBufferDouble();
|
|
if(CheckPointer(m_cGradients) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cGradients.BufferInit(outputs, 0))
|
|
return false;
|
|
//--- Проверка флага слоя исходных данных
|
|
int input_layer = FileReadInteger(file_handle);
|
|
if(input_layer == 1)
|
|
{
|
|
if(CheckPointer(m_cActivation) != POINTER_INVALID)
|
|
delete m_cActivation;
|
|
if(CheckPointer(m_cWeights) != POINTER_INVALID)
|
|
delete m_cWeights;
|
|
if(CheckPointer(m_cDeltaWeights) != POINTER_INVALID)
|
|
delete m_cDeltaWeights;
|
|
if(CheckPointer(m_cMomenum[0]) != POINTER_INVALID)
|
|
delete m_cMomenum[0];
|
|
if(CheckPointer(m_cMomenum[1]) != POINTER_INVALID)
|
|
delete m_cMomenum[1];
|
|
if(CheckPointer(m_cOpenCL) != POINTER_INVALID)
|
|
if(!m_cOutputs.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
m_eOptimization = None;
|
|
return true;
|
|
}
|
|
//--- Создание объектов перед загрузкой данных
|
|
if(CheckPointer(m_cActivation) == POINTER_INVALID)
|
|
{
|
|
m_cActivation = new CActivation();
|
|
if(CheckPointer(m_cActivation) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(CheckPointer(m_cWeights) == POINTER_INVALID)
|
|
{
|
|
m_cWeights = new CBufferDouble();
|
|
if(CheckPointer(m_cWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
m_eOptimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle);
|
|
int momentums = FileReadInteger(file_handle);
|
|
//--- Загрузка данный из файла
|
|
if(!m_cWeights.Load(file_handle) || !m_cActivation.Load(file_handle))
|
|
return false;
|
|
for(int i = 0; i < momentums; i++)
|
|
{
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
{
|
|
m_cMomenum[i] = new CBufferDouble();
|
|
if(CheckPointer(m_cMomenum[i]) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cMomenum[i].Load(file_handle))
|
|
return false;
|
|
}
|
|
//--- Инициализация оставшихся буферов
|
|
if(CheckPointer(m_cDeltaWeights) == POINTER_INVALID)
|
|
{
|
|
m_cDeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(m_cDeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cDeltaWeights.BufferInit(m_cWeights.Total(), 0))
|
|
return false;
|
|
//--- Передача указателя на объекто OpenCL в объект функции активации
|
|
m_cActivation.SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|