680 lines
51 KiB
MQL5
680 lines
51 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronMHAttention.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Подключаем библиотеки |
|
|
//+------------------------------------------------------------------+
|
|
#include "neuronattention.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| Class CNeuronMHAttention |
|
|
//| Назначение: Класс организации работы блока многоголового внимания|
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronMHAttention : public CNeuronAttention
|
|
{
|
|
protected:
|
|
CNeuronConv *m_cW0;
|
|
|
|
int m_iHeads;
|
|
|
|
public:
|
|
CNeuronMHAttention(void);
|
|
~CNeuronMHAttention(void);
|
|
//---
|
|
virtual bool Init(CLayerDescription *desc);
|
|
virtual bool SetOpenCL(CMyOpenCL *opencl);
|
|
virtual bool FeedForward(CNeuronBase *prevLayer);
|
|
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
|
|
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer);
|
|
virtual bool UpdateWeights(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
//--- Методы работы с файлами
|
|
virtual bool Save(const int file_handle);
|
|
virtual bool Load(const int file_handle);
|
|
//--- Метод идентификации объекта
|
|
virtual int Type(void) const { return(defNeuronMHAttention); }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Конструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronMHAttention::CNeuronMHAttention(void) : m_iHeads(8)
|
|
{
|
|
m_cW0 = new CNeuronConv();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Деструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronMHAttention::~CNeuronMHAttention(void)
|
|
{
|
|
if(CheckPointer(m_cW0) != POINTER_INVALID)
|
|
delete m_cW0;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод инициализации класса |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::Init(CLayerDescription *desc)
|
|
{
|
|
//--- Проверяем исходные данные
|
|
if(!desc || desc.type != Type() ||
|
|
desc.count <= 0 || desc.window <= 0 || desc.window_out <= 0 ||
|
|
desc.step <= 0)
|
|
return false;
|
|
//--- Сохраняем константы
|
|
m_iWindow = desc.window;
|
|
m_iUnits = desc.count;
|
|
m_iKeysSize = desc.window_out;
|
|
m_iHeads = desc.step;
|
|
//--- Создаём описание для внутренних нейронных слоёв
|
|
CLayerDescription *temp = new CLayerDescription();
|
|
if(!temp)
|
|
return false;
|
|
temp.type = defNeuronConv;
|
|
temp.window = m_iWindow;
|
|
temp.window_out = (int)(m_iKeysSize * m_iHeads);
|
|
temp.step = m_iWindow;
|
|
temp.count = m_iUnits;
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
temp.optimization = desc.optimization;
|
|
//--- Вызываем метод инициализации родительского класса
|
|
desc.count *= m_iWindow;
|
|
desc.window_out = 1;
|
|
desc.window = 0;
|
|
if(!CNeuronBase::Init(desc))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем Querys
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
|
|
{
|
|
m_cQuerys = new CNeuronConv();
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
if(!m_cQuerys.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cQuerys.SetTransposedOutput(true);
|
|
//--- Инициализируем Keys
|
|
if(CheckPointer(m_cKeys) == POINTER_INVALID)
|
|
{
|
|
m_cKeys = new CNeuronConv();
|
|
if(CheckPointer(m_cKeys) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
if(!m_cKeys.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cKeys.SetTransposedOutput(true);
|
|
//--- Инициализируем Values
|
|
if(CheckPointer(m_cValues) == POINTER_INVALID)
|
|
{
|
|
m_cValues = new CNeuronConv();
|
|
if(CheckPointer(m_cValues) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
if(!m_cValues.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cValues.SetTransposedOutput(true);
|
|
//--- Инициализируем Scores
|
|
if(CheckPointer(m_cScores) == POINTER_INVALID)
|
|
{
|
|
m_cScores = new CBufferDouble();
|
|
if(CheckPointer(m_cScores) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
if(!m_cScores.BufferInit(m_iHeads, m_iUnits * m_iUnits))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем AttentionOut
|
|
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
|
|
{
|
|
m_cAttentionOut = new CNeuronBase();
|
|
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
desc.type = defNeuronBase;
|
|
desc.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);
|
|
if(!m_cAttentionOut.Init(desc))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
desc.count = m_iUnits * m_iWindow;
|
|
//--- Инициализируем W0
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
{
|
|
m_cW0 = new CNeuronConv();
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
temp.window = (int)(m_iKeysSize * m_iHeads);
|
|
temp.step = temp.window;
|
|
temp.window_out = m_iWindow;
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!m_cW0.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cW0.SetTransposedOutput(true);
|
|
//--- Инициализируем FF1
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID)
|
|
{
|
|
m_cFF1 = new CNeuronConv();
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
temp.window = m_iWindow;
|
|
temp.step = temp.window;
|
|
temp.window_out = temp.window * 4;
|
|
temp.activation = ACT_SWISH;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!m_cFF1.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cFF1.SetTransposedOutput(true);
|
|
//--- Инициализируем FF2
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
{
|
|
m_cFF2 = new CNeuronConv();
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
}
|
|
temp.window = temp.window_out;
|
|
temp.window_out = temp.step;
|
|
temp.step = temp.window;
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!m_cFF2.Init(temp))
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
m_cFF2.SetTransposedOutput(true);
|
|
delete temp;
|
|
//--- Для исключениия копирования буферов осуществим их подмену
|
|
if(m_cOutputs)
|
|
delete m_cOutputs;
|
|
m_cOutputs = m_cFF2.GetOutputs();
|
|
if(m_cGradients)
|
|
delete m_cGradients;
|
|
m_cGradients = m_cFF2.GetGradients();
|
|
//---
|
|
SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод передачи указателя на объект OpenCL до всех |
|
|
//| внутренних объектов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::SetOpenCL(CMyOpenCL *opencl)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
CNeuronAttention::SetOpenCL(opencl);
|
|
//--- Вызываем аналогичный метод для внутреннего слоя
|
|
if(CheckPointer(m_cW0) != POINTER_INVALID)
|
|
m_cW0.SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод прямого прохода |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::FeedForward(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
|
|
CheckPointer(m_cQuerys) == POINTER_INVALID ||
|
|
CheckPointer(m_cValues) == POINTER_INVALID ||
|
|
CheckPointer(m_cKeys) == POINTER_INVALID ||
|
|
CheckPointer(m_cW0) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF1) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
if(!m_cQuerys.FeedForward(prevLayer))
|
|
return false;
|
|
if(!m_cKeys.FeedForward(prevLayer))
|
|
return false;
|
|
if(!m_cValues.FeedForward(prevLayer))
|
|
return false;
|
|
//--- Инициализируем Scores
|
|
if(CheckPointer(m_cScores) == POINTER_INVALID)
|
|
{
|
|
m_cScores = new CBufferDouble();
|
|
if(CheckPointer(m_cScores) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
//--- Инициализируем AttentionOut
|
|
if(!m_cAttentionOut)
|
|
return false;
|
|
if(!m_cAttentionOut.GetOutputs())
|
|
return false;
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
MATRIX out;
|
|
if(!m_cOpenCL)
|
|
{
|
|
if(!out.Init(m_iHeads, m_iUnits * m_iKeysSize))
|
|
return false;
|
|
MATRIX querys[], keys[], values[];
|
|
if(!m_cQuerys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, querys))
|
|
return false;
|
|
if(!m_cKeys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, keys))
|
|
return false;
|
|
if(!m_cValues.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values))
|
|
return false;
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
if(!querys[head].Reshape(m_iUnits, m_iKeysSize) ||
|
|
!keys[head].Reshape(m_iUnits, m_iKeysSize) ||
|
|
!values[head].Reshape(m_iUnits, m_iKeysSize))
|
|
return false;
|
|
//--- Определяем Scores
|
|
MATRIX sc = querys[head].MatMul(keys[head].Transpose()) / sqrt(m_iKeysSize);
|
|
for(uint r = 0; r < sc.Rows()*sc.Cols(); r++)
|
|
if(!sc.Flat(r, exp(sc.Flat(r))))
|
|
return false;
|
|
VECTOR sum = sc.Sum(1);
|
|
for(uint r = 0; r < sc.Rows(); r++)
|
|
if(!sc.Row(sc.Row(r) / sum[r], r))
|
|
return false;
|
|
MATRIX temp = sc;
|
|
if(!temp.Reshape(1, m_iUnits * m_iUnits))
|
|
return false;
|
|
if(!m_cScores.m_mMatrix.Row(temp.Row(0), head))
|
|
return false;
|
|
//--- Выход блока внимания
|
|
temp = sc * values[head];
|
|
if(!temp.Reshape(1, m_iUnits * m_iKeysSize))
|
|
return false;
|
|
if(!out.Row(temp.Row(0), head))
|
|
return false;
|
|
}
|
|
if(!out.Reshape(1, m_cAttentionOut.GetOutputs().Total()))
|
|
return false;
|
|
m_cAttentionOut.GetOutputs().m_mMatrix = out;
|
|
}
|
|
else // Блок OpenCL
|
|
{
|
|
//--- Создание буферов данных
|
|
if(m_cQuerys.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cKeys.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cValues.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cScores.GetIndex() < 0)
|
|
return false;
|
|
if(m_cAttentionOut.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
//--- Передача параметров кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_keys, m_cKeys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_outputs, m_cAttentionOut.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_querys, m_cQuerys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_scores, m_cScores.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_values, m_cValues.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_key_size, m_iKeysSize))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_window, m_iKeysSize))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_mask, 0))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int off_set[] = {0, 0};
|
|
int NDRange[] = {m_iUnits, m_iHeads};
|
|
if(!m_cOpenCL.Execute(def_k_AttentionFeedForward, 2, off_set, NDRange))
|
|
return false;
|
|
//--- Считываниие результатов операций
|
|
if(!m_cAttentionOut.GetOutputs().BufferRead())
|
|
return false;
|
|
}
|
|
//---
|
|
if(!m_cW0.FeedForward(m_cAttentionOut))
|
|
return false;
|
|
int total = m_cW0.GetOutputs().GetData(out, false);
|
|
if(total <= 0)
|
|
return false;
|
|
//--- Суммируем с исходными данными и нормализуем
|
|
out += prevLayer.GetOutputs().m_mMatrix;
|
|
double mean = out.Mean();
|
|
m_dStd[0] = out.Std();
|
|
m_cW0.GetOutputs().m_mMatrix = out = (out - mean) / m_dStd[0];
|
|
if(m_cOpenCL && !m_cW0.GetOutputs().BufferWrite())
|
|
return false;
|
|
//---
|
|
if(!m_cFF1.FeedForward(m_cW0))
|
|
return false;
|
|
if(!m_cFF2.FeedForward(m_cFF1))
|
|
return false;
|
|
//--- Суммируем с выходом внимания и нормализуем
|
|
out += m_cOutputs.m_mMatrix;
|
|
mean = out.Mean();
|
|
m_dStd[1] = out.Std();
|
|
m_cOutputs.m_mMatrix = out = (out - mean) / m_dStd[1];
|
|
if(m_cOpenCL && !m_cOutputs.BufferWrite())
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиента ошибки через скрытый слой |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::CalcHiddenGradient(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID ||
|
|
CheckPointer(m_cGradients) == POINTER_INVALID ||
|
|
CheckPointer(m_cScores) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF2) == POINTER_INVALID ||
|
|
CheckPointer(m_cQuerys) == POINTER_INVALID ||
|
|
CheckPointer(m_cKeys) == POINTER_INVALID ||
|
|
CheckPointer(m_cValues) == POINTER_INVALID ||
|
|
m_cOutputs.Total() != m_cGradients.Total())
|
|
return false;
|
|
//--- Масштабирование градиента ошибки
|
|
if(m_dStd[1] != 0 && m_cGradients.Scaling(1 / m_dStd[1]) <= 0)
|
|
return false;
|
|
//--- Проводим градиент ошибки через блок Feed Forward
|
|
if(!m_cFF2.CalcHiddenGradient(m_cFF1))
|
|
return false;
|
|
if(!m_cFF1.CalcHiddenGradient(m_cW0))
|
|
return false;
|
|
m_cW0.GetGradients().m_mMatrix += m_cGradients.m_mMatrix;
|
|
if(m_dStd[0] != 0)
|
|
m_cW0.GetGradients().m_mMatrix /= m_dStd[0];
|
|
if(m_cOpenCL && !m_cW0.GetGradients().BufferWrite())
|
|
return false;
|
|
//--- Распределения гралиента ошибки по головам внимания
|
|
if(!m_cW0.CalcHiddenGradient(m_cAttentionOut))
|
|
return false;
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
MATRIX attention_grad = m_cAttentionOut.GetGradients().m_mMatrix;
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
MATRIX gradients[];
|
|
MATRIX querys[], querys_grad;
|
|
MATRIX keys[], keys_grad;
|
|
MATRIX values[], values_grad;
|
|
if(!m_cQuerys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, querys) ||
|
|
!m_cKeys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, keys) ||
|
|
!m_cValues.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values) ||
|
|
!attention_grad.Vsplit(m_iHeads, gradients))
|
|
return false;
|
|
if(!querys_grad.Init(m_iHeads, m_iUnits * m_iKeysSize) ||
|
|
!keys_grad.Init(m_iHeads, m_iUnits * m_iKeysSize) ||
|
|
!values_grad.Init(m_iHeads, m_iUnits * m_iKeysSize))
|
|
return false;
|
|
//--- Распределение градиента на Values
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
if(!querys[head].Reshape(m_iUnits, m_iKeysSize) ||
|
|
!keys[head].Reshape(m_iUnits, m_iKeysSize) ||
|
|
!values[head].Reshape(m_iUnits, m_iKeysSize))
|
|
return false;
|
|
MATRIX score;
|
|
if(!score.Init(1, m_iUnits * m_iUnits) ||
|
|
!score.Row(m_cScores.m_mMatrix.Row(head), 0) ||
|
|
!score.Reshape(m_iUnits, m_iUnits))
|
|
return false;
|
|
MATRIX temp = score.Transpose().MatMul(gradients[head]);
|
|
if(!temp.Reshape(1, m_iUnits * m_iKeysSize))
|
|
return false;
|
|
if(!values_grad.Row(temp.Row(0), head))
|
|
return false;
|
|
//--- Распределение градиента на Querys и Keys
|
|
gradients[head] = gradients[head].MatMul(values[head].Transpose());
|
|
//---
|
|
for(int r = 0; r < m_iUnits; r++)
|
|
{
|
|
temp.Init(m_iUnits, m_iUnits);
|
|
temp.Identity();
|
|
VECTOR s = score.Row(r);
|
|
for(int c = 0; c < m_iUnits; c++)
|
|
if(!temp.Col((temp.Col(c) - s), c))
|
|
return false;
|
|
s = s.MatMul(temp);
|
|
if(!gradients[head].Row(s * gradients[head].Row(r) / sqrt(m_iKeysSize), r))
|
|
return false;
|
|
}
|
|
temp = gradients[head].MatMul(keys[head]);
|
|
if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
|
|
!querys_grad.Row(temp.Row(0), head))
|
|
return false;
|
|
temp = gradients[head].Transpose().MatMul(querys[head]);
|
|
if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
|
|
!keys_grad.Row(temp.Row(0), head))
|
|
return false;
|
|
}
|
|
//---
|
|
if(!querys_grad.Reshape(1, m_cQuerys.GetGradients().Total()) ||
|
|
!keys_grad.Reshape(1, m_cKeys.GetGradients().Total()) ||
|
|
!values_grad.Reshape(1, m_cValues.GetGradients().Total()))
|
|
return false;
|
|
m_cQuerys.GetGradients().m_mMatrix = querys_grad;
|
|
m_cKeys.GetGradients().m_mMatrix = keys_grad;
|
|
m_cValues.GetGradients().m_mMatrix = values_grad;
|
|
}
|
|
else // Блок OpenCL
|
|
{
|
|
//--- Создание буферов данных
|
|
if(m_cValues.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cValues.GetGradients().GetIndex() < 0)
|
|
return false;
|
|
if(m_cScores.GetIndex() < 0)
|
|
return false;
|
|
if(m_cAttentionOut.GetGradients().GetIndex() < 0)
|
|
return false;
|
|
if(m_cScoreGrad.GetIndex() < 0)
|
|
return false;
|
|
if(m_cScoreTemp.GetIndex() < 0)
|
|
return false;
|
|
//--- Передача параметров кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_outputs_grad, m_cAttentionOut.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores, m_cScores.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores_grad, m_cScoreGrad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores_temp, m_cScoreTemp.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_values, m_cValues.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_values_grad, m_cValues.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AttentionScoreGradients, def_attscr_window, m_iKeysSize))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int off_set[] = {0, 0};
|
|
int NDRange[] = {m_iUnits, m_iHeads};
|
|
if(!m_cOpenCL.Execute(def_k_AttentionScoreGradients, 2, off_set, NDRange))
|
|
return false;
|
|
//--- Загрузка результатов
|
|
if(!m_cValues.GetGradients().BufferRead())
|
|
return false;
|
|
//--- Создание буферов данных
|
|
if(m_cQuerys.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cQuerys.GetGradients().GetIndex() < 0)
|
|
return false;
|
|
if(m_cKeys.GetOutputs().GetIndex() < 0)
|
|
return false;
|
|
if(m_cKeys.GetGradients().GetIndex() < 0)
|
|
return false;
|
|
//--- Передача аргументов кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_keys, m_cKeys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_keys_grad, m_cKeys.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_querys, m_cQuerys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_querys_grad, m_cQuerys.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_scores_grad, m_cScoreGrad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_AttentionHiddenGradients, def_atthgr_key_size, m_iKeysSize))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
if(!m_cOpenCL.Execute(def_k_AttentionHiddenGradients, 2, off_set, NDRange))
|
|
return false;
|
|
//--- Загрузка результатов
|
|
if(!m_cQuerys.GetGradients().BufferRead())
|
|
return false;
|
|
}
|
|
//--- Перенос градиента ошибки на предыдущий слой
|
|
attention_grad = m_cW0.GetGradients().m_mMatrix;
|
|
if(!m_cValues.CalcHiddenGradient(prevLayer))
|
|
return false;
|
|
attention_grad += prevLayer.GetGradients().m_mMatrix;
|
|
if(!m_cQuerys.CalcHiddenGradient(prevLayer))
|
|
return false;
|
|
attention_grad += prevLayer.GetGradients().m_mMatrix;
|
|
if(!m_cKeys.CalcHiddenGradient(prevLayer))
|
|
return false;
|
|
prevLayer.GetGradients().m_mMatrix += attention_grad;
|
|
if(m_cOpenCL && !prevLayer.GetGradients().BufferWrite())
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиента ошибки до матриц весов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::CalcDeltaWeights(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(!m_cFF2.CalcDeltaWeights(m_cFF1))
|
|
return false;
|
|
if(!m_cFF1.CalcDeltaWeights(m_cW0))
|
|
return false;
|
|
if(!m_cW0.CalcDeltaWeights(m_cAttentionOut))
|
|
return false;
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cQuerys.CalcDeltaWeights(prevLayer))
|
|
return false;
|
|
if(CheckPointer(m_cKeys) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cKeys.CalcDeltaWeights(prevLayer))
|
|
return false;
|
|
if(CheckPointer(m_cValues) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cValues.CalcDeltaWeights(prevLayer))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод обновления матриц весовых коэффициентов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronAttention::UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(!m_cW0.UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод сохранения элементов класа в файл |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::Save(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronAttention::Save(file_handle))
|
|
return false;
|
|
//--- Сохраняем константы
|
|
if(FileWriteInteger(file_handle, m_iHeads) <= 0)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
return false;
|
|
if(!m_cW0.Save(file_handle))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод восстановления класса из сохранённых данных |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronMHAttention::Load(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronAttention::Load(file_handle))
|
|
return false;
|
|
//--- Загружаем константы
|
|
m_iHeads = FileReadInteger(file_handle);
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
{
|
|
m_cW0 = new CNeuronConv();
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(FileReadInteger(file_handle) != defNeuronConv || !m_cW0.Load(file_handle))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|