1006 lines
79 KiB
MQL5
1006 lines
79 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronGPT.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Подключаем библиотеки |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef ArrayLayers
|
|
#include "arraylayers.mqh"
|
|
#endif
|
|
//+------------------------------------------------------------------+
|
|
//| Class CNeuronGPT |
|
|
//| Назначение: Класс оргпнизации GPT блока |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronGPT : public CNeuronBase
|
|
{
|
|
protected:
|
|
CArrayLayers *m_cQuerys;
|
|
CArrayLayers *m_cKeys;
|
|
CArrayLayers *m_cValues;
|
|
CArrayLayers *m_cScores;
|
|
CBufferDouble *m_cScoreTemp;
|
|
CArrayLayers *m_cAttentionOut;
|
|
CArrayLayers *m_cW0;
|
|
CArrayLayers *m_cFF1;
|
|
CArrayLayers *m_cFF2;
|
|
//---
|
|
int m_iLayers;
|
|
int m_iWindow;
|
|
int m_iUnits;
|
|
int m_iKeysSize;
|
|
int m_iHeads;
|
|
double m_dStd[][2];
|
|
int m_iCurrentPosition;
|
|
|
|
bool CheckArrayLayers(CArrayLayers *&layers);
|
|
|
|
public:
|
|
CNeuronGPT(void);
|
|
~CNeuronGPT(void);
|
|
//---
|
|
virtual bool Init(CLayerDescription *description);
|
|
virtual bool SetOpenCL(CMyOpenCL *opencl);
|
|
virtual bool FeedForward(CNeuronBase *prevLayer);
|
|
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
|
|
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer);
|
|
virtual bool UpdateWeights(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
//---
|
|
virtual int GetUnits(void) const { return m_iUnits; }
|
|
virtual int GetLayers(void) const { return m_iLayers; }
|
|
//--- methods for working with files
|
|
virtual bool Save(const int file_handle);
|
|
virtual bool Load(const int file_handle);
|
|
//--- method of identifying the object
|
|
virtual int Type(void) const { return(defNeuronGPT); }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Конструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronGPT::CNeuronGPT(void) : m_iHeads(8),
|
|
m_iWindow(0),
|
|
m_iKeysSize(0),
|
|
m_iUnits(0),
|
|
m_iLayers(0),
|
|
m_iCurrentPosition(0)
|
|
{
|
|
m_cQuerys = new CArrayLayers();
|
|
m_cKeys = new CArrayLayers();
|
|
m_cValues = new CArrayLayers();
|
|
m_cScores = new CArrayLayers();
|
|
m_cAttentionOut = new CArrayLayers();
|
|
m_cW0 = new CArrayLayers();
|
|
m_cFF1 = new CArrayLayers();
|
|
m_cFF2 = new CArrayLayers();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Деструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronGPT::~CNeuronGPT(void)
|
|
{
|
|
if(CheckPointer(m_cQuerys) != POINTER_INVALID)
|
|
delete m_cQuerys;
|
|
if(CheckPointer(m_cKeys) != POINTER_INVALID)
|
|
delete m_cKeys;
|
|
if(CheckPointer(m_cValues) != POINTER_INVALID)
|
|
delete m_cValues;
|
|
if(CheckPointer(m_cScores) != POINTER_INVALID)
|
|
delete m_cScores;
|
|
if(CheckPointer(m_cScoreTemp) != POINTER_INVALID)
|
|
delete m_cScoreTemp;
|
|
if(CheckPointer(m_cAttentionOut) != POINTER_INVALID)
|
|
delete m_cAttentionOut;
|
|
if(CheckPointer(m_cW0) != POINTER_INVALID)
|
|
delete m_cW0;
|
|
if(CheckPointer(m_cFF1) != POINTER_INVALID)
|
|
delete m_cFF1;
|
|
if(CheckPointer(m_cFF2) != POINTER_INVALID)
|
|
delete m_cFF2;
|
|
m_iLayers = 0;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Мутод инициализации класса |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::Init(CLayerDescription *description)
|
|
{
|
|
//--- Проверяем исходные данные
|
|
if(CheckPointer(description) == POINTER_INVALID || description.type != Type() ||
|
|
description.count <= 0 || description.window <= 0 || description.window_out <= 0 ||
|
|
description.step <= 0 || description.layers <= 0)
|
|
return false;
|
|
//--- Сохраняем константы
|
|
m_iWindow = description.window;
|
|
m_iUnits = description.count;
|
|
m_iKeysSize = description.window_out;
|
|
m_iHeads = description.step;
|
|
m_iLayers = description.layers;
|
|
if(!ArrayResize(m_dStd, m_iLayers))
|
|
return false;
|
|
//--- Вызываем метод инициализации родительского класса
|
|
description.count *= m_iWindow;
|
|
description.window_out = 1;
|
|
description.window = 0;
|
|
if(!CNeuronBase::Init(description))
|
|
return false;
|
|
//--- Создаём динамические массивы для хранения указателей на объекты внутренних слоёв
|
|
if(!CheckArrayLayers(m_cQuerys))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cKeys))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cValues))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cScores))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cAttentionOut))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cW0))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cFF1))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cFF2))
|
|
return false;
|
|
//--- Запускаем цикл для создания объектов внутренних слоёв
|
|
for(int layer = 0; layer < m_iLayers; layer++)
|
|
{
|
|
//--- Создаём описание для внутренних нейронных слоёв
|
|
CLayerDescription *temp = new CLayerDescription();
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
|
return false;
|
|
temp.type = defNeuronBase;
|
|
temp.window = m_iWindow;
|
|
temp.count = (int)(3 * m_iKeysSize * m_iHeads);
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
temp.optimization = description.optimization;
|
|
//--- Инициализируем Querys
|
|
CNeuronBase *Querys = new CNeuronBase();
|
|
if(CheckPointer(Querys) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!Querys.Init(temp))
|
|
{
|
|
delete Querys;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cQuerys.Add(Querys))
|
|
{
|
|
delete Querys;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем Keys
|
|
CNeuronBase *Keys = new CNeuronBase();
|
|
if(CheckPointer(Keys) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.window = 0;
|
|
temp.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);
|
|
if(!Keys.Init(temp))
|
|
{
|
|
delete Keys;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cKeys.Add(Keys))
|
|
{
|
|
delete Keys;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем Values
|
|
CNeuronBase *Values = new CNeuronBase();
|
|
if(CheckPointer(Values) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!Values.Init(temp))
|
|
{
|
|
delete Values;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cValues.Add(Values))
|
|
{
|
|
delete Values;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем Scores
|
|
CNeuronBase *Scores = new CNeuronBase();
|
|
if(CheckPointer(Scores) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.count = (int)(m_iUnits * m_iHeads);
|
|
if(!Scores.Init(temp))
|
|
{
|
|
delete Scores;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cScores.Add(Scores))
|
|
{
|
|
delete Scores;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем AttentionOut
|
|
CNeuronBase *AttentionOut = new CNeuronBase();
|
|
if(CheckPointer(AttentionOut) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.count = (int)(m_iKeysSize * m_iHeads);
|
|
if(!AttentionOut.Init(temp))
|
|
{
|
|
delete AttentionOut;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cAttentionOut.Add(AttentionOut))
|
|
{
|
|
delete AttentionOut;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем W0
|
|
CNeuronBase *W0 = new CNeuronBase();
|
|
if(CheckPointer(W0) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.window = temp.count;
|
|
temp.count = m_iWindow;
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!W0.Init(temp))
|
|
{
|
|
delete W0;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cW0.Add(W0))
|
|
{
|
|
delete W0;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем FF1
|
|
CNeuronBase *FF1 = new CNeuronBase();
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.window = m_iWindow;
|
|
temp.count = temp.window * 4;
|
|
temp.activation = ACT_SWISH;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!FF1.Init(temp))
|
|
{
|
|
delete FF1;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cFF1.Add(FF1))
|
|
{
|
|
delete FF1;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
//--- Инициализируем FF2
|
|
CNeuronBase *FF2 = new CNeuronBase();
|
|
if(CheckPointer(FF2) == POINTER_INVALID)
|
|
{
|
|
delete temp;
|
|
return false;
|
|
}
|
|
temp.window = temp.count;
|
|
temp.count = m_iWindow;
|
|
temp.activation = ACT_None;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
if(!FF2.Init(temp))
|
|
{
|
|
delete FF2;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
if(!m_cFF2.Add(FF2))
|
|
{
|
|
delete FF2;
|
|
delete temp;
|
|
return false;
|
|
}
|
|
delete temp;
|
|
}
|
|
//--- Для исключениия копирования буферов осуществим их подмену
|
|
if(m_cFF2.Total() < m_iLayers)
|
|
return false;
|
|
if(CheckPointer(m_cOutputs) != POINTER_INVALID)
|
|
delete m_cOutputs;
|
|
CNeuronBase *temp = m_cFF2.At(m_iLayers - 1);
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
|
return false;
|
|
m_cOutputs = temp.GetOutputs();
|
|
if(CheckPointer(m_cGradients) != POINTER_INVALID)
|
|
delete m_cGradients;
|
|
m_cGradients = temp.GetGradients();
|
|
//---
|
|
SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод передачи указателя на объект OpenCL во все внутренние |
|
|
//| объекты |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::SetOpenCL(CMyOpenCL *opencl)
|
|
{
|
|
CNeuronBase::SetOpenCL(opencl);
|
|
if(CheckPointer(m_cQuerys) != POINTER_INVALID)
|
|
m_cQuerys.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cKeys) != POINTER_INVALID)
|
|
m_cKeys.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cValues) != POINTER_INVALID)
|
|
m_cValues.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cScores) != POINTER_INVALID)
|
|
m_cScores.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cScoreTemp) != POINTER_INVALID)
|
|
m_cScoreTemp.BufferCreate(m_cOpenCL);
|
|
if(CheckPointer(m_cAttentionOut) != POINTER_INVALID)
|
|
m_cAttentionOut.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cW0) != POINTER_INVALID)
|
|
m_cW0.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cFF1) != POINTER_INVALID)
|
|
m_cFF1.SetOpencl(m_cOpenCL);
|
|
if(CheckPointer(m_cFF2) != POINTER_INVALID)
|
|
m_cFF2.SetOpencl(m_cOpenCL);
|
|
//---
|
|
return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод прямого прохода |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::FeedForward(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
|
|
CheckPointer(m_cQuerys) == POINTER_INVALID ||
|
|
CheckPointer(m_cValues) == POINTER_INVALID ||
|
|
CheckPointer(m_cKeys) == POINTER_INVALID ||
|
|
CheckPointer(m_cScores) == POINTER_INVALID ||
|
|
CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
|
|
CheckPointer(m_cW0) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF1) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
return false;
|
|
//--- Увеличиваем указатель на текущий объект в стеке данных
|
|
m_iCurrentPosition++;
|
|
if(m_iCurrentPosition >= m_iUnits)
|
|
m_iCurrentPosition = 0;
|
|
//--- Запускаем цикл перебора всех внутренних слоёв
|
|
CNeuronBase *prevL = prevLayer;
|
|
for(int layer = 0; layer < m_iLayers; layer++)
|
|
{
|
|
CNeuronBase *Querys = m_cQuerys.At(layer);
|
|
if(CheckPointer(Querys) == POINTER_INVALID ||
|
|
!Querys.FeedForward(prevL))
|
|
return false;
|
|
CNeuronBase *Keys = m_cKeys.At(layer);
|
|
if(CheckPointer(Keys) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *Values = m_cValues.At(layer);
|
|
if(CheckPointer(Values) == POINTER_INVALID)
|
|
return false;
|
|
int shift_key = m_iCurrentPosition * m_iKeysSize * m_iHeads;
|
|
if(!Keys.GetOutputs().UpdateArray(shift_key, Querys.GetOutputs(), m_iKeysSize * m_iHeads, m_iKeysSize * m_iHeads))
|
|
return false;
|
|
if(!Values.GetOutputs().UpdateArray(shift_key, Querys.GetOutputs(), 2 * m_iKeysSize * m_iHeads, m_iKeysSize * m_iHeads))
|
|
return false;
|
|
//--- Инициализируем Scores
|
|
CNeuronBase *Scores = m_cScores.At(layer);
|
|
if(CheckPointer(Scores) == POINTER_INVALID)
|
|
return false;
|
|
//--- Инициализируем AttentionOut
|
|
CNeuronBase *AttentionOut = m_cAttentionOut.At(layer);
|
|
if(CheckPointer(AttentionOut) == POINTER_INVALID)
|
|
return false;
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
double summs[];
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
CBufferDouble *querys = Querys.GetOutputs();
|
|
CBufferDouble *keys = Keys.GetOutputs();
|
|
double scores[];
|
|
if(ArrayResize(scores, m_iUnits * m_iHeads) <= 0)
|
|
return false;
|
|
//--- Определяем Scores
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
int shift_query = head * m_iKeysSize;
|
|
double summ = 0;
|
|
for(int key = 0; key < m_iUnits; key++)
|
|
{
|
|
shift_key = key * m_iKeysSize * m_iHeads + head * m_iKeysSize;
|
|
int shift_score = head * m_iUnits + key;
|
|
double score = 0;
|
|
for(int i = 0; i < m_iKeysSize; i++)
|
|
score += querys.At(shift_query + i) * keys.At(shift_key + i);
|
|
scores[shift_score] = MathExp(score / MathSqrt(m_iKeysSize));
|
|
summ += scores[shift_score];
|
|
}
|
|
//--- Нормализуем Scores
|
|
if(summ == 0)
|
|
continue;
|
|
for(int key = 0; key < m_iUnits; key++)
|
|
scores[head * m_iUnits + key] /= summ;
|
|
}
|
|
if(!Scores.GetOutputs().AssignArray(scores))
|
|
return false;
|
|
//--- Выход блока внимания
|
|
int total = m_iKeysSize * m_iHeads;
|
|
if(ArrayResize(summs, total) < total)
|
|
return false;
|
|
if(ArrayInitialize(summs, 0) < total)
|
|
return false;
|
|
CBufferDouble *values = Values.GetOutputs();
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
for(int value = 0; value < m_iUnits; value++)
|
|
{
|
|
int shift_value = m_iKeysSize * (value * m_iHeads + head);
|
|
for(int pos = 0; pos < m_iKeysSize; pos++)
|
|
{
|
|
double val = values.At(shift_value + pos);
|
|
summs[m_iKeysSize * head + pos] += val * scores[m_iUnits * head + value];
|
|
}
|
|
}
|
|
}
|
|
if(!AttentionOut.GetOutputs().AssignArray(summs))
|
|
return false;
|
|
}
|
|
else // Блок OpenCL
|
|
{
|
|
//--- Создание буферов данных
|
|
if(Querys.GetOutputs().GetIndex() < 0 && !Querys.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(Keys.GetOutputs().GetIndex() < 0 && !Keys.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(Values.GetOutputs().GetIndex() < 0 && !Values.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
int scores = (int)MathPow(m_iUnits, 2) * m_iHeads;
|
|
if(Scores.GetOutputs().Total() != scores &&
|
|
!Scores.GetOutputs().BufferInit(scores, 0))
|
|
return false;
|
|
if(Scores.GetOutputs().GetIndex() < 0 && !Scores.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(AttentionOut.GetOutputs().GetIndex() < 0 && !AttentionOut.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача параметров кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_keys, Keys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_outputs, AttentionOut.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_querys, Querys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_scores, Scores.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_values, Values.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_key_size, m_iKeysSize))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_units, m_iUnits))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int off_set[] = {0};
|
|
int NDRange[] = {m_iHeads};
|
|
if(!m_cOpenCL.Execute(def_k_GPTFeedForward, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Считываниие результатов операций
|
|
if(!AttentionOut.GetOutputs().BufferRead())
|
|
return false;
|
|
if(!Scores.GetOutputs().BufferRead())
|
|
return false;
|
|
Querys.GetOutputs().BufferFree();
|
|
Keys.GetOutputs().BufferFree();
|
|
Values.GetOutputs().BufferFree();
|
|
Scores.GetOutputs().BufferFree();
|
|
prevL.GetOutputs().BufferFree();
|
|
}
|
|
//--- Взвешенный выхщд всех голов внимания
|
|
CNeuronBase *W0 = m_cW0.At(layer);
|
|
if(CheckPointer(W0) == POINTER_INVALID ||
|
|
!W0.FeedForward(AttentionOut))
|
|
return false;
|
|
int total = W0.GetOutputs().GetData(summs, false);
|
|
if(total <= 0)
|
|
return false;
|
|
if(ArrayResize(summs, total) != total)
|
|
return false;
|
|
//--- Суммируем с исходными данными и нормализуем
|
|
double mean = 0;
|
|
CBufferDouble *prev = prevL.GetOutputs();
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
summs[i] += prev.At(i);
|
|
mean += summs[i];
|
|
}
|
|
mean /= total;
|
|
m_dStd[layer][0] = MathStandardDeviation(summs);
|
|
for(int i = 0; i < total; i++)
|
|
summs[i] = (summs[i] - mean) / m_dStd[layer][0];
|
|
if(!W0.GetOutputs().AssignArray(summs))
|
|
return false;
|
|
//--- Прямой проход блока Feed Forward
|
|
CNeuronBase *FF1 = m_cFF1.At(layer);
|
|
if(CheckPointer(FF1) == POINTER_INVALID ||
|
|
!FF1.FeedForward(W0))
|
|
return false;
|
|
CNeuronBase *FF2 = m_cFF2.At(layer);
|
|
if(CheckPointer(FF2) == POINTER_INVALID ||
|
|
!FF2.FeedForward(FF1))
|
|
return false;
|
|
//--- Суммируем с выходом внимания и нормализуем
|
|
mean = 0;
|
|
prev = FF2.GetOutputs();
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
summs[i] += prev.At(i);
|
|
mean += summs[i];
|
|
}
|
|
mean /= total;
|
|
m_dStd[layer][1] = MathStandardDeviation(summs);
|
|
for(int i = 0; i < total; i++)
|
|
summs[i] = (summs[i] - mean) / m_dStd[layer][1];
|
|
if(!prev.AssignArray(summs))
|
|
return false;
|
|
prevL = FF2;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиента через скрытый слой |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::CalcHiddenGradient(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(m_cOutputs) == POINTER_INVALID ||
|
|
CheckPointer(m_cGradients) == POINTER_INVALID ||
|
|
CheckPointer(m_cScores) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF2) == POINTER_INVALID ||
|
|
CheckPointer(m_cFF1) == POINTER_INVALID ||
|
|
CheckPointer(m_cW0) == POINTER_INVALID ||
|
|
CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
|
|
CheckPointer(m_cQuerys) == POINTER_INVALID ||
|
|
CheckPointer(m_cKeys) == POINTER_INVALID ||
|
|
CheckPointer(m_cValues) == POINTER_INVALID ||
|
|
m_cOutputs.Total() != m_cGradients.Total())
|
|
return false;
|
|
//--- Запускаем цикл перебора всех внутренних слоёв в обратном порядке
|
|
for(int layer = m_iLayers - 1; layer >= 0; layer--)
|
|
{
|
|
CNeuronBase *FF2 = m_cFF2.At(layer);
|
|
if(CheckPointer(FF2) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *Gradients = FF2.GetGradients();
|
|
if(m_dStd[layer][1] != 0 && Gradients.Scaling(1 / m_dStd[layer][1]) <= 0)
|
|
return false;
|
|
//--- Проводим градиент через блок Feed Forward
|
|
CNeuronBase *FF1 = m_cFF1.At(layer);
|
|
if(!FF2.CalcHiddenGradient(FF1))
|
|
return false;
|
|
CNeuronBase *W0 = m_cW0.At(layer);
|
|
if(!FF1.CalcHiddenGradient(W0))
|
|
return false;
|
|
CBufferDouble *attention_grad = W0.GetGradients();
|
|
if(!attention_grad.SumArray(Gradients))
|
|
return false;
|
|
if(m_dStd[layer][0] != 0 && attention_grad.Scaling(1 / m_dStd[layer][0]) <= 0)
|
|
return false;
|
|
//--- Распределеяем градиент ошибки по головам внимания
|
|
CNeuronBase *AttentionOut = m_cAttentionOut.At(layer);
|
|
if(!W0.CalcHiddenGradient(AttentionOut))
|
|
return false;
|
|
//--- Получаем указатели на объекты Querys, Keys, Values
|
|
CNeuronBase *Querys = m_cQuerys.At(layer);
|
|
if(CheckPointer(Querys) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *Keys = m_cKeys.At(layer);
|
|
if(CheckPointer(Keys) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *Values = m_cValues.At(layer);
|
|
if(CheckPointer(Values) == POINTER_INVALID)
|
|
return false;
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
attention_grad = AttentionOut.GetGradients();
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
int total = attention_grad.Total();
|
|
double values[];
|
|
double gradients[];
|
|
if(ArrayResize(values, total) < total || attention_grad.GetData(gradients, false) < total)
|
|
return false;
|
|
if(ArrayInitialize(values, 0) < total)
|
|
return false;
|
|
//--- Распределение градиента на Values
|
|
AttentionOut = m_cScores.At(layer);
|
|
if(CheckPointer(AttentionOut) == POINTER_INVALID ||
|
|
CheckPointer(AttentionOut.GetOutputs()) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *Scores = AttentionOut.GetOutputs();
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
double score = Scores.At(m_iUnits * head + m_iCurrentPosition);
|
|
for(int i = 0; i < m_iKeysSize; i++)
|
|
values[m_iKeysSize * head + i] += gradients[m_iKeysSize * head + i] * score;
|
|
}
|
|
if(!Querys.GetGradients().UpdateArray(2 * m_iKeysSize * m_iHeads, values, 0, m_iKeysSize * m_iHeads))
|
|
return false;
|
|
//--- Распределение градиента на Querys и Keys
|
|
if(Values.GetOutputs().GetData(values, false) <= 0)
|
|
return false;
|
|
double querys[], querys_grad[];
|
|
double keys[];
|
|
int keys_total = m_iKeysSize * m_iHeads;
|
|
if(Querys.GetOutputs().GetData(querys, false) < 3 * keys_total)
|
|
return false;
|
|
if(Keys.GetOutputs().GetData(keys, false) < keys_total * m_iUnits)
|
|
return false;
|
|
if(ArrayResize(querys_grad, 2 * keys_total) <= 0)
|
|
return false;
|
|
if(ArrayInitialize(querys_grad, 0) <= 0)
|
|
return false;
|
|
double score_grad[];
|
|
if(ArrayResize(score_grad, m_iUnits) <= 0)
|
|
return false;
|
|
for(int head = 0; head < m_iHeads; head++)
|
|
{
|
|
if(ArrayInitialize(score_grad, 0) <= 0)
|
|
return false;
|
|
int shift_grad = m_iKeysSize * head;
|
|
for(int k = 0; k < m_iUnits; k++)
|
|
{
|
|
for(int i = 0; i < m_iKeysSize; i++)
|
|
score_grad[k] += gradients[shift_grad + i] * values[m_iKeysSize * (k * m_iHeads + head) + i];
|
|
}
|
|
//---
|
|
for(int k = 0; k < m_iUnits; k++)
|
|
{
|
|
double score = Scores.At(m_iUnits * head + k);
|
|
if(score == 0)
|
|
continue;
|
|
double grad = 0;
|
|
for(int i = 0; i < m_iUnits; i++)
|
|
grad += Scores.At(m_iUnits * head + i) * ((int)(i == k) - score) * score_grad[i];
|
|
grad /= MathSqrt(m_iKeysSize);
|
|
//---
|
|
int shift_key = m_iKeysSize * (k * m_iHeads + head);
|
|
for(int i = 0; i < m_iKeysSize; i++)
|
|
{
|
|
querys_grad[shift_grad + i] += grad * keys[shift_key + i];
|
|
if(k == m_iCurrentPosition)
|
|
querys_grad[m_iKeysSize * (m_iHeads + head) + i ] += grad * querys[shift_grad + i];
|
|
}
|
|
}
|
|
}
|
|
if(!Querys.GetGradients().UpdateArray(0, querys_grad, 0, 2 * keys_total))
|
|
return false;
|
|
}
|
|
else // Блок OpenCL
|
|
{
|
|
//--- Создание буферов данных
|
|
if(Values.GetOutputs().GetIndex() < 0 && !Values.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(Querys.GetGradients().GetIndex() < 0 && !Querys.GetGradients().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
CNeuronBase *Scores = m_cScores.At(layer);
|
|
if(CheckPointer(Scores) == POINTER_INVALID)
|
|
return false;
|
|
if(Scores.GetOutputs().GetIndex() < 0 && !Scores.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(attention_grad.GetIndex() < 0 && !attention_grad.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(CheckPointer(Scores.GetGradients()) == POINTER_INVALID)
|
|
{
|
|
return false;
|
|
}
|
|
if(Scores.GetGradients().Total() != Scores.GetOutputs().Total() &&
|
|
!Scores.GetGradients().BufferInit(Scores.GetOutputs().Total(), 0))
|
|
return false;
|
|
if(Scores.GetGradients().GetIndex() < 0 && !Scores.GetGradients().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(m_cScoreTemp) == POINTER_INVALID)
|
|
{
|
|
m_cScoreTemp = new CBufferDouble();
|
|
if(CheckPointer(m_cScoreTemp) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(m_cScoreTemp.Total() != Scores.GetGradients().Total() &&
|
|
!m_cScoreTemp.BufferInit(Scores.GetGradients().Total(), 0))
|
|
return false;
|
|
if(m_cScoreTemp.GetIndex() < 0 && !m_cScoreTemp.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передача параметров кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_outputs_grad, attention_grad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores, Scores.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_grad, Scores.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_temp, m_cScoreTemp.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values, Values.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values_grad, Querys.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_window, m_iKeysSize))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_units, m_iUnits))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_current, m_iCurrentPosition))
|
|
return false;
|
|
//--- Постановка кернела в очередь выполнения
|
|
int off_set[] = {0};
|
|
int NDRange[] = {m_iHeads};
|
|
if(!m_cOpenCL.Execute(def_k_GPTScoreGradients, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Загрузка результатов
|
|
if(!Querys.GetGradients().BufferRead())
|
|
return false;
|
|
Values.GetOutputs().BufferFree();
|
|
Scores.GetOutputs().BufferFree();
|
|
m_cScoreTemp.BufferFree();
|
|
AttentionOut.GetOutputs().BufferFree();
|
|
//---
|
|
if(Querys.GetOutputs().GetIndex() < 0 && !Querys.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(Keys.GetOutputs().GetIndex() < 0 && !Keys.GetOutputs().BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_keys, Keys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys, Querys.GetOutputs().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys_grad, Querys.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_scores_grad, Scores.GetGradients().GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_key_size, m_iKeysSize))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_units, m_iUnits))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_current, m_iCurrentPosition))
|
|
return false;
|
|
if(!m_cOpenCL.Execute(def_k_GPTHiddenGradients, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Загрузка результатов
|
|
if(!Querys.GetGradients().BufferRead())
|
|
return false;
|
|
//---
|
|
Scores.GetGradients().BufferFree();
|
|
Keys.GetOutputs().BufferFree();
|
|
Querys.GetOutputs().BufferFree();
|
|
}
|
|
//--- Перенос градиента ошибки на предыдущий слой
|
|
CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1));
|
|
if(!Querys.CalcHiddenGradient(prevL))
|
|
return false;
|
|
if(!prevL.GetGradients().SumArray(W0.GetGradients()))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиентов ошибки до матриц весовых |
|
|
//| коэффициентов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::CalcDeltaWeights(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
|
|
return false;
|
|
//--- В цикле вызываем аналогичный метод для каждого внутреннего объекта
|
|
for(int layer = 0; layer < m_iLayers; layer++)
|
|
{
|
|
if(CheckPointer(m_cFF2.At(layer)) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *temp = m_cFF2.At(layer);
|
|
if(!temp.CalcDeltaWeights(m_cFF1.At(layer)))
|
|
return false;
|
|
temp = m_cFF1.At(layer);
|
|
if(!temp.CalcDeltaWeights(m_cW0.At(layer)))
|
|
return false;
|
|
temp = m_cW0.At(layer);
|
|
if(!temp.CalcDeltaWeights(m_cAttentionOut.At(layer)))
|
|
return false;
|
|
temp = m_cQuerys.At(layer);
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1));
|
|
if(!temp.CalcDeltaWeights(prevL))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод обновления параметров матрицы весовых коэффициентов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
|
|
return false;
|
|
//--- В цикле вызываем аналогичный метод для каждого внутреннего объекта
|
|
for(int layer = 0; layer < m_iLayers; layer++)
|
|
{
|
|
CNeuronBase *temp = m_cFF2.At(layer);
|
|
if(CheckPointer(temp) == POINTER_INVALID ||
|
|
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
temp = m_cFF1.At(layer);
|
|
if(CheckPointer(temp) == POINTER_INVALID ||
|
|
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
temp = m_cW0.At(layer);
|
|
if(CheckPointer(temp) == POINTER_INVALID ||
|
|
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
temp = m_cQuerys.At(layer);
|
|
if(CheckPointer(temp) == POINTER_INVALID ||
|
|
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод сохранения элементов класса в файл |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::Save(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronBase::Save(file_handle))
|
|
return false;
|
|
//--- Сохраняем константы
|
|
if(FileWriteInteger(file_handle, m_iLayers) <= 0)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, m_iWindow) <= 0)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, m_iKeysSize) <= 0)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, m_iHeads) <= 0)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, m_iUnits) <= 0)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, m_iCurrentPosition) <= 0)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех колекций внутренних слоёв
|
|
if(CheckPointer(m_cQuerys) == POINTER_INVALID ||
|
|
!m_cQuerys.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cKeys) == POINTER_INVALID ||
|
|
!m_cKeys.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cValues) == POINTER_INVALID ||
|
|
!m_cValues.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cScores) == POINTER_INVALID ||
|
|
!m_cScores.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
|
|
!m_cAttentionOut.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cW0) == POINTER_INVALID ||
|
|
!m_cW0.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cFF1) == POINTER_INVALID ||
|
|
!m_cFF1.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(m_cFF2) == POINTER_INVALID ||
|
|
!m_cFF2.Save(file_handle))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод восстановления работы класса из файла |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::Load(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronBase::Load(file_handle))
|
|
return false;
|
|
//--- Считываем константы из файла
|
|
m_iLayers = FileReadInteger(file_handle);
|
|
m_iWindow = FileReadInteger(file_handle);
|
|
m_iKeysSize = FileReadInteger(file_handle);
|
|
m_iHeads = FileReadInteger(file_handle);
|
|
m_iUnits = FileReadInteger(file_handle);
|
|
m_iCurrentPosition = FileReadInteger(file_handle);
|
|
if(ArrayResize(m_dStd, m_iLayers) <= 0)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех колекций внутренних слоёв
|
|
if(!CheckArrayLayers(m_cQuerys) ||
|
|
!m_cQuerys.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cKeys) ||
|
|
!m_cKeys.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cValues) ||
|
|
!m_cValues.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cScores) ||
|
|
!m_cScores.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cAttentionOut) ||
|
|
!m_cAttentionOut.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cW0) ||
|
|
!m_cW0.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cFF1) ||
|
|
!m_cFF1.Load(file_handle))
|
|
return false;
|
|
if(!CheckArrayLayers(m_cFF2) ||
|
|
!m_cFF2.Load(file_handle))
|
|
return false;
|
|
//--- Осуществляем подмену буферов данных для исключения излишнего копирования
|
|
CNeuronBase *last=m_cFF2.At(m_cFF2.Total()-1);
|
|
if(CheckPointer(last)==POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(m_cOutputs)!=POINTER_INVALID)
|
|
delete m_cOutputs;
|
|
m_cOutputs=last.GetOutputs();
|
|
if(CheckPointer(m_cGradients)!=POINTER_INVALID)
|
|
delete m_cGradients;
|
|
m_cGradients=last.GetGradients();
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод проверки актуальности указателя на объект коллекции |
|
|
//| нейроных слоёв |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronGPT::CheckArrayLayers(CArrayLayers *&layers)
|
|
{
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
layers = new CArrayLayers();
|
|
//---
|
|
return CheckPointer(layers) != POINTER_INVALID;
|
|
}
|
|
//+------------------------------------------------------------------+
|