dng/NeuroNetworksBook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
NeuroNetworksBook/Include/realization/neurongpt.mqh
super.admin 4a9222852c convert
2025-05-30 16:12:34 +02:00

919 lines
70 KiB
MQL5
Raw Permalink Blame History

//+------------------------------------------------------------------+
//| NeuronGPT.mqh |
//| Copyright 2021, MetaQuotes Ltd. |
//| https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link "https://www.mql5.com"
//+------------------------------------------------------------------+
//| Подключаем библиотеки |
//+------------------------------------------------------------------+
#ifndef ArrayLayers
#include "arraylayers.mqh"
#endif
//+------------------------------------------------------------------+
//| Class CNeuronGPT |
//| Назначение: Класс оргпнизации GPT блока |
//+------------------------------------------------------------------+
class CNeuronGPT : public CNeuronBase
{
protected:
CArrayLayers *m_cQuerys;
CArrayLayers *m_cKeys;
CArrayLayers *m_cValues;
CArrayLayers *m_cScores;
CBufferDouble *m_cScoreTemp;
CArrayLayers *m_cAttentionOut;
CArrayLayers *m_cW0;
CArrayLayers *m_cFF1;
CArrayLayers *m_cFF2;
//---
int m_iLayers;
int m_iWindow;
int m_iUnits;
int m_iKeysSize;
int m_iHeads;
double m_dStd[][2];
int m_iCurrentPosition;
bool CheckArrayLayers(CArrayLayers *&layers);
public:
CNeuronGPT(void);
~CNeuronGPT(void);
//---
virtual bool Init(CLayerDescription *desc);
virtual bool SetOpenCL(CMyOpenCL *opencl);
virtual bool FeedForward(CNeuronBase *prevLayer);
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer);
virtual bool UpdateWeights(int batch_size, double learningRate,
double &Beta[], double &Lambda[]);
//---
virtual int GetUnits(void) const { return m_iUnits; }
virtual int GetLayers(void) const { return m_iLayers; }
//--- methods for working with files
virtual bool Save(const int file_handle);
virtual bool Load(const int file_handle);
//--- method of identifying the object
virtual int Type(void) const { return(defNeuronGPT); }
};
//+------------------------------------------------------------------+
//| Конструктор класса |
//+------------------------------------------------------------------+
CNeuronGPT::CNeuronGPT(void) : m_iHeads(8),
m_iWindow(0),
m_iKeysSize(0),
m_iUnits(0),
m_iLayers(0),
m_iCurrentPosition(0)
{
m_cQuerys = new CArrayLayers();
m_cKeys = new CArrayLayers();
m_cValues = new CArrayLayers();
m_cScores = new CArrayLayers();
m_cAttentionOut = new CArrayLayers();
m_cW0 = new CArrayLayers();
m_cFF1 = new CArrayLayers();
m_cFF2 = new CArrayLayers();
}
//+------------------------------------------------------------------+
//| Деструктор класса |
//+------------------------------------------------------------------+
CNeuronGPT::~CNeuronGPT(void)
{
if(m_cQuerys)
delete m_cQuerys;
if(m_cKeys)
delete m_cKeys;
if(m_cValues)
delete m_cValues;
if(m_cScores)
delete m_cScores;
if(m_cScoreTemp)
delete m_cScoreTemp;
if(m_cAttentionOut)
delete m_cAttentionOut;
if(m_cW0)
delete m_cW0;
if(m_cFF1)
delete m_cFF1;
if(m_cFF2)
delete m_cFF2;
m_iLayers = 0;
}
//+------------------------------------------------------------------+
//| Мутод инициализации класса |
//+------------------------------------------------------------------+
bool CNeuronGPT::Init(CLayerDescription *desc)
{
//--- Проверяем исходные данные
if(!desc || desc.type != Type() || desc.count <= 0 || desc.window <= 0 || desc.window_out <= 0 || desc.step <= 0 || desc.layers <= 0)
return false;
//--- Сохраняем константы
m_iWindow = desc.window;
m_iUnits = desc.count;
m_iKeysSize = desc.window_out;
m_iHeads = desc.step;
m_iLayers = desc.layers;
if(!ArrayResize(m_dStd, m_iLayers))
return false;
//--- Вызываем метод инициализации родительского класса
desc.count *= m_iWindow;
desc.window_out = 1;
desc.window = 0;
if(!CNeuronBase::Init(desc))
return false;
//--- Создаём динамические массивы для хранения указателей на объекты внутренних слоёв
if(!CheckArrayLayers(m_cQuerys))
return false;
if(!CheckArrayLayers(m_cKeys))
return false;
if(!CheckArrayLayers(m_cValues))
return false;
if(!CheckArrayLayers(m_cScores))
return false;
if(!CheckArrayLayers(m_cAttentionOut))
return false;
if(!CheckArrayLayers(m_cW0))
return false;
if(!CheckArrayLayers(m_cFF1))
return false;
if(!CheckArrayLayers(m_cFF2))
return false;
//--- Запускаем цикл для создания объектов внутренних слоёв
for(int layer = 0; layer < m_iLayers; layer++)
{
//--- Создаём описание для внутренних нейронных слоёв
CLayerDescription *temp = new CLayerDescription();
if(!temp)
return false;
temp.type = defNeuronBase;
temp.window = m_iWindow;
temp.count = (int)(3 * m_iKeysSize * m_iHeads);
temp.activation = ACT_None;
temp.activation_params[0] = 1;
temp.activation_params[1] = 0;
temp.optimization = desc.optimization;
//--- Инициализируем Querys
CNeuronBase *Querys = new CNeuronBase();
if(!Querys)
{
delete temp;
return false;
}
if(!Querys.Init(temp))
{
delete Querys;
delete temp;
return false;
}
if(!m_cQuerys.Add(Querys))
{
delete Querys;
delete temp;
return false;
}
//--- Инициализируем Keys
CNeuronBase *Keys = new CNeuronBase();
if(!Keys)
{
delete temp;
return false;
}
temp.window = 0;
temp.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);
if(!Keys.Init(temp))
{
delete Keys;
delete temp;
return false;
}
if(!Keys.GetOutputs().m_mMatrix.Reshape(m_iUnits, m_iKeysSize * m_iHeads))
return false;
if(!m_cKeys.Add(Keys))
{
delete Keys;
delete temp;
return false;
}
//--- Инициализируем Values
CNeuronBase *Values = new CNeuronBase();
if(!Values)
{
delete temp;
return false;
}
if(!Values.Init(temp))
{
delete Values;
delete temp;
return false;
}
if(!Values.GetOutputs().m_mMatrix.Reshape(m_iUnits, m_iKeysSize * m_iHeads))
return false;
if(!m_cValues.Add(Values))
{
delete Values;
delete temp;
return false;
}
//--- Инициализируем Scores
CNeuronBase *Scores = new CNeuronBase();
if(CheckPointer(Scores) == POINTER_INVALID)
{
delete temp;
return false;
}
temp.count = (int)(m_iUnits * m_iHeads);
if(!Scores.Init(temp))
{
delete Scores;
delete temp;
return false;
}
if(!Scores.GetOutputs().m_mMatrix.Reshape(m_iHeads, m_iUnits))
return false;
if(!m_cScores.Add(Scores))
{
delete Scores;
delete temp;
return false;
}
//--- Инициализируем AttentionOut
CNeuronBase *AttentionOut = new CNeuronBase();
if(CheckPointer(AttentionOut) == POINTER_INVALID)
{
delete temp;
return false;
}
temp.count = (int)(m_iKeysSize * m_iHeads);
if(!AttentionOut.Init(temp))
{
delete AttentionOut;
delete temp;
return false;
}
if(!m_cAttentionOut.Add(AttentionOut))
{
delete AttentionOut;
delete temp;
return false;
}
//--- Инициализируем W0
CNeuronBase *W0 = new CNeuronBase();
if(CheckPointer(W0) == POINTER_INVALID)
{
delete temp;
return false;
}
temp.window = temp.count;
temp.count = m_iWindow;
temp.activation = ACT_None;
temp.activation_params[0] = 1;
temp.activation_params[1] = 0;
if(!W0.Init(temp))
{
delete W0;
delete temp;
return false;
}
if(!m_cW0.Add(W0))
{
delete W0;
delete temp;
return false;
}
//--- Инициализируем FF1
CNeuronBase *FF1 = new CNeuronBase();
if(CheckPointer(m_cFF1) == POINTER_INVALID)
{
delete temp;
return false;
}
temp.window = m_iWindow;
temp.count = temp.window * 4;
temp.activation = ACT_SWISH;
temp.activation_params[0] = 1;
temp.activation_params[1] = 0;
if(!FF1.Init(temp))
{
delete FF1;
delete temp;
return false;
}
if(!m_cFF1.Add(FF1))
{
delete FF1;
delete temp;
return false;
}
//--- Инициализируем FF2
CNeuronBase *FF2 = new CNeuronBase();
if(CheckPointer(FF2) == POINTER_INVALID)
{
delete temp;
return false;
}
temp.window = temp.count;
temp.count = m_iWindow;
temp.activation = ACT_None;
temp.activation_params[0] = 1;
temp.activation_params[1] = 0;
if(!FF2.Init(temp))
{
delete FF2;
delete temp;
return false;
}
if(!m_cFF2.Add(FF2))
{
delete FF2;
delete temp;
return false;
}
delete temp;
}
//--- Для исключениия копирования буферов осуществим их подмену
if(m_cFF2.Total() < m_iLayers)
return false;
if(CheckPointer(m_cOutputs) != POINTER_INVALID)
delete m_cOutputs;
CNeuronBase *temp = m_cFF2.At(m_iLayers - 1);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
m_cOutputs = temp.GetOutputs();
if(CheckPointer(m_cGradients) != POINTER_INVALID)
delete m_cGradients;
m_cGradients = temp.GetGradients();
//---
SetOpenCL(m_cOpenCL);
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод передачи указателя на объект OpenCL во все внутренние |
//| объекты |
//+------------------------------------------------------------------+
bool CNeuronGPT::SetOpenCL(CMyOpenCL *opencl)
{
CNeuronBase::SetOpenCL(opencl);
if(CheckPointer(m_cQuerys) != POINTER_INVALID)
m_cQuerys.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cKeys) != POINTER_INVALID)
m_cKeys.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cValues) != POINTER_INVALID)
m_cValues.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cScores) != POINTER_INVALID)
m_cScores.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cScoreTemp) != POINTER_INVALID)
m_cScoreTemp.BufferCreate(m_cOpenCL);
if(CheckPointer(m_cAttentionOut) != POINTER_INVALID)
m_cAttentionOut.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cW0) != POINTER_INVALID)
m_cW0.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cFF1) != POINTER_INVALID)
m_cFF1.SetOpencl(m_cOpenCL);
if(CheckPointer(m_cFF2) != POINTER_INVALID)
m_cFF2.SetOpencl(m_cOpenCL);
//---
return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
}
//+------------------------------------------------------------------+
//| Метод прямого прохода |
//+------------------------------------------------------------------+
bool CNeuronGPT::FeedForward(CNeuronBase *prevLayer)
{
//--- Проверяем актуальность всех объектов
if(CheckPointer(prevLayer) == POINTER_INVALID ||
CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
CheckPointer(m_cQuerys) == POINTER_INVALID ||
CheckPointer(m_cValues) == POINTER_INVALID ||
CheckPointer(m_cKeys) == POINTER_INVALID ||
CheckPointer(m_cScores) == POINTER_INVALID ||
CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
CheckPointer(m_cW0) == POINTER_INVALID ||
CheckPointer(m_cFF1) == POINTER_INVALID ||
CheckPointer(m_cFF2) == POINTER_INVALID)
return false;
//--- Увеличиваем указатель на текущий объект в стеке данных
m_iCurrentPosition++;
if(m_iCurrentPosition >= m_iUnits)
m_iCurrentPosition = 0;
//--- Запускаем цикл перебора всех внутренних слоёв
CNeuronBase *prevL = prevLayer;
for(int layer = 0; layer < m_iLayers; layer++)
{
CNeuronBase *Querys = m_cQuerys.At(layer);
if(CheckPointer(Querys) == POINTER_INVALID ||
!Querys.FeedForward(prevL))
return false;
CNeuronBase *Keys = m_cKeys.At(layer);
if(CheckPointer(Keys) == POINTER_INVALID)
return false;
CNeuronBase *Values = m_cValues.At(layer);
if(CheckPointer(Values) == POINTER_INVALID)
return false;
MATRIX array[];
if(!Querys.GetOutputs().m_mMatrix.Vsplit(3, array))
return false;
if(!Keys.GetOutputs().m_mMatrix.Row(array[1].Row(0), m_iCurrentPosition))
return false;
if(!Values.GetOutputs().m_mMatrix.Row(array[2].Row(0), m_iCurrentPosition))
return false;
//--- Инициализируем Scores
CNeuronBase *Scores = m_cScores.At(layer);
if(CheckPointer(Scores) == POINTER_INVALID)
return false;
//--- Инициализируем AttentionOut
CNeuronBase *AttentionOut = m_cAttentionOut.At(layer);
if(CheckPointer(AttentionOut) == POINTER_INVALID)
return false;
//--- Разветвление алгоритма по вычислительному устройству
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
{
MATRIX out;
if(!out.Init(m_iHeads, m_iKeysSize))
return false;
MATRIX array_keys[], array_values[];
MATRIX array_querys[];
MATRIX keys = Keys.GetOutputs().m_mMatrix;
MATRIX values = Values.GetOutputs().m_mMatrix;
if(!array[0].Vsplit(m_iHeads, array_querys))
return false;
if(!keys.Reshape(m_iUnits, m_iHeads * m_iKeysSize))
return false;
if(!keys.Vsplit(m_iHeads, array_keys))
return false;
if(!values.Reshape(m_iUnits, m_iHeads * m_iKeysSize))
return false;
if(!values.Vsplit(m_iHeads, array_values))
return false;
//--- Определяем Scores
for(int head = 0; head < m_iHeads; head++)
{
MATRIX score = array_querys[head].MatMul(array_keys[head].Transpose()) / sqrt(m_iKeysSize);
for(int s = 0; s < m_iUnits; s++)
score[0, s] = MathExp(score[0, s]);
//--- Нормализуем Scores
score = score / (score.Sum() + 1e-8);
if(!Scores.GetOutputs().m_mMatrix.Row(score.Row(0), head))
return false;
MATRIX o = score.MatMul(array_values[head].Transpose());
if(!o.Reshape(1, m_iKeysSize) ||
!out.Row(o.Row(0), head))
return false;
}
if(!out.Reshape(1, m_iHeads * m_iKeysSize))
return false;
AttentionOut.GetOutputs().m_mMatrix = out;
}
else // Блок OpenCL
{
//--- Создание буферов данных
if(Querys.GetOutputs().GetIndex() < 0)
return false;
if(Keys.GetOutputs().GetIndex() < 0)
return false;
if(Values.GetOutputs().GetIndex() < 0)
return false;
if(Scores.GetOutputs().GetIndex() < 0)
return false;
if(AttentionOut.GetOutputs().GetIndex() < 0)
return false;
//--- Передача параметров кернелу
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_keys, Keys.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_outputs, AttentionOut.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_querys, Querys.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_scores, Scores.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_values, Values.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_key_size, m_iKeysSize))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_units, m_iUnits))
return false;
//--- Постановка кернела в очередь выполнения
int off_set[] = {0};
int NDRange[] = {m_iHeads};
if(!m_cOpenCL.Execute(def_k_GPTFeedForward, 1, off_set, NDRange))
return false;
//--- Считываниие результатов операций
if(!AttentionOut.GetOutputs().BufferRead())
return false;
}
//--- Взвешенный выхщд всех голов внимания
CNeuronBase *W0 = m_cW0.At(layer);
if(CheckPointer(W0) == POINTER_INVALID ||
!W0.FeedForward(AttentionOut))
return false;
//--- Суммируем с исходными данными и нормализуем
W0.GetOutputs().m_mMatrix += prevL.GetOutputs().m_mMatrix;
double mean = W0.GetOutputs().m_mMatrix.Mean();
m_dStd[layer][0] = W0.GetOutputs().m_mMatrix.Std();
W0.GetOutputs().m_mMatrix = (W0.GetOutputs().m_mMatrix - mean) / m_dStd[layer][0];
if(m_cOpenCL && !W0.GetOutputs().BufferWrite())
return false;
//--- Прямой проход блока Feed Forward
CNeuronBase *FF1 = m_cFF1.At(layer);
if(CheckPointer(FF1) == POINTER_INVALID ||
!FF1.FeedForward(W0))
return false;
CNeuronBase *FF2 = m_cFF2.At(layer);
if(CheckPointer(FF2) == POINTER_INVALID ||
!FF2.FeedForward(FF1))
return false;
//--- Суммируем с выходом внимания и нормализуем
CBufferDouble *prev = FF2.GetOutputs();
prev.m_mMatrix += W0.GetOutputs().m_mMatrix;
mean = prev.m_mMatrix.Mean();
m_dStd[layer][1] = prev.m_mMatrix.Std();
prev.m_mMatrix = (prev.m_mMatrix - mean) / m_dStd[layer][1];
if(m_cOpenCL && !prev.BufferWrite())
return false;
prevL = FF2;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод распределения градиента через скрытый слой |
//+------------------------------------------------------------------+
bool CNeuronGPT::CalcHiddenGradient(CNeuronBase *prevLayer)
{
//--- Проверяем актуальность всех объектов
if(CheckPointer(m_cOutputs) == POINTER_INVALID ||
CheckPointer(m_cGradients) == POINTER_INVALID ||
CheckPointer(m_cScores) == POINTER_INVALID ||
CheckPointer(m_cFF2) == POINTER_INVALID ||
CheckPointer(m_cFF1) == POINTER_INVALID ||
CheckPointer(m_cW0) == POINTER_INVALID ||
CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
CheckPointer(m_cQuerys) == POINTER_INVALID ||
CheckPointer(m_cKeys) == POINTER_INVALID ||
CheckPointer(m_cValues) == POINTER_INVALID ||
m_cOutputs.Total() != m_cGradients.Total())
return false;
//--- Запускаем цикл перебора всех внутренних слоёв в обратном порядке
for(int layer = m_iLayers - 1; layer >= 0; layer--)
{
CNeuronBase *FF2 = m_cFF2.At(layer);
if(CheckPointer(FF2) == POINTER_INVALID)
return false;
CBufferDouble *Gradients = FF2.GetGradients();
if(m_dStd[layer][1] != 0 && Gradients.Scaling(1 / m_dStd[layer][1]) <= 0)
return false;
//--- Проводим градиент через блок Feed Forward
CNeuronBase *FF1 = m_cFF1.At(layer);
if(!FF2.CalcHiddenGradient(FF1))
return false;
CNeuronBase *W0 = m_cW0.At(layer);
if(!FF1.CalcHiddenGradient(W0))
return false;
CBufferDouble *attention_grad = W0.GetGradients();
if(!attention_grad.SumArray(Gradients))
return false;
//--- Инициализируем Scores
CNeuronBase *Scores = m_cScores.At(layer);
if(CheckPointer(Scores) == POINTER_INVALID)
return false;
if(m_dStd[layer][0] != 0 && attention_grad.Scaling(1 / m_dStd[layer][0]) <= 0)
return false;
//--- Распределеяем градиент ошибки по головам внимания
CNeuronBase *AttentionOut = m_cAttentionOut.At(layer);
if(!W0.CalcHiddenGradient(AttentionOut))
return false;
//--- Получаем указатели на объекты Querys, Keys, Values
CNeuronBase *Querys = m_cQuerys.At(layer);
if(CheckPointer(Querys) == POINTER_INVALID)
return false;
CNeuronBase *Keys = m_cKeys.At(layer);
if(CheckPointer(Keys) == POINTER_INVALID)
return false;
CNeuronBase *Values = m_cValues.At(layer);
if(CheckPointer(Values) == POINTER_INVALID)
return false;
//--- Разветвление алгоритма по вычислительному устройству
attention_grad = AttentionOut.GetGradients();
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
{
MATRIX gradients[];
if(!attention_grad.m_mMatrix.Vsplit(m_iHeads, gradients))
return false;
//--- Распределение градиента на Values
if(!Querys.GetGradients().m_mMatrix.Reshape(3, m_iHeads * m_iKeysSize))
return false;
MATRIX values[];
if(!Values.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values))
return false;
MATRIX querys_gard;
MATRIX keys_gard;
MATRIX values_gard;
if(!querys_gard.Init(m_iHeads, m_iKeysSize) ||
!keys_gard.Init(m_iHeads, m_iKeysSize) ||
!values_gard.Init(m_iHeads, m_iKeysSize))
return false;
for(int head = 0; head < m_iHeads; head++)
{
//if(!values_gard.Row((gradients[head]*Scores[head, m_iCurrentPosition]).Row(0), head))
// return false;
//--- Распределение градиента на Querys и Keys
//MATRIX score_grad = gradients[head].MatMul(values[head].Transpose());
////---
//VECTOR temp;
//temp.Init(m_iUnits);
//VECTOR s = Scores.Row(head);
//for(int c = 0; c < m_iUnits; c++)
// temp[c]=((int)(c==m_iCurrentPosition) - s);
//s = s.MatMul(temp);
//if(!gradients[head].Row(s * gradients[head].Row(r) / sqrt(m_iKeysSize), r))
// return false;
//temp = gradients[head].MatMul(keys[head]);
//if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
// !querys_grad.Row(temp.Row(0), head))
// return false;
//temp = gradients[head].Transpose().MatMul(querys[head]);
//if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
// !keys_grad.Row(temp.Row(0), head))
// return false;
}
//if(!Querys.GetGradients().UpdateArray(2 * m_iKeysSize * m_iHeads, values, 0, m_iKeysSize * m_iHeads))
// return false;
//if(!Querys.GetGradients().UpdateArray(0, querys_grad, 0, 2 * keys_total))
// return false;
}
else // Блок OpenCL
{
//--- Создание буферов данных
if(Values.GetOutputs().GetIndex() < 0)
return false;
if(Querys.GetGradients().GetIndex() < 0)
return false;
if(Scores.GetOutputs().GetIndex() < 0)
return false;
if(attention_grad.GetIndex() < 0)
return false;
if(Scores.GetGradients().GetIndex() < 0)
return false;
//---
if(m_cScoreTemp.GetIndex() < 0)
return false;
//--- Передача параметров кернелу
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_outputs_grad, attention_grad.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores, Scores.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_grad, Scores.GetGradients().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_scores_temp, m_cScoreTemp.GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values, Values.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTScoreGradients, def_gptscr_values_grad, Querys.GetGradients().GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_window, m_iKeysSize))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_units, m_iUnits))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTScoreGradients, def_gptscr_current, m_iCurrentPosition))
return false;
//--- Постановка кернела в очередь выполнения
int off_set[] = {0};
int NDRange[] = {m_iHeads};
if(!m_cOpenCL.Execute(def_k_GPTScoreGradients, 1, off_set, NDRange))
return false;
//--- Загрузка результатов
if(!Querys.GetGradients().BufferRead())
return false;
//---
if(Querys.GetOutputs().GetIndex() < 0)
return false;
if(Keys.GetOutputs().GetIndex() < 0)
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_keys, Keys.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys, Querys.GetOutputs().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_querys_grad, Querys.GetGradients().GetIndex()))
return false;
if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTHiddenGradients, def_gpthgr_scores_grad, Scores.GetGradients().GetIndex()))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_key_size, m_iKeysSize))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_units, m_iUnits))
return false;
if(!m_cOpenCL.SetArgument(def_k_GPTHiddenGradients, def_gpthgr_current, m_iCurrentPosition))
return false;
if(!m_cOpenCL.Execute(def_k_GPTHiddenGradients, 1, off_set, NDRange))
return false;
//--- Загрузка результатов
if(!Querys.GetGradients().BufferRead())
return false;
}
//--- Перенос градиента ошибки на предыдущий слой
CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1));
if(!Querys.CalcHiddenGradient(prevL))
return false;
if(!prevL.GetGradients().SumArray(W0.GetGradients()))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод распределения градиентов ошибки до матриц весовых |
//| коэффициентов |
//+------------------------------------------------------------------+
bool CNeuronGPT::CalcDeltaWeights(CNeuronBase *prevLayer)
{
//--- Проверяем актуальность всех объектов
if(CheckPointer(m_cFF2) == POINTER_INVALID)
return false;
if(CheckPointer(m_cFF1) == POINTER_INVALID)
return false;
if(CheckPointer(m_cW0) == POINTER_INVALID)
return false;
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
return false;
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
return false;
//--- В цикле вызываем аналогичный метод для каждого внутреннего объекта
for(int layer = 0; layer < m_iLayers; layer++)
{
if(CheckPointer(m_cFF2.At(layer)) == POINTER_INVALID)
return false;
CNeuronBase *temp = m_cFF2.At(layer);
if(!temp.CalcDeltaWeights(m_cFF1.At(layer)))
return false;
temp = m_cFF1.At(layer);
if(!temp.CalcDeltaWeights(m_cW0.At(layer)))
return false;
temp = m_cW0.At(layer);
if(!temp.CalcDeltaWeights(m_cAttentionOut.At(layer)))
return false;
temp = m_cQuerys.At(layer);
if(CheckPointer(temp) == POINTER_INVALID)
return false;
CNeuronBase *prevL = (layer == 0 ? prevLayer : m_cFF2.At(layer - 1));
if(!temp.CalcDeltaWeights(prevL))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод обновления параметров матрицы весовых коэффициентов |
//+------------------------------------------------------------------+
bool CNeuronGPT::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
{
//--- Блок контролей
if(CheckPointer(m_cFF2) == POINTER_INVALID)
return false;
if(CheckPointer(m_cFF1) == POINTER_INVALID)
return false;
if(CheckPointer(m_cW0) == POINTER_INVALID)
return false;
if(CheckPointer(m_cQuerys) == POINTER_INVALID)
return false;
//--- В цикле вызываем аналогичный метод для каждого внутреннего объекта
for(int layer = 0; layer < m_iLayers; layer++)
{
CNeuronBase *temp = m_cFF2.At(layer);
if(CheckPointer(temp) == POINTER_INVALID ||
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
return false;
temp = m_cFF1.At(layer);
if(CheckPointer(temp) == POINTER_INVALID ||
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
return false;
temp = m_cW0.At(layer);
if(CheckPointer(temp) == POINTER_INVALID ||
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
return false;
temp = m_cQuerys.At(layer);
if(CheckPointer(temp) == POINTER_INVALID ||
!temp.UpdateWeights(batch_size, learningRate, Beta, Lambda))
return false;
}
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод сохранения элементов класса в файл |
//+------------------------------------------------------------------+
bool CNeuronGPT::Save(const int file_handle)
{
//--- Вызов метода родительского класса
if(!CNeuronBase::Save(file_handle))
return false;
//--- Сохраняем константы
if(FileWriteInteger(file_handle, m_iLayers) <= 0)
return false;
if(FileWriteInteger(file_handle, m_iWindow) <= 0)
return false;
if(FileWriteInteger(file_handle, m_iKeysSize) <= 0)
return false;
if(FileWriteInteger(file_handle, m_iHeads) <= 0)
return false;
if(FileWriteInteger(file_handle, m_iUnits) <= 0)
return false;
if(FileWriteInteger(file_handle, m_iCurrentPosition) <= 0)
return false;
//--- Вызываем аналогичный метод для всех колекций внутренних слоёв
if(CheckPointer(m_cQuerys) == POINTER_INVALID ||
!m_cQuerys.Save(file_handle))
return false;
if(CheckPointer(m_cKeys) == POINTER_INVALID ||
!m_cKeys.Save(file_handle))
return false;
if(CheckPointer(m_cValues) == POINTER_INVALID ||
!m_cValues.Save(file_handle))
return false;
if(CheckPointer(m_cScores) == POINTER_INVALID ||
!m_cScores.Save(file_handle))
return false;
if(CheckPointer(m_cAttentionOut) == POINTER_INVALID ||
!m_cAttentionOut.Save(file_handle))
return false;
if(CheckPointer(m_cW0) == POINTER_INVALID ||
!m_cW0.Save(file_handle))
return false;
if(CheckPointer(m_cFF1) == POINTER_INVALID ||
!m_cFF1.Save(file_handle))
return false;
if(CheckPointer(m_cFF2) == POINTER_INVALID ||
!m_cFF2.Save(file_handle))
return false;
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод восстановления работы класса из файла |
//+------------------------------------------------------------------+
bool CNeuronGPT::Load(const int file_handle)
{
//--- Вызов метода родительского класса
if(!CNeuronBase::Load(file_handle))
return false;
//--- Считываем константы из файла
m_iLayers = FileReadInteger(file_handle);
m_iWindow = FileReadInteger(file_handle);
m_iKeysSize = FileReadInteger(file_handle);
m_iHeads = FileReadInteger(file_handle);
m_iUnits = FileReadInteger(file_handle);
m_iCurrentPosition = FileReadInteger(file_handle);
if(ArrayResize(m_dStd, m_iLayers) <= 0)
return false;
//--- Вызываем аналогичный метод для всех колекций внутренних слоёв
if(!CheckArrayLayers(m_cQuerys) ||
!m_cQuerys.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cKeys) ||
!m_cKeys.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cValues) ||
!m_cValues.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cScores) ||
!m_cScores.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cAttentionOut) ||
!m_cAttentionOut.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cW0) ||
!m_cW0.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cFF1) ||
!m_cFF1.Load(file_handle))
return false;
if(!CheckArrayLayers(m_cFF2) ||
!m_cFF2.Load(file_handle))
return false;
//--- Осуществляем подмену буферов данных для исключения излишнего копирования
CNeuronBase *last = m_cFF2.At(m_cFF2.Total() - 1);
if(CheckPointer(last) == POINTER_INVALID)
return false;
if(CheckPointer(m_cOutputs) != POINTER_INVALID)
delete m_cOutputs;
m_cOutputs = last.GetOutputs();
if(CheckPointer(m_cGradients) != POINTER_INVALID)
delete m_cGradients;
m_cGradients = last.GetGradients();
//---
return true;
}
//+------------------------------------------------------------------+
//| Метод проверки актуальности указателя на объект коллекции |
//| нейроных слоёв |
//+------------------------------------------------------------------+
bool CNeuronGPT::CheckArrayLayers(CArrayLayers *&layers)
{
if(CheckPointer(layers) == POINTER_INVALID)
layers = new CArrayLayers();
//---
return CheckPointer(layers) != POINTER_INVALID;
}
//+------------------------------------------------------------------+
Reference in a new issue View git blame Copy permalink