905 lines
65 KiB
MQL5
905 lines
65 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronLSTM.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Подключаем библиотеки |
|
|
//+------------------------------------------------------------------+
|
|
#include "neuronbase.mqh"
|
|
#include <Arrays\ArrayObj.mqh>
|
|
//+------------------------------------------------------------------+
|
|
//| Class CNeuronLSTM |
|
|
//| Назначение: Класс организации рекуррентного LSTM блока |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronLSTM : public CNeuronBase
|
|
{
|
|
protected:
|
|
CNeuronBase *m_cForgetGate;
|
|
CNeuronBase *m_cInputGate;
|
|
CNeuronBase *m_cNewContent;
|
|
CNeuronBase *m_cOutputGate;
|
|
CArrayObj *m_cMemorys;
|
|
CArrayObj *m_cHiddenStates;
|
|
CArrayObj *m_cInputs;
|
|
CArrayObj *m_cForgetGateOuts;
|
|
CArrayObj *m_cInputGateOuts;
|
|
CArrayObj *m_cNewContentOuts;
|
|
CArrayObj *m_cOutputGateOuts;
|
|
CBufferDouble *m_cInputGradient;
|
|
int m_iDepth;
|
|
|
|
void ClearBuffer(CArrayObj *buffer);
|
|
bool InsertBuffer(CArrayObj *&array, CBufferDouble *element, bool create_new = true);
|
|
CBufferDouble *CreateBuffer(CArrayObj *&array);
|
|
|
|
public:
|
|
CNeuronLSTM(void);
|
|
~CNeuronLSTM(void);
|
|
//---
|
|
virtual bool Init(CLayerDescription *description);
|
|
virtual bool SetOpenCL(CMyOpenCL *opencl);
|
|
virtual bool FeedForward(CNeuronBase *prevLayer);
|
|
virtual bool CalcHiddenGradient(CNeuronBase *prevLayer);
|
|
virtual bool CalcDeltaWeights(CNeuronBase *prevLayer) { return true; }
|
|
virtual bool UpdateWeights(int batch_size, double learningRate,
|
|
double &Beta[], double &Lambda[]);
|
|
//---
|
|
virtual int GetDepth(void) const { return m_iDepth; }
|
|
//--- Методы работы с файлами
|
|
virtual bool Save(const int file_handle);
|
|
virtual bool Load(const int file_handle);
|
|
//--- Метод идентификации объекта
|
|
virtual int Type(void) const { return(defNeuronLSTM); }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Конструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronLSTM::CNeuronLSTM(void) : m_iDepth(2)
|
|
{
|
|
m_cForgetGate = new CNeuronBase();
|
|
m_cInputGate = new CNeuronBase();
|
|
m_cNewContent = new CNeuronBase();
|
|
m_cOutputGate = new CNeuronBase();
|
|
m_cMemorys = new CArrayObj();
|
|
m_cHiddenStates = new CArrayObj();
|
|
m_cInputs = new CArrayObj();
|
|
m_cForgetGateOuts = new CArrayObj();
|
|
m_cInputGateOuts = new CArrayObj();
|
|
m_cNewContentOuts = new CArrayObj();
|
|
m_cOutputGateOuts = new CArrayObj();
|
|
m_cInputGradient = new CBufferDouble();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Деструктор класса |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronLSTM::~CNeuronLSTM(void)
|
|
{
|
|
if(CheckPointer(m_cForgetGate) != POINTER_INVALID)
|
|
delete m_cForgetGate;
|
|
if(CheckPointer(m_cInputGate) != POINTER_INVALID)
|
|
delete m_cInputGate;
|
|
if(CheckPointer(m_cNewContent) != POINTER_INVALID)
|
|
delete m_cNewContent;
|
|
if(CheckPointer(m_cOutputGate) != POINTER_INVALID)
|
|
delete m_cOutputGate;
|
|
if(CheckPointer(m_cMemorys) != POINTER_INVALID)
|
|
delete m_cMemorys;
|
|
if(CheckPointer(m_cHiddenStates) != POINTER_INVALID)
|
|
delete m_cHiddenStates;
|
|
if(CheckPointer(m_cInputs) != POINTER_INVALID)
|
|
delete m_cInputs;
|
|
if(CheckPointer(m_cForgetGateOuts) != POINTER_INVALID)
|
|
delete m_cForgetGateOuts;
|
|
if(CheckPointer(m_cInputGateOuts) != POINTER_INVALID)
|
|
delete m_cInputGateOuts;
|
|
if(CheckPointer(m_cNewContentOuts) != POINTER_INVALID)
|
|
delete m_cNewContentOuts;
|
|
if(CheckPointer(m_cOutputGateOuts) != POINTER_INVALID)
|
|
delete m_cOutputGateOuts;
|
|
if(CheckPointer(m_cInputGradient) != POINTER_INVALID)
|
|
delete m_cInputGradient;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод инициализации класса |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::Init(CLayerDescription *description)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(description) == POINTER_INVALID || description.type != Type() ||
|
|
description.count <= 0 || description.window == 0)
|
|
return false;
|
|
//--- Создаём описание для внутренних нейронных слоёв
|
|
CLayerDescription *temp = new CLayerDescription();
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
|
return false;
|
|
temp.type = defNeuronBase;
|
|
temp.window = description.window + description.count;
|
|
temp.count = description.count;
|
|
temp.activation = ACT_SIGMOID;
|
|
temp.activation_params[0] = 1;
|
|
temp.activation_params[1] = 0;
|
|
temp.optimization = description.optimization;
|
|
//--- Вызываем метод инициализации родительского класса
|
|
description.window = 0;
|
|
if(!CNeuronBase::Init(description))
|
|
return false;
|
|
m_iDepth = (int)fmax(description.window_out, 2);
|
|
//--- Инициализируем ForgetGate
|
|
if(CheckPointer(m_cForgetGate) == POINTER_INVALID)
|
|
{
|
|
m_cForgetGate = new CNeuronBase();
|
|
if(CheckPointer(m_cForgetGate) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cForgetGate.Init(temp))
|
|
return false;
|
|
//--- Инициализируем InputGate
|
|
if(CheckPointer(m_cInputGate) == POINTER_INVALID)
|
|
{
|
|
m_cInputGate = new CNeuronBase();
|
|
if(CheckPointer(m_cInputGate) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cInputGate.Init(temp))
|
|
return false;
|
|
//--- Инициализируем OutputGate
|
|
if(CheckPointer(m_cOutputGate) == POINTER_INVALID)
|
|
{
|
|
m_cOutputGate = new CNeuronBase();
|
|
if(CheckPointer(m_cOutputGate) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cOutputGate.Init(temp))
|
|
return false;
|
|
//--- Инициализируем NewContent
|
|
if(CheckPointer(m_cNewContent) == POINTER_INVALID)
|
|
{
|
|
m_cNewContent = new CNeuronBase();
|
|
if(CheckPointer(m_cNewContent) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
temp.activation = ACT_TANH;
|
|
if(!m_cNewContent.Init(temp))
|
|
return false;
|
|
//--- Инициализируем буфер InputGradient
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
{
|
|
m_cInputGradient = new CBufferDouble();
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cInputGradient.BufferInit(temp.window, 0))
|
|
return false;
|
|
delete temp;
|
|
//--- Инициализируем Memory
|
|
CBufferDouble *buffer = CreateBuffer(m_cMemorys);
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return false;
|
|
if(m_cMemorys.Total() > 0)
|
|
{
|
|
if(!buffer.BufferInit(description.count, 0))
|
|
{
|
|
delete buffer;
|
|
return false;
|
|
}
|
|
m_cMemorys.Clear();
|
|
}
|
|
if(!m_cMemorys.Add(buffer))
|
|
{
|
|
delete buffer;
|
|
return false;
|
|
}
|
|
//--- Инициализируем HiddenStates
|
|
buffer = CreateBuffer(m_cHiddenStates);
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return false;
|
|
if(m_cHiddenStates.Total() > 0)
|
|
{
|
|
if(!buffer.BufferInit(description.count, 0))
|
|
return false;
|
|
m_cHiddenStates.Clear();
|
|
}
|
|
if(!m_cHiddenStates.Add(buffer))
|
|
return false;
|
|
//---
|
|
SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод передачи указателя на объект OpenCL до всех |
|
|
//| внутренних объектов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::SetOpenCL(CMyOpenCL *opencl)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
CNeuronBase::SetOpenCL(opencl);
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
m_cForgetGate.SetOpenCL(m_cOpenCL);
|
|
m_cInputGate.SetOpenCL(m_cOpenCL);
|
|
m_cOutputGate.SetOpenCL(m_cOpenCL);
|
|
m_cNewContent.SetOpenCL(m_cOpenCL);
|
|
//---
|
|
return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод удаления из стека мзлишних данных |
|
|
//+------------------------------------------------------------------+
|
|
void CNeuronLSTM::ClearBuffer(CArrayObj *buffer)
|
|
{
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return;
|
|
int total = buffer.Total();
|
|
if(total > m_iDepth + 1)
|
|
buffer.DeleteRange(m_iDepth + 1, total);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод прямого прохода |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::FeedForward(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputs) == POINTER_INVALID ||
|
|
CheckPointer(m_cForgetGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cInputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cNewContent) == POINTER_INVALID)
|
|
return false;
|
|
//--- Подготавиваем заготовки для новых буферов памяти и скрытого состояния
|
|
CBufferDouble *memory = CreateBuffer(m_cMemorys);
|
|
if(CheckPointer(memory) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *hidden = CreateBuffer(m_cHiddenStates);
|
|
if(CheckPointer(hidden) == POINTER_INVALID)
|
|
{
|
|
delete memory;
|
|
return false;
|
|
}
|
|
//--- Только для проверки градиента
|
|
//memory.BufferInit(m_cOutputs.Total(), 0);
|
|
//hidden.BufferInit(m_cOutputs.Total(), 0);
|
|
//--- Создаём буфер исходных данных
|
|
if(CheckPointer(m_cInputs) == POINTER_INVALID)
|
|
{
|
|
m_cInputs = new CArrayObj();
|
|
if(CheckPointer(m_cInputs) == POINTER_INVALID)
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
}
|
|
CNeuronBase *inputs = new CNeuronBase();
|
|
if(CheckPointer(inputs) == POINTER_INVALID)
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
CLayerDescription *description = new CLayerDescription();
|
|
if(CheckPointer(description) == POINTER_INVALID)
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
description.type = defNeuronBase;
|
|
description.count = prevLayer.GetOutputs().Total() + m_cOutputs.Total();
|
|
description.window = 0;
|
|
if(!inputs.Init(description))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
delete description;
|
|
CBufferDouble *inputs_buffer = inputs.GetOutputs();
|
|
if(CheckPointer(inputs_buffer) == POINTER_INVALID)
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!inputs_buffer.AssignArray(prevLayer.GetOutputs()))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!inputs_buffer.AddArray(hidden))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
//--- Делаем прямой проход внутренних нейронных слоёв
|
|
if(!m_cForgetGate.FeedForward(inputs))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!m_cInputGate.FeedForward(inputs))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!m_cOutputGate.FeedForward(inputs))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!m_cNewContent.FeedForward(inputs))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
CBufferDouble *fg = m_cForgetGate.GetOutputs();
|
|
CBufferDouble *ig = m_cInputGate.GetOutputs();
|
|
CBufferDouble *og = m_cOutputGate.GetOutputs();
|
|
CBufferDouble *nc = m_cNewContent.GetOutputs();
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
int total = m_cOutputs.Total();
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
double temp = memory.At(i) * fg.At(i);
|
|
temp += ig.At(i) * nc.At(i);
|
|
memory.Update(i, temp);
|
|
temp = MathTanh(temp) * og.At(i);
|
|
hidden.Update(i, temp);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создаём буферы
|
|
if(!fg.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!ig.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!og.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!nc.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!memory.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!hidden.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передаём параметры кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_forgetgate, fg.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_inputgate, ig.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_newcontent, nc.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_outputgate, og.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_memory, memory.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_hiddenstate, hidden.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_LSTMFeedForward, def_lstmff_outputs_total, m_cOutputs.Total()))
|
|
return false;
|
|
//--- Запуск кернела
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_LSTMFeedForward, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получаем результаты
|
|
if(!memory.BufferRead())
|
|
return false;
|
|
if(!hidden.BufferRead())
|
|
return false;
|
|
//--- Удаляем буферы из памяти контекста OpenCL
|
|
fg.BufferFree();
|
|
ig.BufferFree();
|
|
og.BufferFree();
|
|
nc.BufferFree();
|
|
memory.BufferFree();
|
|
hidden.BufferFree();
|
|
prevLayer.GetOutputs().BufferFree();
|
|
}
|
|
//--- Копируем скрытое состояние в буфер результатов нейронного слоя
|
|
if(!m_cOutputs.AssignArray(hidden))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
//--- Сохраним текущее состояние
|
|
if(!m_cInputs.Insert(inputs, 0))
|
|
{
|
|
delete inputs;
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
ClearBuffer(m_cInputs);
|
|
if(!InsertBuffer(m_cForgetGateOuts, m_cForgetGate.GetOutputs()))
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!InsertBuffer(m_cInputGateOuts, m_cInputGate.GetOutputs()))
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!InsertBuffer(m_cOutputGateOuts, m_cOutputGate.GetOutputs()))
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!InsertBuffer(m_cNewContentOuts, m_cNewContent.GetOutputs()))
|
|
{
|
|
delete memory;
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!InsertBuffer(m_cMemorys, memory, false))
|
|
{
|
|
delete hidden;
|
|
return false;
|
|
}
|
|
if(!InsertBuffer(m_cHiddenStates, hidden, false))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод добавления данных в стек |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::InsertBuffer(CArrayObj *&array, CBufferDouble *element, bool create_new = true)
|
|
{
|
|
//--- Блок контролей
|
|
if(CheckPointer(element) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(array) == POINTER_INVALID)
|
|
{
|
|
array = new CArrayObj();
|
|
if(CheckPointer(array) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
//---
|
|
if(create_new)
|
|
{
|
|
CBufferDouble *buffer = new CBufferDouble();
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return false;
|
|
if(!buffer.AssignArray(element))
|
|
{
|
|
delete buffer;
|
|
return false;
|
|
}
|
|
if(!array.Insert(buffer, 0))
|
|
{
|
|
delete buffer;
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!array.Insert(element, 0))
|
|
{
|
|
delete element;
|
|
return false;
|
|
}
|
|
}
|
|
//--- Удалим из буфера излишнюю историю
|
|
ClearBuffer(array);
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод создания нового буфера данных |
|
|
//+------------------------------------------------------------------+
|
|
CBufferDouble *CNeuronLSTM::CreateBuffer(CArrayObj *&array)
|
|
{
|
|
if(CheckPointer(array) == POINTER_INVALID)
|
|
{
|
|
array = new CArrayObj();
|
|
if(CheckPointer(array) == POINTER_INVALID)
|
|
return NULL;
|
|
}
|
|
CBufferDouble *buffer = new CBufferDouble();
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return NULL;
|
|
if(array.Total() <= 0)
|
|
{
|
|
if(!buffer.BufferInit(m_cOutputs.Total(), 0))
|
|
{
|
|
delete buffer;
|
|
return NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
CBufferDouble *temp = array.At(0);
|
|
if(CheckPointer(temp) == POINTER_INVALID)
|
|
{
|
|
delete buffer;
|
|
return NULL;
|
|
}
|
|
if(!buffer.AssignArray(temp))
|
|
{
|
|
delete buffer;
|
|
return NULL;
|
|
}
|
|
}
|
|
//---
|
|
return buffer;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод распределения градиента ошибки через скрытый слой |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::CalcHiddenGradient(CNeuronBase *prevLayer)
|
|
{
|
|
//--- Проверяем актуальность всех объектов
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID ||
|
|
CheckPointer(prevLayer.GetGradients()) == POINTER_INVALID ||
|
|
CheckPointer(m_cGradients) == POINTER_INVALID ||
|
|
CheckPointer(m_cForgetGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cForgetGateOuts) == POINTER_INVALID ||
|
|
CheckPointer(m_cInputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cInputGateOuts) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputGateOuts) == POINTER_INVALID ||
|
|
CheckPointer(m_cNewContent) == POINTER_INVALID ||
|
|
CheckPointer(m_cNewContentOuts) == POINTER_INVALID)
|
|
return false;
|
|
//--- Проверяем наличие данных прямого прохода
|
|
int total = (int)fmin(m_cMemorys.Total(), m_cHiddenStates.Total()) - 1;
|
|
if(total <= 0)
|
|
return false;
|
|
//--- Делаем укзатели на буферы градиентов и результатов внутренних слоёв
|
|
CBufferDouble *fg_grad = m_cForgetGate.GetGradients();
|
|
if(CheckPointer(fg_grad) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *fg_out = m_cForgetGate.GetOutputs();
|
|
if(CheckPointer(fg_out) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *ig_grad = m_cInputGate.GetGradients();
|
|
if(CheckPointer(ig_grad) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *ig_out = m_cInputGate.GetOutputs();
|
|
if(CheckPointer(ig_out) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *og_grad = m_cOutputGate.GetGradients();
|
|
if(CheckPointer(og_grad) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *og_out = m_cOutputGate.GetOutputs();
|
|
if(CheckPointer(og_out) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *nc_grad = m_cNewContent.GetGradients();
|
|
if(CheckPointer(nc_grad) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *nc_out = m_cNewContent.GetOutputs();
|
|
if(CheckPointer(nc_out) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
int out_total = m_cOutputs.Total();
|
|
//--- Цикл перебора накопленной истории
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
//--- Получаем указатели на буферы из стека
|
|
CBufferDouble *fg = m_cForgetGateOuts.At(i);
|
|
if(CheckPointer(fg) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *ig = m_cInputGateOuts.At(i);
|
|
if(CheckPointer(ig) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *og = m_cOutputGateOuts.At(i);
|
|
if(CheckPointer(og) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *nc = m_cNewContentOuts.At(i);
|
|
if(CheckPointer(nc) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *memory = m_cMemorys.At(i + 1);
|
|
if(CheckPointer(memory) == POINTER_INVALID)
|
|
return false;
|
|
CBufferDouble *hidden = m_cHiddenStates.At(i);
|
|
if(CheckPointer(hidden) == POINTER_INVALID)
|
|
return false;
|
|
CNeuronBase *inputs = m_cInputs.At(i);
|
|
if(CheckPointer(inputs) == POINTER_INVALID)
|
|
return false;
|
|
//--- Разветвление алгоритма по вычислительному устройству
|
|
if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
|
|
{
|
|
//--- Посчитаем градиент на выходе каждого внутреннего слоя
|
|
for(int o = 0; o < out_total; o++)
|
|
{
|
|
double m = (og.At(o) != 0 ? hidden.At(o) / og.At(o) : 0);
|
|
double grad = m_cGradients.At(o);
|
|
//--- OutputGate градиент
|
|
double temp = grad * m;
|
|
if(!og_grad.Update(o, temp))
|
|
return false;
|
|
//--- Градиент памяти
|
|
grad *= og.At(o);
|
|
//--- Скорректируем градиент на производную
|
|
grad *= 1 - pow(m, 2);
|
|
//--- InputGate градиент
|
|
temp = grad * nc.At(o);
|
|
if(!ig_grad.Update(o, temp))
|
|
return false;
|
|
//--- NewContent градиент
|
|
temp = grad * ig.At(o);
|
|
if(!nc_grad.Update(o, temp))
|
|
return false;
|
|
//--- ForgetGates градиент
|
|
temp = grad * memory.At(o);
|
|
if(!fg_grad.Update(o, temp))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//--- Создаём буферы
|
|
if(!hidden.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!m_cGradients.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!ig.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!og.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!nc.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!memory.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!fg_grad.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!ig_grad.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!og_grad.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
if(!nc_grad.BufferCreate(m_cOpenCL))
|
|
return false;
|
|
//--- Передаём параметры кернелу
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_fg_gradients, fg_grad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_gradients, m_cGradients.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_ig_gradients, ig_grad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_inputgate, ig.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_memory, memory.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_nc_gradients, nc_grad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_newcontent, nc.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_og_gradients, og_grad.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_outputgate, og.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMHiddenGradients, def_lstmhgr_outputs, hidden.GetIndex()))
|
|
return false;
|
|
if(!m_cOpenCL.SetArgument(def_k_LSTMHiddenGradients, def_lstmhgr_outputs_total, m_cOutputs.Total()))
|
|
return false;
|
|
//--- Запуск кернела
|
|
int s = m_cOutputs.Total();
|
|
int d = s % 4;
|
|
s = (s - d) / 4 + (d > 0 ? 1 : 0);
|
|
int NDRange[] = {s};
|
|
int off_set[] = {0};
|
|
if(!m_cOpenCL.Execute(def_k_LSTMHiddenGradients, 1, off_set, NDRange))
|
|
return false;
|
|
//--- Получаем результаты
|
|
if(!fg_grad.BufferRead())
|
|
return false;
|
|
if(!ig_grad.BufferRead())
|
|
return false;
|
|
if(!og_grad.BufferRead())
|
|
return false;
|
|
if(!nc_grad.BufferRead())
|
|
return false;
|
|
//--- Удаляем буферы из памяти контекста OpenCL
|
|
memory.BufferFree();
|
|
hidden.BufferFree();
|
|
ig.BufferFree();
|
|
og.BufferFree();
|
|
nc.BufferFree();
|
|
m_cGradients.BufferFree();
|
|
}
|
|
//--- Скопируем соответсвующие исторические данные в буфера внутренних слоёв
|
|
if(!fg_out.AssignArray(fg))
|
|
return false;
|
|
if(!ig_out.AssignArray(ig))
|
|
return false;
|
|
if(!og_out.AssignArray(og))
|
|
return false;
|
|
if(!nc_out.AssignArray(nc))
|
|
return false;
|
|
//--- Проведём градиент через внутренние слои
|
|
if(!m_cForgetGate.CalcHiddenGradient(inputs))
|
|
return false;
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
{
|
|
m_cInputGradient = new CBufferDouble();
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!m_cInputGradient.AssignArray(inputs.GetGradients()))
|
|
return false;
|
|
if(!m_cInputGate.CalcHiddenGradient(inputs))
|
|
return false;
|
|
if(!m_cInputGradient.SumArray(inputs.GetGradients()))
|
|
return false;
|
|
if(!m_cOutputGate.CalcHiddenGradient(inputs))
|
|
return false;
|
|
if(!m_cInputGradient.SumArray(inputs.GetGradients()))
|
|
return false;
|
|
if(!m_cNewContent.CalcHiddenGradient(inputs))
|
|
return false;
|
|
if(!inputs.GetGradients().SumArray(m_cInputGradient))
|
|
return false;
|
|
//--- Спроэцируем градиент на матрицы весов внутренних слоёв
|
|
if(!m_cForgetGate.CalcDeltaWeights(inputs))
|
|
return false;
|
|
if(!m_cInputGate.CalcDeltaWeights(inputs))
|
|
return false;
|
|
if(!m_cOutputGate.CalcDeltaWeights(inputs))
|
|
return false;
|
|
if(!m_cNewContent.CalcDeltaWeights(inputs))
|
|
return false;
|
|
//--- Если посчитан градиент текущего состояния, то передаём на предыдущий слой
|
|
int shift = prevLayer.GetGradients().Total();
|
|
if(i == 0)
|
|
{
|
|
CBufferDouble *prevLayer_grad = prevLayer.GetGradients();
|
|
if(!prevLayer_grad.AssignArray(inputs.GetGradients()))
|
|
return false;
|
|
if(!prevLayer_grad.DeleteRange(shift, prevLayer_grad.Total()))
|
|
return false;
|
|
}
|
|
//--- Запишем градиент скрытого состояния в буфер градиентов для новой итерации
|
|
for(int o = 0; o < out_total; o++)
|
|
{
|
|
if(!m_cGradients.Update(o, inputs.GetGradients().At(shift + o)))
|
|
return false;
|
|
}
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод обновления матриц весовых коэффициентов |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
|
|
{
|
|
//--- Проверяем состояние объектов
|
|
if(CheckPointer(m_cForgetGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cInputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cOutputGate) == POINTER_INVALID ||
|
|
CheckPointer(m_cNewContent) == POINTER_INVALID ||
|
|
m_iDepth <= 0)
|
|
return false;
|
|
int batch = batch_size * m_iDepth;
|
|
//--- Обновляем матрицы весов внутренних слоёв
|
|
if(!m_cForgetGate.UpdateWeights(batch, learningRate, Beta, Lambda))
|
|
return false;
|
|
if(!m_cInputGate.UpdateWeights(batch, learningRate, Beta, Lambda))
|
|
return false;
|
|
if(!m_cOutputGate.UpdateWeights(batch, learningRate, Beta, Lambda))
|
|
return false;
|
|
if(!m_cNewContent.UpdateWeights(batch, learningRate, Beta, Lambda))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод сохранениия элементов класса в файл |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::Save(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronBase::Save(file_handle))
|
|
return false;
|
|
//--- Созраняем константы
|
|
if(FileWriteInteger(file_handle, m_iDepth) <= 0)
|
|
return false;
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(!m_cForgetGate.Save(file_handle))
|
|
return false;
|
|
if(!m_cInputGate.Save(file_handle))
|
|
return false;
|
|
if(!m_cOutputGate.Save(file_handle))
|
|
return false;
|
|
if(!m_cNewContent.Save(file_handle))
|
|
return false;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Метод восстановления класса из файла |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronLSTM::Load(const int file_handle)
|
|
{
|
|
//--- Вызов метода родительского класса
|
|
if(!CNeuronBase::Load(file_handle))
|
|
return false;
|
|
//--- Считываем константы
|
|
m_iDepth = FileReadInteger(file_handle);
|
|
//--- Вызываем аналогичный метод для всех внутренних слоёв
|
|
if(FileReadInteger(file_handle) != defNeuronBase || !m_cForgetGate.Load(file_handle))
|
|
return false;
|
|
if(FileReadInteger(file_handle) != defNeuronBase || !m_cInputGate.Load(file_handle))
|
|
return false;
|
|
if(FileReadInteger(file_handle) != defNeuronBase || !m_cOutputGate.Load(file_handle))
|
|
return false;
|
|
if(FileReadInteger(file_handle) != defNeuronBase || !m_cNewContent.Load(file_handle))
|
|
return false;
|
|
//--- Инициализируем буфер InputGradient
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
{
|
|
m_cInputGradient = new CBufferDouble();
|
|
if(CheckPointer(m_cInputGradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
//--- Инициализируем Memory
|
|
CBufferDouble *buffer = CreateBuffer(m_cMemorys);
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return false;
|
|
if(m_cMemorys.Total() > 0)
|
|
{
|
|
if(!buffer.BufferInit(m_cOutputs.Total(), 0))
|
|
return false;
|
|
m_cMemorys.Clear();
|
|
}
|
|
if(!m_cMemorys.Add(buffer))
|
|
return false;
|
|
//--- Инициализируем HiddenStates
|
|
buffer = CreateBuffer(m_cHiddenStates);
|
|
if(CheckPointer(buffer) == POINTER_INVALID)
|
|
return false;
|
|
if(m_cHiddenStates.Total() > 0)
|
|
{
|
|
if(!buffer.BufferInit(m_cOutputs.Total(), 0))
|
|
return false;
|
|
m_cHiddenStates.Clear();
|
|
}
|
|
if(!m_cHiddenStates.Add(buffer))
|
|
return false;
|
|
//--- Очищаем остальные стеки
|
|
if(CheckPointer(m_cInputs) != POINTER_INVALID)
|
|
m_cInputs.Clear();
|
|
if(CheckPointer(m_cForgetGateOuts) != POINTER_INVALID)
|
|
m_cForgetGateOuts.Clear();
|
|
if(CheckPointer(m_cInputGateOuts) != POINTER_INVALID)
|
|
m_cInputGateOuts.Clear();
|
|
if(CheckPointer(m_cNewContentOuts) != POINTER_INVALID)
|
|
m_cNewContentOuts.Clear();
|
|
if(CheckPointer(m_cOutputGateOuts) != POINTER_INVALID)
|
|
m_cOutputGateOuts.Clear();
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|