NeuroBook/Include/realization/neuronlstm.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   NeuronLSTM.mqh |

//|                                  Copyright 2021, MetaQuotes Ltd. |

//|                                             https://www.mql5.com |

//+------------------------------------------------------------------+

#property copyright "Copyright 2021, MetaQuotes Ltd."

#property link      "https://www.mql5.com"

//+------------------------------------------------------------------+

//| Connect libraries                                                |

//+------------------------------------------------------------------+

#include "neuronbase.mqh"

#include <Arrays\ArrayObj.mqh>

//+------------------------------------------------------------------+

//| Class CNeuronLSTM                                                |

//| Purpose: Class for implementing a recurrent LSTM block           |

//+------------------------------------------------------------------+

class CNeuronLSTM    :  public CNeuronBase

  {

protected:

   CNeuronBase*      m_cForgetGate;

   CNeuronBase*      m_cInputGate;

   CNeuronBase*      m_cNewContent;

   CNeuronBase*      m_cOutputGate;

   CArrayObj*        m_cMemorys;

   CArrayObj*        m_cHiddenStates;

   CArrayObj*        m_cInputs;

   CArrayObj*        m_cForgetGateOuts;

   CArrayObj*        m_cInputGateOuts;

   CArrayObj*        m_cNewContentOuts;

   CArrayObj*        m_cOutputGateOuts;

   CBufferType*      m_cInputGradient;

   int               m_iDepth;



   void              ClearBuffer(CArrayObj *buffer);

   bool              InsertBuffer(CArrayObj *&array, CBufferType *element, bool create_new = true);

   CBufferType*      CreateBuffer(CArrayObj *&array);



public:

                     CNeuronLSTM(void);

                    ~CNeuronLSTM(void);

   //---

   virtual bool      Init(const CLayerDescription *desc) override;

   virtual bool      SetOpenCL(CMyOpenCL *opencl) override;

   virtual bool      FeedForward(CNeuronBase *prevLayer) override;

   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer) override;

   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer, bool read) override

                                { return (!m_cOpenCL ? true : m_cDeltaWeights.BufferRead()); }

   virtual bool      UpdateWeights(int batch_size, TYPE learningRate,

                                   VECTOR &Beta, VECTOR &Lambda) override;

   //---

   virtual int       GetDepth(void)  const { return m_iDepth; }

   //--- file handling methods

   virtual bool      Save(const int file_handle) override;

   virtual bool      Load(const int file_handle) override;

   //--- object identification method

   virtual int       Type(void)  override         const { return(defNeuronLSTM); }

  };

//+------------------------------------------------------------------+

//| Class constructor                                                |

//+------------------------------------------------------------------+

CNeuronLSTM::CNeuronLSTM(void)   : m_iDepth(2)

  {

   m_cForgetGate = new CNeuronBase();

   m_cInputGate = new CNeuronBase();

   m_cNewContent = new CNeuronBase();

   m_cOutputGate = new CNeuronBase();

   m_cMemorys = new CArrayObj();

   m_cHiddenStates = new CArrayObj();

   m_cInputs = new CArrayObj();

   m_cForgetGateOuts = new CArrayObj();

   m_cInputGateOuts = new CArrayObj();

   m_cNewContentOuts = new CArrayObj();

   m_cOutputGateOuts = new CArrayObj();

   m_cInputGradient = new CBufferType();

  }

//+------------------------------------------------------------------+

//| Class destructor                                                 |

//+------------------------------------------------------------------+

CNeuronLSTM::~CNeuronLSTM(void)

  {

   if(m_cForgetGate)

      delete m_cForgetGate;

   if(m_cInputGate)

      delete m_cInputGate;

   if(m_cNewContent)

      delete m_cNewContent;

   if(m_cOutputGate)

      delete m_cOutputGate;

   if(m_cMemorys)

      delete m_cMemorys;

   if(m_cHiddenStates)

      delete m_cHiddenStates;

   if(m_cInputs)

      delete m_cInputs;

   if(m_cForgetGateOuts)

      delete m_cForgetGateOuts;

   if(m_cInputGateOuts)

      delete m_cInputGateOuts;

   if(m_cNewContentOuts)

      delete m_cNewContentOuts;

   if(m_cOutputGateOuts)

      delete m_cOutputGateOuts;

   if(m_cInputGradient)

      delete m_cInputGradient;

  }

//+------------------------------------------------------------------+

//| Class initialization method                                      |

//+------------------------------------------------------------------+

bool CNeuronLSTM::Init(const CLayerDescription *desc)

  {

//--- control block

   if(!desc || desc.type != Type() || desc.count <= 0 || desc.window == 0)

      return false;

//--- create a description for the internal neural layers

   CLayerDescription *temp = new CLayerDescription();

   if(!temp)

      return false;

   temp.type = defNeuronBase;

   temp.window = desc.window + desc.count;

   temp.count = desc.count;

   temp.activation = AF_SIGMOID;

   temp.activation_params[0] = 1;

   temp.activation_params[1] = 0;

   temp.optimization = desc.optimization;

//--- call the initialization method of the parent class

   CLayerDescription *temp2 = new CLayerDescription();

   if(!temp2 || !temp2.Copy(desc))

      return false;

   temp2.window = 0;

   if(!CNeuronBase::Init(temp2))

      return false;

   delete temp2;

   if(!InsertBuffer(m_cHiddenStates, m_cOutputs, false))

      return false;

   m_iDepth = (int)fmax(desc.window_out, 2);

//--- initialize ForgetGate

   if(!m_cForgetGate)

     {

      if(!(m_cForgetGate = new CNeuronBase()))

         return false;

     }

   if(!m_cForgetGate.Init(temp))

      return false;

   if(!InsertBuffer(m_cForgetGateOuts, m_cForgetGate.GetOutputs(), false))

      return false;

//--- initialize InputGate

   if(!m_cInputGate)

     {

      if(!(m_cInputGate = new CNeuronBase()))

         return false;

     }

   if(!m_cInputGate.Init(temp))

      return false;

   if(!InsertBuffer(m_cInputGateOuts, m_cInputGate.GetOutputs(), false))

      return false;

//--- initialize OutputGate

   if(!m_cOutputGate)

     {

      if(!(m_cOutputGate = new CNeuronBase()))

         return false;

     }

   if(!m_cOutputGate.Init(temp))

      return false;

   if(!InsertBuffer(m_cOutputGateOuts, m_cOutputGate.GetOutputs(), false))

      return false;

//--- initialize NewContent

   if(!m_cNewContent)

     {

      if(!(m_cNewContent = new CNeuronBase()))

         return false;

     }

   temp.activation = AF_TANH;

   if(!m_cNewContent.Init(temp))

      return false;

   if(!InsertBuffer(m_cNewContentOuts, m_cNewContent.GetOutputs(), false))

      return false;

//--- initialize InputGradient buffer

   if(!m_cInputGradient)

     {

      if(!(m_cInputGradient = new CBufferType()))

         return false;

     }

   if(!m_cInputGradient.BufferInit(1, temp.window, 0))

      return false;

   delete temp;

//--- initialize Memory

   CBufferType *buffer =  CreateBuffer(m_cMemorys);

   if(!buffer)

      return false;

   if(!InsertBuffer(m_cMemorys, buffer, false))

     {

      delete buffer;

      return false;

     }

//--- initialize HiddenStates

   if(!(buffer =  CreateBuffer(m_cHiddenStates)))

      return false;

   if(!InsertBuffer(m_cHiddenStates, buffer, false))

     {

      delete buffer;

      return false;

     }

//---

   SetOpenCL(m_cOpenCL);

//---

   return true;

  }

//+------------------------------------------------------------------+

//| Method for passing a pointer to the OpenCL object to all         |

//| internal objects                                                 |

//+------------------------------------------------------------------+

bool CNeuronLSTM::SetOpenCL(CMyOpenCL *opencl)

  {

//--- call of the method of the parent class

   CNeuronBase::SetOpenCL(opencl);

//--- call the relevant method for all internal layers

   m_cForgetGate.SetOpenCL(m_cOpenCL);

   m_cInputGate.SetOpenCL(m_cOpenCL);

   m_cOutputGate.SetOpenCL(m_cOpenCL);

   m_cNewContent.SetOpenCL(m_cOpenCL);

   m_cInputGradient.BufferCreate(m_cOpenCL);

   for(int i = 0; i < m_cMemorys.Total(); i++)

     {

      CBufferType *temp = m_cMemorys.At(i);

      temp.BufferCreate(m_cOpenCL);

     }

   for(int i = 0; i < m_cHiddenStates.Total(); i++)

     {

      CBufferType *temp = m_cHiddenStates.At(i);

      temp.BufferCreate(m_cOpenCL);

     }

//---

   return(!!m_cOpenCL);

  }

//+------------------------------------------------------------------+

//| Method for deleting unnecessary data from the stack              |

//+------------------------------------------------------------------+

void CNeuronLSTM::ClearBuffer(CArrayObj *buffer)

  {

   if(!buffer)

      return;

   int total = buffer.Total();

   if(total > m_iDepth + 1)

      buffer.DeleteRange(m_iDepth + 1, total);

  }

//+------------------------------------------------------------------+

//| Feed-forward method                                              |

//+------------------------------------------------------------------+

bool CNeuronLSTM::FeedForward(CNeuronBase *prevLayer)

  {

//--- check the relevance of all objects

   if(!prevLayer || !prevLayer.GetOutputs() || !m_cOutputs ||

      !m_cForgetGate || !m_cInputGate || !m_cOutputGate ||

      !m_cNewContent)

      return false;

//--- prepare blanks for new buffers

   if(!m_cForgetGate.SetOutputs(CreateBuffer(m_cForgetGateOuts), false))

      return false;

   if(!m_cInputGate.SetOutputs(CreateBuffer(m_cInputGateOuts), false))

      return false;

   if(!m_cOutputGate.SetOutputs(CreateBuffer(m_cOutputGateOuts), false))

      return false;

   if(!m_cNewContent.SetOutputs(CreateBuffer(m_cNewContentOuts), false))

      return false;

   CBufferType *memory = CreateBuffer(m_cMemorys);

   if(!memory)

      return false;

   CBufferType *hidden = CreateBuffer(m_cHiddenStates);

   if(!hidden)

     {

      delete memory;

      return false;

     }

//--- only to check the gradient

//memory.m_mMatrix.Fill(0);

//hidden.m_mMatrix.Fill(0);

//if(!!m_cOpenCL)

//  {

//   memory.BufferWrite();

//   hidden.BufferWrite();

//  }

//--- create the buffer of source data

   if(!m_cInputs)

     {

      m_cInputs = new CArrayObj();

      if(!m_cInputs)

        {

         delete memory;

         delete hidden;

         return false;

        }

     }

   CNeuronBase *inputs = new CNeuronBase();

   if(!inputs)

     {

      delete memory;

      delete hidden;

      return false;

     }

   CLayerDescription *desc = new CLayerDescription();

   if(!desc)

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

   desc.type = defNeuronBase;

   desc.count = (int)(prevLayer.GetOutputs().Total() + m_cOutputs.Total());

   desc.window = 0;

   if(!inputs.Init(desc))

     {

      delete inputs;

      delete memory;

      delete hidden;

      delete desc;

      return false;

     }

   delete desc;

   inputs.SetOpenCL(m_cOpenCL);

   CBufferType *inputs_buffer = inputs.GetOutputs();

   if(!inputs_buffer)

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

   if(!inputs_buffer.Concatenate(prevLayer.GetOutputs(), hidden, prevLayer.Total(), hidden.Total()))

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

//--- feed-forward pass through internal neural layers

   if(!m_cForgetGate.FeedForward(inputs))

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

   if(!m_cInputGate.FeedForward(inputs))

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

   if(!m_cOutputGate.FeedForward(inputs))

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

   if(!m_cNewContent.FeedForward(inputs))

     {

      delete inputs;

      delete memory;

      delete hidden;

      return false;

     }

//--- branching of the algorithm across computing devices

   CBufferType *fg = m_cForgetGate.GetOutputs();

   CBufferType *ig = m_cInputGate.GetOutputs();

   CBufferType *og = m_cOutputGate.GetOutputs();

   CBufferType *nc = m_cNewContent.GetOutputs();

   if(!m_cOpenCL)

     {

      memory.m_mMatrix *= fg.m_mMatrix;

      memory.m_mMatrix += ig.m_mMatrix * nc.m_mMatrix;

      hidden.m_mMatrix = MathTanh(memory.m_mMatrix) * og.m_mMatrix;

     }

   else

     {

      //--- check buffers

      if(fg.GetIndex() < 0 || ig.GetIndex() < 0 || og.GetIndex() < 0 ||

         nc.GetIndex() < 0 || memory.GetIndex() < 0 || hidden.GetIndex() < 0)

         return false;

      //--- pass parameters to the kernel

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_forgetgate, fg.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_inputgate, ig.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_newcontent, nc.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_outputgate, og.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_memory, memory.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_LSTMFeedForward, def_lstmff_hiddenstate, hidden.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgument(def_k_LSTMFeedForward, def_lstmff_outputs_total, (int)m_cOutputs.Total()))

         return false;

      //--- launch the kernel

      int NDRange[] = {(int)(m_cOutputs.Total() + 3) / 4};

      int off_set[] = {0};

      if(!m_cOpenCL.Execute(def_k_LSTMFeedForward, 1, off_set, NDRange))

         return false;

     }

//--- copy the hidden state to the results buffer of the neural layer

   m_cOutputs = hidden;

//--- save the current state

   if(!m_cInputs.Insert(inputs, 0))

     {

      delete inputs;