NeuroBook/Include/realization/neurongpt.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                    NeuronGPT.mqh |

//|                                  Copyright 2021, MetaQuotes Ltd. |

//|                                             https://www.mql5.com |

//+------------------------------------------------------------------+

#property copyright "Copyright 2021, MetaQuotes Ltd."

#property link      "https://www.mql5.com"

//+------------------------------------------------------------------+

//| Connect libraries                                                |

//+------------------------------------------------------------------+

#ifndef ArrayLayers

#include "arraylayers.mqh"

#endif

//+------------------------------------------------------------------+

//| Class CNeuronGPT                                                 |

//| Purpose: GPT block implementing class                            |

//+------------------------------------------------------------------+

class CNeuronGPT    :  public CNeuronBase

  {

protected:

   CArrayLayers      m_cQuerys;

   CArrayLayers      m_cKeys;

   CArrayLayers      m_cValues;

   CArrayLayers      m_cScores;

   CArrayLayers      m_cAttentionOut;

   CArrayLayers      m_cW0;

   CArrayLayers      m_cFF1;

   CArrayLayers      m_cFF2;

   //---

   int               m_iLayers;

   int               m_iWindow;

   int               m_iUnits;

   int               m_iKeysSize;

   int               m_iHeads;

   CBufferType       m_dStd[];

   int               m_iCurrentPosition;

   int               m_iScoreTemp;



   virtual bool      NormlizeBuffer(CBufferType *buffer, CBufferType *std, uint std_shift);

   virtual bool      NormlizeBufferGradient(CBufferType *output, CBufferType *gradient, CBufferType *std, uint std_shift);



public:

                     CNeuronGPT(void);

                    ~CNeuronGPT(void);

   //---

   virtual bool      Init(const CLayerDescription *desc) override;

   virtual bool      SetOpenCL(CMyOpenCL *opencl) override;

   virtual bool      FeedForward(CNeuronBase *prevLayer) override;

   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer) override;

   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer, bool read) override;

   virtual bool      UpdateWeights(int batch_size, TYPE learningRate,

                                   VECTOR &Beta, VECTOR &Lambda) override;

   //---

   virtual int       GetUnits(void) const { return m_iUnits;   }

   virtual int       GetLayers(void) const { return m_iLayers; }

   //--- file handling methods

   virtual bool      Save(const int file_handle) override;

   virtual bool      Load(const int file_handle) override;

   //--- object identification method

   virtual int       Type(void) override  const { return(defNeuronGPT);  }

  };

//+------------------------------------------------------------------+

//| Class constructor                                                |

//+------------------------------------------------------------------+

CNeuronGPT::CNeuronGPT(void) :   m_iHeads(8),

   m_iWindow(0),

   m_iKeysSize(0),

   m_iUnits(0),

   m_iLayers(0),

   m_iCurrentPosition(0)

  {

  }

//+------------------------------------------------------------------+

//| Class destructor                                                 |

//+------------------------------------------------------------------+

CNeuronGPT::~CNeuronGPT(void)

  {

  }

//+------------------------------------------------------------------+

//| Class initialization method                                      |

//+------------------------------------------------------------------+

bool CNeuronGPT::Init(const CLayerDescription *desc)

  {

//--- check source data

   if(!desc || desc.type != Type() || desc.count <= 0 || desc.window <= 0 ||

      desc.window_out <= 0 || desc.step <= 0 || desc.layers <= 0)

      return false;

//--- save constants

   m_iWindow   = desc.window;

   m_iUnits    = desc.count;

   m_iKeysSize = desc.window_out;

   m_iHeads    = desc.step;

   m_iLayers   = desc.layers;

   if(!ArrayResize(m_dStd, m_iLayers))

      return false;

   for(int l = 0; l < m_iLayers; l++)

      if(!m_dStd[l].BufferInit(1, 2, 1))

         return false;

//--- call the initialization method of the parent class

   CLayerDescription *temp = new CLayerDescription();

   if(!temp || !temp.Copy(desc))

      return false;

   temp.window_out = 1;

   temp.window     = 0;

   temp.activation = AF_NONE;

   if(!CNeuronBase::Init(desc))

      return false;

   delete temp;

//--- run a loop to create internal layer objects

   for(int layer = 0; layer < m_iLayers; layer++)

     {

      //--- create a description for the internal neural layers

      temp = new CLayerDescription();

      if(!temp)

         return false;

      temp.type = defNeuronBase;

      temp.window = m_iWindow;

      temp.count = (int)(3 * m_iKeysSize * m_iHeads);

      temp.activation = AF_NONE;

      temp.optimization = desc.optimization;

      //--- initialize Querys

      CNeuronBase *Querys = new CNeuronBase();

      if(!Querys)

        {

         delete temp;

         return false;

        }

      if(!Querys.Init(temp))

        {

         delete Querys;

         delete temp;

         return false;

        }

      if(!m_cQuerys.Add(Querys))

        {

         delete Querys;

         delete temp;

         return false;

        }

      //--- initialize Keys

      CNeuronBase *Keys = new CNeuronBase();

      if(!Keys)

        {

         delete temp;

         return false;

        }

      temp.window = 0;

      temp.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);

      if(!Keys.Init(temp))

        {

         delete Keys;

         delete temp;

         return false;

        }

      if(!Keys.GetOutputs().Reshape(m_iUnits, m_iKeysSize * m_iHeads))

         return false;

      if(!m_cKeys.Add(Keys))

        {

         delete Keys;

         delete temp;

         return false;

        }

      //--- initialize Values

      CNeuronBase *Values = new CNeuronBase();

      if(!Values)

        {

         delete temp;

         return false;

        }

      if(!Values.Init(temp))

        {

         delete Values;

         delete temp;

         return false;

        }

      if(!Values.GetOutputs().Reshape(m_iUnits, m_iKeysSize * m_iHeads))

         return false;

      if(!m_cValues.Add(Values))

        {

         delete Values;

         delete temp;

         return false;

        }

      //--- initialize Scores

      CNeuronBase *Scores = new CNeuronBase();

      if(!Scores)

        {

         delete temp;

         return false;

        }

      temp.count = (int)(m_iUnits * m_iHeads);

      if(!Scores.Init(temp))

        {

         delete Scores;

         delete temp;

         return false;

        }

      if(!Scores.GetOutputs().Reshape(m_iHeads, m_iUnits))

         return false;

      if(!m_cScores.Add(Scores))

        {

         delete Scores;

         delete temp;

         return false;

        }

      //--- initialize AttentionOut

      CNeuronBase *AttentionOut = new CNeuronBase();

      if(!AttentionOut)

        {

         delete temp;

         return false;

        }

      temp.count = (int)(m_iKeysSize * m_iHeads);

      if(!AttentionOut.Init(temp))

        {

         delete AttentionOut;

         delete temp;

         return false;

        }

      if(!AttentionOut.GetOutputs().Reshape(m_iHeads, m_iKeysSize))

         return false;

      if(!m_cAttentionOut.Add(AttentionOut))

        {

         delete AttentionOut;

         delete temp;

         return false;

        }

      //--- initialize W0

      CNeuronBase *W0 = new CNeuronBase();

      if(!W0)

        {

         delete temp;

         return false;

        }

      temp.window = temp.count;

      temp.count = m_iWindow;

      temp.activation = AF_NONE;

      if(!W0.Init(temp))

        {

         delete W0;

         delete temp;

         return false;

        }

      if(!m_cW0.Add(W0))

        {

         delete W0;

         delete temp;

         return false;

        }

      //--- initialize FF1

      CNeuronBase *FF1 = new CNeuronBase();

      if(!FF1)

        {

         delete temp;

         return false;

        }

      temp.window = m_iWindow;

      temp.count = temp.window * 4;

      temp.activation = AF_SWISH;

      temp.activation_params[0] = 1;

      temp.activation_params[1] = 0;

      if(!FF1.Init(temp))

        {

         delete FF1;

         delete temp;

         return false;

        }

      if(!m_cFF1.Add(FF1))

        {

         delete FF1;

         delete temp;

         return false;

        }

      //--- initialize FF2

      CNeuronBase *FF2 = new CNeuronBase();

      if(!FF2)

        {

         delete temp;

         return false;

        }

      temp.window = temp.count;

      temp.count = m_iWindow;

      temp.activation = AF_NONE;

      if(!FF2.Init(temp))

        {

         delete FF2;

         delete temp;

         return false;

        }

      if(!m_cFF2.Add(FF2))

        {

         delete FF2;

         delete temp;

         return false;

        }

      delete temp;

     }

//--- to avoid copying buffers, substitute them

   if(m_cFF2.Total() < m_iLayers)

      return false;

   if(!m_cOutputs)

      delete m_cOutputs;

   CNeuronBase *neuron = m_cFF2.At(m_iLayers - 1);

   if(!neuron)

      return false;

   m_cOutputs = neuron.GetOutputs();

   if(!m_cGradients)

      delete m_cGradients;

   m_cGradients = neuron.GetGradients();

//---

   SetOpenCL(m_cOpenCL);

//---

   return true;

  }

//+------------------------------------------------------------------+

//| Method for passing a pointer to the OpenCL object to all         |

//| internal objects                                                 |

//+------------------------------------------------------------------+

bool CNeuronGPT::SetOpenCL(CMyOpenCL *opencl)

  {

   CNeuronBase::SetOpenCL(opencl);

   m_cQuerys.SetOpencl(m_cOpenCL);

   m_cKeys.SetOpencl(m_cOpenCL);

   m_cValues.SetOpencl(m_cOpenCL);

   m_cScores.SetOpencl(m_cOpenCL);

   m_cAttentionOut.SetOpencl(m_cOpenCL);

   m_cW0.SetOpencl(m_cOpenCL);

   m_cFF1.SetOpencl(m_cOpenCL);

   m_cFF2.SetOpencl(m_cOpenCL);

   if(m_cOpenCL)

     {

      uint size = sizeof(TYPE) * m_iUnits * m_iHeads;

      m_iScoreTemp = m_cOpenCL.AddBuffer(size, CL_MEM_READ_WRITE);

      for(int l = 0; l < m_iLayers; l++)

         m_dStd[l].BufferCreate(m_cOpenCL);

     }

   else

     {

      for(int l = 0; l < m_iLayers; l++)

         m_dStd[l].BufferFree();

     }

//---

   return(!!m_cOpenCL);

  }

//+------------------------------------------------------------------+

//| Feed-forward method                                              |

//+------------------------------------------------------------------+

bool CNeuronGPT::FeedForward(CNeuronBase *prevLayer)

  {

//--- check the relevance of all objects

   if(!prevLayer || !prevLayer.GetOutputs())

      return false;

//--- increment the pointer to the current object on the data stack

   m_iCurrentPosition++;

   if(m_iCurrentPosition >= m_iUnits)

      m_iCurrentPosition = 0;

//--- run a loop iterating through all internal layers

   CNeuronBase *prevL = prevLayer;

   for(int layer = 0; layer < m_iLayers; layer++)

     {

      CNeuronBase *Querys = m_cQuerys.At(layer);

      if(!Querys || !Querys.FeedForward(prevL))

         return false;

      CNeuronBase *Keys = m_cKeys.At(layer);

      if(!Keys)

         return false;

      CNeuronBase *Values = m_cValues.At(layer);

      if(!Values)

         return false;

      //--- initialize Scores

      CNeuronBase *Scores = m_cScores.At(layer);

      if(!Scores)

         return false;

      //--- initialize AttentionOut

      CNeuronBase *AttentionOut = m_cAttentionOut.At(layer);

      if(!AttentionOut)

         return false;

      //--- branching of the algorithm across computing devices

      if(!m_cOpenCL)

        {

         MATRIX array[];

         if(!Querys.GetOutputs().m_mMatrix.Vsplit(3, array))

            return false;

         if(!Keys.GetOutputs().Row(array[1].Row(0), m_iCurrentPosition))

            return false;

         if(!Values.GetOutputs().Row(array[2].Row(0), m_iCurrentPosition))

            return false;

         MATRIX out;

         if(!out.Init(m_iHeads, m_iKeysSize))

            return false;

         MATRIX array_keys[], array_values[];

         MATRIX array_querys[];

         MATRIX keys = Keys.GetOutputs().m_mMatrix;

         MATRIX values = Values.GetOutputs().m_mMatrix;

         if(!array[0].Vsplit(m_iHeads, array_querys))

            return false;

         if(!keys.Reshape(m_iUnits, m_iHeads * m_iKeysSize))

            return false;

         if(!keys.Vsplit(m_iHeads, array_keys))

            return false;

         if(!values.Reshape(m_iUnits, m_iHeads * m_iKeysSize))

            return false;

         if(!values.Vsplit(m_iHeads, array_values))

            return false;

         //--- define Scores

         for(int head = 0; head < m_iHeads; head++)

           {

            MATRIX score = array_querys[head].MatMul(array_keys[head].Transpose()) / sqrt(m_iKeysSize);

            //--- normalize Scores

            if(!score.Activation(score, AF_SOFTMAX))

               return false;

            if(!Scores.GetOutputs().Row(score.Row(0), head))

               return false;

            //--- output of the Attention block

            if(!out.Row(score.MatMul(array_values[head]).Row(0), head))

               return false;

           }

         if(!out.Reshape(1, m_iHeads * m_iKeysSize))

            return false;

         AttentionOut.GetOutputs().m_mMatrix = out;

        }

      else // OpenCL block

        {

         //--- check data buffers

         if(Querys.GetOutputs().GetIndex() < 0)

            return false;

         if(Keys.GetOutputs().GetIndex() < 0)

            return false;

         if(Values.GetOutputs().GetIndex() < 0)

            return false;

         if(Scores.GetOutputs().GetIndex() < 0)

            return false;

         if(AttentionOut.GetOutputs().GetIndex() < 0)

            return false;

         //--- pass parameters to the kernel

         if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_keys, Keys.GetOutputs().GetIndex()))

            return false;

         if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_outputs, AttentionOut.GetOutputs().GetIndex()))

            return false;

         if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_querys, Querys.GetOutputs().GetIndex()))

            return false;

         if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_scores, Scores.GetOutputs().GetIndex()))

            return false;

         if(!m_cOpenCL.SetArgumentBuffer(def_k_GPTFeedForward, def_gptff_values, Values.GetOutputs().GetIndex()))

            return false;

         if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_key_size, m_iKeysSize))

            return false;

         if(!m_cOpenCL.SetArgument(def_k_GPTFeedForward, def_gptff_units, m_iUnits))