NeuroBook/Include/realization/neuronmhattention.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                            NeuronMHAttention.mqh |

//|                                  Copyright 2021, MetaQuotes Ltd. |

//|                                             https://www.mql5.com |

//+------------------------------------------------------------------+

#property copyright "Copyright 2021, MetaQuotes Ltd."

#property link      "https://www.mql5.com"

//+------------------------------------------------------------------+

//| Connect libraries                                                |

//+------------------------------------------------------------------+

#include "neuronattention.mqh"

//+------------------------------------------------------------------+

//| Class CNeuronMHAttention                                         |

//| Purpose: Class for implementing the multi-head attention block   |

//+------------------------------------------------------------------+

class CNeuronMHAttention    :  public CNeuronAttention

  {

protected:

   CNeuronConv       m_cW0;



   int               m_iHeads;



public:

                     CNeuronMHAttention(void);

                    ~CNeuronMHAttention(void);

   //---

   virtual bool      Init(const CLayerDescription *desc) override;

   virtual bool      SetOpenCL(CMyOpenCL *opencl) override;

   virtual bool      FeedForward(CNeuronBase *prevLayer) override;

   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer) override;

   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer, bool read) override;

   virtual bool      UpdateWeights(int batch_size, TYPE learningRate,

                                   VECTOR &Beta, VECTOR &Lambda) override;

   //--- file handling methods

   virtual bool      Save(const int file_handle) override;

   virtual bool      Load(const int file_handle) override;

   //--- object identification method

   virtual int       Type(void) override const { return(defNeuronMHAttention);  }

  };

//+------------------------------------------------------------------+

//| Class constructor                                                |

//+------------------------------------------------------------------+

CNeuronMHAttention::CNeuronMHAttention(void) :  m_iHeads(8)

  {

  }

//+------------------------------------------------------------------+

//| Class destructor                                                 |

//+------------------------------------------------------------------+

CNeuronMHAttention::~CNeuronMHAttention(void)

  {

  }

//+------------------------------------------------------------------+

//| Class initialization method                                      |

//+------------------------------------------------------------------+

bool CNeuronMHAttention::Init(const CLayerDescription *desc)

  {

//--- check source data

   if(!desc || desc.type != Type() ||

      desc.count <= 0 || desc.window <= 0 || desc.window_out <= 0 ||

      desc.step <= 0)

      return false;

//--- save constants

   m_iWindow = desc.window;

   m_iUnits = desc.count;

   m_iKeysSize = desc.window_out;

   m_iHeads = desc.step;

//--- call the initialization method of the parent class

   CLayerDescription* temp = new CLayerDescription();

   if(!temp)

      return false;

   temp.type = desc.type;

   temp.optimization = desc.optimization;

   temp.activation = AF_NONE;

   temp.count = desc.count;

   temp.window_out = 1;

   temp.window = 0;

   if(!CNeuronBase::Init(temp))

     {

      delete temp;

      return false;

     }

//--- initialize AttentionOut

   temp.type = defNeuronBase;

   temp.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);

   if(!m_cAttentionOut.Init(temp))

     {

      delete temp;

      return false;

     }

   if(!m_cAttentionOut.GetOutputs().m_mMatrix.Reshape(m_iUnits, m_iKeysSize * m_iHeads) ||

      !m_cAttentionOut.GetGradients().m_mMatrix.Reshape(m_iUnits, m_iKeysSize * m_iHeads))

      return false;

//--- create a description for the internal neural layers

   if(!temp)

      return false;

   temp.type = defNeuronConv;

   temp.window = m_iWindow;

   temp.window_out = (int)(m_iKeysSize * m_iHeads);

   temp.step = m_iWindow;

   temp.count = m_iUnits;

//--- initialize Querys

   if(!m_cQuerys.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cQuerys.SetTransposedOutput(true);

//--- initialize Keys

   if(!m_cKeys.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cKeys.SetTransposedOutput(true);

//--- initialize Values

   if(!m_cValues.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cValues.SetTransposedOutput(true);

//--- initialize Scores

   if(!m_cScores.BufferInit(m_iHeads, m_iUnits * m_iUnits))

     {

      delete temp;

      return false;

     }

//--- initialize W0

   temp.window = (int)(m_iKeysSize * m_iHeads);

   temp.step = temp.window;

   temp.window_out = m_iWindow;

   if(!m_cW0.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cW0.SetTransposedOutput(true);

//--- initialize FF1

   temp.window = m_iWindow;

   temp.step = temp.window;

   temp.window_out = temp.window * 4;

   temp.activation = AF_SWISH;

   temp.activation_params[0] = 1;

   temp.activation_params[1] = 0;

   if(!m_cFF1.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cFF1.SetTransposedOutput(true);

//--- initialize FF2

   temp.window = temp.window_out;

   temp.window_out = temp.step;

   temp.step = temp.window;

   temp.activation = desc.activation;

   temp.activation_params = desc.activation_params;

   if(!m_cFF2.Init(temp))

     {

      delete temp;

      return false;

     }

   m_cFF2.SetTransposedOutput(true);

   delete temp;

//--- to avoid copying buffers, substitute them

   if(!SetOutputs(m_cFF2.GetOutputs()))

      return false;

   if(m_cGradients)

      delete m_cGradients;

   m_cGradients = m_cFF2.GetGradients();

//---

   SetOpenCL(m_cOpenCL);

//---

   return true;

  }

//+------------------------------------------------------------------+

//| Method for passing a pointer to the OpenCL object to all         |

//| internal objects                                                 |

//+------------------------------------------------------------------+

bool CNeuronMHAttention::SetOpenCL(CMyOpenCL *opencl)

  {

//--- call of the method of the parent class

   CNeuronAttention::SetOpenCL(opencl);

//--- call the relevant method for the inner layer

   m_cW0.SetOpenCL(m_cOpenCL);

//---

   return(!!m_cOpenCL);

  }

//+------------------------------------------------------------------+

//| Feed-forward method                                              |

//+------------------------------------------------------------------+

bool CNeuronMHAttention::FeedForward(CNeuronBase *prevLayer)

  {

//--- check the relevance of all objects

   if(!prevLayer || !prevLayer.GetOutputs())

      return false;

//---

   if(!m_cQuerys.FeedForward(prevLayer))

      return false;

   if(!m_cKeys.FeedForward(prevLayer))

      return false;

   if(!m_cValues.FeedForward(prevLayer))

      return false;

//--- initialize AttentionOut

   if(!m_cAttentionOut.GetOutputs())

      return false;

//--- branching of the algorithm across computing devices

   MATRIX out;

   if(!m_cOpenCL)

     {

      if(!out.Init(m_iHeads, m_iUnits * m_iKeysSize))

         return false;

      MATRIX querys[], keys[], values[];

      if(!m_cQuerys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, querys))

         return false;

      if(!m_cKeys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, keys))

         return false;

      if(!m_cValues.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values))

         return false;

      for(int head = 0; head < m_iHeads; head++)

        {

         //--- define Scores

         MATRIX sc = exp(querys[head].MatMul(keys[head].Transpose()) / sqrt(m_iKeysSize));

         VECTOR sum = sc.Sum(1);

         for(uint r = 0; r < sc.Rows(); r++)

            if(!sc.Row(sc.Row(r) / sum[r], r))

               return false;

         //--- output of the Attention block

         MATRIX temp = sc.MatMul(values[head]).Transpose();

         if(!temp.Reshape(1, m_iUnits * m_iKeysSize))

            return false;

         if(!sc.Reshape(1, m_iUnits * m_iUnits))

            return false;

         if(!m_cScores.m_mMatrix.Row(sc.Row(0), head))

            return false;

         if(!out.Row(temp.Row(0), head))

            return false;

        }

      if(!out.Reshape(m_iHeads * m_iKeysSize, m_iUnits))

         return false;

      m_cAttentionOut.GetOutputs().m_mMatrix = out.Transpose();

     }

   else // OpenCL block

     {

      //--- check data buffers

      if(m_cQuerys.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cKeys.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cValues.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cScores.GetIndex() < 0)

         return false;

      if(m_cAttentionOut.GetOutputs().GetIndex() < 0)

         return false;

      //--- pass parameters to the kernel

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_keys, m_cKeys.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_outputs, m_cAttentionOut.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_querys, m_cQuerys.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_scores, m_cScores.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_values, m_cValues.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_key_size, m_iKeysSize))

         return false;