NeuroBook/Include/realization/neuronattention.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                              NeuronAttention.mqh |

//|                                  Copyright 2021, MetaQuotes Ltd. |

//|                                             https://www.mql5.com |

//+------------------------------------------------------------------+

#property copyright "Copyright 2021, MetaQuotes Ltd."

#property link      "https://www.mql5.com"

//+------------------------------------------------------------------+

//| Connect libraries                                                |

//+------------------------------------------------------------------+

#ifndef Defines

#include "defines.mqh"

#endif

#include "neuronconv.mqh"

#include <Math\Stat\Math.mqh>

//+------------------------------------------------------------------+

//| Class CNeuronAttention                                           |

//| Purpose: Self-Attention block class                              |

//+------------------------------------------------------------------+

class CNeuronAttention    :  public CNeuronBase

  {

protected:

   CNeuronConv       m_cQuerys;

   CNeuronConv       m_cKeys;

   CNeuronConv       m_cValues;

   CBufferType       m_cScores;

   int               m_cScoreGrad;

   int               m_cScoreTemp;

   CNeuronBase       m_cAttentionOut;

   CNeuronConv       m_cFF1;

   CNeuronConv       m_cFF2;

   //---

   int               m_iWindow;

   int               m_iUnits;

   int               m_iKeysSize;

   CBufferType       m_cStd;

   //---

   virtual bool      NormlizeBuffer(CBufferType *buffer, CBufferType *std, uint std_shift);

   virtual bool      NormlizeBufferGradient(CBufferType *output, CBufferType *gradient, CBufferType *std, uint std_shift);



public:

                     CNeuronAttention(void);

                    ~CNeuronAttention(void);

   //---

   virtual bool      Init(const CLayerDescription *desc) override;

   virtual bool      SetOpenCL(CMyOpenCL *opencl) override;

   virtual bool      FeedForward(CNeuronBase *prevLayer) override;

   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer) override;

   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer, bool read) override;

   virtual bool      UpdateWeights(int batch_size, TYPE learningRate,

                                   VECTOR &Beta, VECTOR &Lambda) override;

   //--- file handling methods

   virtual bool      Save(const int file_handle) override;

   virtual bool      Load(const int file_handle) override;

   //--- object identification method

   virtual int       Type(void) override  const { return(defNeuronAttention); }

  };

//+------------------------------------------------------------------+

//| Class constructor                                                |

//+------------------------------------------------------------------+

CNeuronAttention::CNeuronAttention(void) :   m_iWindow(1),

   m_iUnits(0),

   m_iKeysSize(1)

  {

   m_cStd.BufferInit(1, 2, 1);

  }

//+------------------------------------------------------------------+

//| Class destructor                                                 |

//+------------------------------------------------------------------+

CNeuronAttention::~CNeuronAttention(void)

  {

  }

//+------------------------------------------------------------------+

//| Class initialization method                                      |

//+------------------------------------------------------------------+

bool CNeuronAttention::Init(const CLayerDescription *desc)

  {

//--- check source data

   if(!desc || desc.type != Type() || desc.count <= 0 || desc.window <= 0 || desc.window_out <= 0)

      return false;

//---

   m_iWindow   = desc.window;

   m_iUnits    = desc.count;

   m_iKeysSize = desc.window_out;

//--- call the initialization method of the parent class

   CLayerDescription *temp = new CLayerDescription();

   if(!temp)

      return false;

   temp.count = desc.count * desc.window;

   temp.window_out = 1;

   temp.window     = 0;

   temp.optimization = desc.optimization;

   temp.activation = desc.activation;

   temp.activation_params = desc.activation_params;

   temp.type = desc.type;

   if(!CNeuronBase::Init(temp))

     {

      delete temp;

      return false;

     }

//--- initialize AttentionOut

   temp.type = defNeuronBase;

   temp.activation = AF_NONE;

   if(!m_cAttentionOut.Init(temp))

     {

      delete temp;

      return false;

     }

//--- create a description for the internal neural layers

   temp.type = defNeuronConv;

   temp.window = desc.window;

   temp.window_out = m_iKeysSize;

   temp.step = desc.window;

   temp.count = desc.count;

   temp.probability = 1;

//--- initialize Querys

   if(!m_cQuerys.Init(temp))

     {

      delete temp;

      return false;

     }

//--- initialize Keys

   if(!m_cKeys.Init(temp))

     {

      delete temp;

      return false;

     }

//--- initialize Values

   temp.window_out = m_iWindow;

   if(!m_cValues.Init(temp))

     {

      delete temp;

      return false;

     }

//--- initialize Scores

   if(!m_cScores.BufferInit(temp.count, temp.count, 0))

     {

      delete temp;

      return false;

     }

//--- initialize FF1

   temp.window_out *= 4;

   temp.activation = AF_SWISH;

   temp.activation_params[0] = 1;

   temp.activation_params[1] = 0;

   if(!m_cFF1.Init(temp) || !m_cFF1.SetTransposedOutput(true))

     {

      delete temp;

      return false;

     }

//--- initialize FF2

   temp.window = temp.window_out;

   temp.window_out = temp.step;

   temp.step = temp.window;

   temp.activation = desc.activation;

   temp.activation_params = desc.activation_params;

   if(!m_cFF2.Init(temp) || !m_cFF2.SetTransposedOutput(true))

     {

      delete temp;

      return false;

     }

   delete temp;

//--- to avoid copying buffers, substitute them

   if(m_cOutputs)

      delete m_cOutputs;

   m_cOutputs = m_cFF2.GetOutputs();

   if(m_cGradients)

      delete m_cGradients;

   m_cGradients = m_cFF2.GetGradients();

//--- pass the pointer to the OpenCL working object to all internal object

   SetOpenCL(m_cOpenCL);

//---

   return true;

  }

//+------------------------------------------------------------------+

//| Method for passing a pointer to the OpenCL object to all         |

//| internal object of the class                                     |

//+------------------------------------------------------------------+

bool CNeuronAttention::SetOpenCL(CMyOpenCL *opencl)

  {

   CNeuronBase::SetOpenCL(opencl);

   m_cQuerys.SetOpenCL(m_cOpenCL);

   m_cKeys.SetOpenCL(m_cOpenCL);

   m_cValues.SetOpenCL(m_cOpenCL);

   m_cAttentionOut.SetOpenCL(m_cOpenCL);

   m_cFF1.SetOpenCL(m_cOpenCL);

   m_cFF2.SetOpenCL(m_cOpenCL);

   if(m_cOpenCL)

     {

      m_cScores.BufferCreate(m_cOpenCL);

      ulong size = sizeof(TYPE) * m_cScores.Total();

      m_cScoreGrad = m_cOpenCL.AddBuffer((uint)size, CL_MEM_READ_WRITE);

      m_cScoreTemp = m_cOpenCL.AddBuffer((uint)size, CL_MEM_READ_WRITE);

      m_cStd.BufferCreate(m_cOpenCL);

     }

   else

     {

      m_cScores.BufferFree();

      m_cStd.BufferFree();

     }

//---

   return(!!m_cOpenCL);

  }

//+------------------------------------------------------------------+

//| Feed-forward method                                              |

//+------------------------------------------------------------------+

bool CNeuronAttention::FeedForward(CNeuronBase *prevLayer)

  {

//--- calculate vectors Query, Key, Value

   if(!m_cQuerys.FeedForward(prevLayer))

      return false;

   if(!m_cKeys.FeedForward(prevLayer))

      return false;

   if(!m_cValues.FeedForward(prevLayer))

      return false;

//--- branching of the algorithm across computing devices

   MATRIX out;

   if(!m_cOpenCL)

     {

      MATRIX querys = m_cQuerys.GetOutputs().m_mMatrix;

      MATRIX keys = m_cKeys.GetOutputs().m_mMatrix;

      //--- define Scores

      MATRIX scores = MathExp(querys.MatMul(keys.Transpose()) / sqrt(m_iKeysSize));

      //--- normalize Scores

      VECTOR summs = scores.Sum(1);

      for(int r = 0; r < m_iUnits; r++)

         if(!scores.Row(scores.Row(r) / summs[r], r))

            return false;

      m_cScores.m_mMatrix = scores;

      //--- output of the Attention block

      MATRIX values = m_cValues.GetOutputs().m_mMatrix;

      out = scores.MatMul(values);

      //--- sum with source data and normalize

      if(!out.Reshape(prevLayer.Rows(), prevLayer.Cols()))

         return false;

      m_cAttentionOut.GetOutputs().m_mMatrix = out;

     }

   else // OpenCL block

     {

      //--- check data buffers

      if(m_cQuerys.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cKeys.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cValues.GetOutputs().GetIndex() < 0)

         return false;

      if(m_cScores.GetIndex() < 0)

         return false;

      if(m_cAttentionOut.GetOutputs().GetIndex() < 0)

         return false;

      //--- pass parameters to the kernel

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_keys, m_cKeys.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_outputs, m_cAttentionOut.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_querys, m_cQuerys.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_scores, m_cScores.GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_values, m_cValues.GetOutputs().GetIndex()))

         return false;

      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_key_size, m_iKeysSize))

         return false;

      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_window, m_iWindow))

         return false;

      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_mask, 0))

         return false;

      //--- place kernel to the execution queue

      int off_set[] = {0, 0};

      int NDRange[] = {m_iUnits, 1};

      if(!m_cOpenCL.Execute(def_k_AttentionFeedForward, 2, off_set, NDRange))

         return false;

     }

//--- sum with the source data

   if(!m_cAttentionOut.GetOutputs().SumArray(prevLayer.GetOutputs()))

      return false;

//--- normalize

   if(!NormlizeBuffer(m_cAttentionOut.GetOutputs(), GetPointer(m_cStd), 0))

      return false;

//--- call feed-forward pass methods for the Feed Forward block levels

   if(!m_cFF1.FeedForward(GetPointer(m_cAttentionOut)))

      return false;

   if(!m_cFF2.FeedForward(GetPointer(m_cFF1)))

      return false;

//--- sum with the Attention output and normalize

   if(!m_cOutputs.SumArray(m_cAttentionOut.GetOutputs()))

      return false;

//--- normalize

   if(!NormlizeBuffer(m_cOutputs, GetPointer(m_cStd), 1))

      return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//| Method for propagating gradient through the hidden layer         |

//+------------------------------------------------------------------+

bool CNeuronAttention::CalcHiddenGradient(CNeuronBase *prevLayer)

  {

//--- check the relevance of all objects

   if(!m_cOutputs || !m_cGradients ||

      m_cOutputs.Total() != m_cGradients.Total())

      return false;