NeuroNetworksBook/Include/realization/neuronmhattention.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+
//|                                            NeuronMHAttention.mqh |
//|                                  Copyright 2021, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
//+------------------------------------------------------------------+
//| >4:;NG05< 181;8>B5:8                                            |
//+------------------------------------------------------------------+
#include "neuronattention.mqh"
//+------------------------------------------------------------------+
//| Class CNeuronMHAttention                                         |
//| 07=0G5=85: ;0AA >@30=870F88 @01>BK 1;>:0 <=>3>3>;>2>3> 2=8<0=8O|
//+------------------------------------------------------------------+
class CNeuronMHAttention    :  public CNeuronAttention
  {
protected:
   CNeuronConv       *m_cW0;

   int               m_iHeads;

public:
                     CNeuronMHAttention(void);
                    ~CNeuronMHAttention(void);
   //---
   virtual bool      Init(CLayerDescription *desc);
   virtual bool      SetOpenCL(CMyOpenCL *opencl);
   virtual bool      FeedForward(CNeuronBase *prevLayer);
   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer);
   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer);
   virtual bool      UpdateWeights(int batch_size, double learningRate,
                                   double &Beta[], double &Lambda[]);
   //--- 5B>4K @01>BK A D09;0<8
   virtual bool      Save(const int file_handle);
   virtual bool      Load(const int file_handle);
   //--- 5B>4 845=B8D8:0F88 >1J5:B0
   virtual int       Type(void)     const { return(defNeuronMHAttention);  }
  };
//+------------------------------------------------------------------+
//| >=AB@C:B>@ :;0AA0                                               |
//+------------------------------------------------------------------+
CNeuronMHAttention::CNeuronMHAttention(void) :  m_iHeads(8)
  {
   m_cW0 = new CNeuronConv();
  }
//+------------------------------------------------------------------+
//| 5AB@C:B>@ :;0AA0                                                |
//+------------------------------------------------------------------+
CNeuronMHAttention::~CNeuronMHAttention(void)
  {
   if(CheckPointer(m_cW0) != POINTER_INVALID)
      delete m_cW0;
  }
//+------------------------------------------------------------------+
//| 5B>4 8=8F80;870F88 :;0AA0                                       |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::Init(CLayerDescription *desc)
  {
//--- @>25@O5< 8AE>4=K5 40==K5
   if(!desc || desc.type != Type() ||
      desc.count <= 0 || desc.window <= 0 || desc.window_out <= 0 ||
      desc.step <= 0)
      return false;
//--- !>E@0=O5< :>=AB0=BK
   m_iWindow = desc.window;
   m_iUnits = desc.count;
   m_iKeysSize = desc.window_out;
   m_iHeads = desc.step;
//--- !>740Q< >?8A0=85 4;O 2=CB@5==8E =59@>==KE A;>Q2
   CLayerDescription *temp = new CLayerDescription();
   if(!temp)
      return false;
   temp.type = defNeuronConv;
   temp.window = m_iWindow;
   temp.window_out = (int)(m_iKeysSize * m_iHeads);
   temp.step = m_iWindow;
   temp.count = m_iUnits;
   temp.activation = ACT_None;
   temp.activation_params[0] = 1;
   temp.activation_params[1] = 0;
   temp.optimization = desc.optimization;
//--- K7K205< <5B>4 8=8F80;870F88 @>48B5;LA:>3> :;0AA0
   desc.count *= m_iWindow;
   desc.window_out = 1;
   desc.window = 0;
   if(!CNeuronBase::Init(desc))
     {
      delete temp;
      return false;
     }
//--- =8F80;878@C5< Querys
   if(CheckPointer(m_cQuerys) == POINTER_INVALID)
     {
      m_cQuerys = new CNeuronConv();
      if(CheckPointer(m_cQuerys) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   if(!m_cQuerys.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cQuerys.SetTransposedOutput(true);
//--- =8F80;878@C5< Keys
   if(CheckPointer(m_cKeys) == POINTER_INVALID)
     {
      m_cKeys = new CNeuronConv();
      if(CheckPointer(m_cKeys) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   if(!m_cKeys.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cKeys.SetTransposedOutput(true);
//--- =8F80;878@C5< Values
   if(CheckPointer(m_cValues) == POINTER_INVALID)
     {
      m_cValues = new CNeuronConv();
      if(CheckPointer(m_cValues) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   if(!m_cValues.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cValues.SetTransposedOutput(true);
//--- =8F80;878@C5< Scores
   if(CheckPointer(m_cScores) == POINTER_INVALID)
     {
      m_cScores = new CBufferDouble();
      if(CheckPointer(m_cScores) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   if(!m_cScores.BufferInit(m_iHeads, m_iUnits * m_iUnits))
     {
      delete temp;
      return false;
     }
//--- =8F80;878@C5< AttentionOut
   if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
     {
      m_cAttentionOut = new CNeuronBase();
      if(CheckPointer(m_cAttentionOut) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   desc.type = defNeuronBase;
   desc.count = (int)(m_iUnits * m_iKeysSize * m_iHeads);
   if(!m_cAttentionOut.Init(desc))
     {
      delete temp;
      return false;
     }
   desc.count = m_iUnits * m_iWindow;
//--- =8F80;878@C5< W0
   if(CheckPointer(m_cW0) == POINTER_INVALID)
     {
      m_cW0 = new CNeuronConv();
      if(CheckPointer(m_cW0) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   temp.window = (int)(m_iKeysSize * m_iHeads);
   temp.step = temp.window;
   temp.window_out = m_iWindow;
   temp.activation = ACT_None;
   temp.activation_params[0] = 1;
   temp.activation_params[1] = 0;
   if(!m_cW0.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cW0.SetTransposedOutput(true);
//--- =8F80;878@C5< FF1
   if(CheckPointer(m_cFF1) == POINTER_INVALID)
     {
      m_cFF1 = new CNeuronConv();
      if(CheckPointer(m_cFF1) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   temp.window = m_iWindow;
   temp.step = temp.window;
   temp.window_out = temp.window * 4;
   temp.activation = ACT_SWISH;
   temp.activation_params[0] = 1;
   temp.activation_params[1] = 0;
   if(!m_cFF1.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cFF1.SetTransposedOutput(true);
//--- =8F80;878@C5< FF2
   if(CheckPointer(m_cFF2) == POINTER_INVALID)
     {
      m_cFF2 = new CNeuronConv();
      if(CheckPointer(m_cFF2) == POINTER_INVALID)
        {
         delete temp;
         return false;
        }
     }
   temp.window = temp.window_out;
   temp.window_out = temp.step;
   temp.step = temp.window;
   temp.activation = ACT_None;
   temp.activation_params[0] = 1;
   temp.activation_params[1] = 0;
   if(!m_cFF2.Init(temp))
     {
      delete temp;
      return false;
     }
   m_cFF2.SetTransposedOutput(true);
   delete temp;
//--- ;O 8A:;NG5=88O :>?8@>20=8O 1CD5@>2 >ACI5AB28< 8E ?>4<5=C
   if(m_cOutputs)
      delete m_cOutputs;
   m_cOutputs = m_cFF2.GetOutputs();
   if(m_cGradients)
      delete m_cGradients;
   m_cGradients = m_cFF2.GetGradients();
//---
   SetOpenCL(m_cOpenCL);
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 ?5@540G8 C:070B5;O =0 >1J5:B OpenCL 4> 2A5E                |
//| 2=CB@5==8E >1J5:B>2                                              |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::SetOpenCL(CMyOpenCL *opencl)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   CNeuronAttention::SetOpenCL(opencl);
//--- K7K205< 0=0;>38G=K9 <5B>4 4;O 2=CB@5==53> A;>O
   if(CheckPointer(m_cW0) != POINTER_INVALID)
      m_cW0.SetOpenCL(m_cOpenCL);
//---
   return(CheckPointer(m_cOpenCL) != POINTER_INVALID);
  }
//+------------------------------------------------------------------+
//| 5B>4 ?@O<>3> ?@>E>40                                            |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::FeedForward(CNeuronBase *prevLayer)
  {
//--- @>25@O5< 0:BC0;L=>ABL 2A5E >1J5:B>2
   if(CheckPointer(prevLayer) == POINTER_INVALID ||
      CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
      CheckPointer(m_cQuerys) == POINTER_INVALID ||
      CheckPointer(m_cValues) == POINTER_INVALID ||
      CheckPointer(m_cKeys) == POINTER_INVALID ||
      CheckPointer(m_cW0) == POINTER_INVALID ||
      CheckPointer(m_cFF1) == POINTER_INVALID ||
      CheckPointer(m_cFF2) == POINTER_INVALID)
      return false;
//---
   if(!m_cQuerys.FeedForward(prevLayer))
      return false;
   if(!m_cKeys.FeedForward(prevLayer))
      return false;
   if(!m_cValues.FeedForward(prevLayer))
      return false;
//--- =8F80;878@C5< Scores
   if(CheckPointer(m_cScores) == POINTER_INVALID)
     {
      m_cScores = new CBufferDouble();
      if(CheckPointer(m_cScores) == POINTER_INVALID)
         return false;
     }
//--- =8F80;878@C5< AttentionOut
   if(!m_cAttentionOut)
      return false;
   if(!m_cAttentionOut.GetOutputs())
      return false;
//---  0725B2;5=85 0;3>@8B<0 ?> 2KG8A;8B5;L=><C CAB@>9AB2C
   MATRIX out;
   if(!m_cOpenCL)
     {
      if(!out.Init(m_iHeads, m_iUnits * m_iKeysSize))
         return false;
      MATRIX querys[], keys[], values[];
      if(!m_cQuerys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, querys))
         return false;
      if(!m_cKeys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, keys))
         return false;
      if(!m_cValues.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values))
         return false;
      for(int head = 0; head < m_iHeads; head++)
        {
         if(!querys[head].Reshape(m_iUnits, m_iKeysSize) ||
            !keys[head].Reshape(m_iUnits, m_iKeysSize) ||
            !values[head].Reshape(m_iUnits, m_iKeysSize))
            return false;
         //--- ?@545;O5< Scores
         MATRIX sc = querys[head].MatMul(keys[head].Transpose()) / sqrt(m_iKeysSize);
         for(uint r = 0; r < sc.Rows()*sc.Cols(); r++)
            if(!sc.Flat(r, exp(sc.Flat(r))))
               return false;
         VECTOR sum = sc.Sum(1);
         for(uint r = 0; r < sc.Rows(); r++)
            if(!sc.Row(sc.Row(r) / sum[r], r))
               return false;
         MATRIX temp = sc;
         if(!temp.Reshape(1, m_iUnits * m_iUnits))
            return false;
         if(!m_cScores.m_mMatrix.Row(temp.Row(0), head))
            return false;
         //--- KE>4 1;>:0 2=8<0=8O
         temp = sc * values[head];
         if(!temp.Reshape(1, m_iUnits * m_iKeysSize))
            return false;
         if(!out.Row(temp.Row(0), head))
            return false;
        }
      if(!out.Reshape(1, m_cAttentionOut.GetOutputs().Total()))
         return false;
      m_cAttentionOut.GetOutputs().m_mMatrix = out;
     }
   else // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      if(m_cQuerys.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cKeys.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cValues.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cScores.GetIndex() < 0)
         return false;
      if(m_cAttentionOut.GetOutputs().GetIndex() < 0)
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_keys, m_cKeys.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_outputs, m_cAttentionOut.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_querys, m_cQuerys.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_scores, m_cScores.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionFeedForward, def_attff_values, m_cValues.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_key_size, m_iKeysSize))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_window, m_iKeysSize))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_AttentionFeedForward, def_attff_mask, 0))
         return false;
      //--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0, 0};
      int NDRange[] = {m_iUnits, m_iHeads};
      if(!m_cOpenCL.Execute(def_k_AttentionFeedForward, 2, off_set, NDRange))
         return false;
      //--- !G8BK20=885 @57C;LB0B>2 >?5@0F89
      if(!m_cAttentionOut.GetOutputs().BufferRead())
         return false;
     }
//---
   if(!m_cW0.FeedForward(m_cAttentionOut))
      return false;
   int total = m_cW0.GetOutputs().GetData(out, false);
   if(total <= 0)
      return false;
//--- !C<<8@C5< A 8AE>4=K<8 40==K<8 8 =>@<0;87C5<
   out += prevLayer.GetOutputs().m_mMatrix;
   double mean = out.Mean();
   m_dStd[0] = out.Std();
   m_cW0.GetOutputs().m_mMatrix = out = (out - mean) / m_dStd[0];
   if(m_cOpenCL && !m_cW0.GetOutputs().BufferWrite())
      return false;
//---
   if(!m_cFF1.FeedForward(m_cW0))
      return false;
   if(!m_cFF2.FeedForward(m_cFF1))
      return false;
//--- !C<<8@C5< A 2KE>4>< 2=8<0=8O 8 =>@<0;87C5<
   out += m_cOutputs.m_mMatrix;
   mean = out.Mean();
   m_dStd[1] = out.Std();
   m_cOutputs.m_mMatrix = out = (out - mean) / m_dStd[1];
   if(m_cOpenCL && !m_cOutputs.BufferWrite())
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O 3@0485=B0 >H81:8 G5@57 A:@KBK9 A;>9          |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::CalcHiddenGradient(CNeuronBase *prevLayer)
  {
//--- @>25@O5< 0:BC0;L=>ABL 2A5E >1J5:B>2
   if(CheckPointer(m_cOutputs) == POINTER_INVALID ||
      CheckPointer(m_cGradients) == POINTER_INVALID ||
      CheckPointer(m_cScores) == POINTER_INVALID ||
      CheckPointer(m_cFF2) == POINTER_INVALID ||
      CheckPointer(m_cQuerys) == POINTER_INVALID ||
      CheckPointer(m_cKeys) == POINTER_INVALID ||
      CheckPointer(m_cValues) == POINTER_INVALID ||
      m_cOutputs.Total() != m_cGradients.Total())
      return false;
//--- 0AHB018@>20=85 3@0485=B0 >H81:8
   if(m_dStd[1] != 0 && m_cGradients.Scaling(1 / m_dStd[1]) <= 0)
      return false;
//--- @>2>48< 3@0485=B >H81:8 G5@57 1;>: Feed Forward
   if(!m_cFF2.CalcHiddenGradient(m_cFF1))
      return false;
   if(!m_cFF1.CalcHiddenGradient(m_cW0))
      return false;
   m_cW0.GetGradients().m_mMatrix += m_cGradients.m_mMatrix;
   if(m_dStd[0] != 0)
      m_cW0.GetGradients().m_mMatrix /= m_dStd[0];
   if(m_cOpenCL && !m_cW0.GetGradients().BufferWrite())
      return false;
//---  0A?@545;5=8O 3@0;85=B0 >H81:8 ?> 3>;>20< 2=8<0=8O
   if(!m_cW0.CalcHiddenGradient(m_cAttentionOut))
      return false;
//---  0725B2;5=85 0;3>@8B<0 ?> 2KG8A;8B5;L=><C CAB@>9AB2C
   MATRIX attention_grad = m_cAttentionOut.GetGradients().m_mMatrix;
   if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
     {
      MATRIX gradients[];
      MATRIX querys[], querys_grad;
      MATRIX keys[], keys_grad;
      MATRIX values[], values_grad;
      if(!m_cQuerys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, querys) ||
         !m_cKeys.GetOutputs().m_mMatrix.Vsplit(m_iHeads, keys) ||
         !m_cValues.GetOutputs().m_mMatrix.Vsplit(m_iHeads, values) ||
         !attention_grad.Vsplit(m_iHeads, gradients))
         return false;
      if(!querys_grad.Init(m_iHeads, m_iUnits * m_iKeysSize) ||
         !keys_grad.Init(m_iHeads, m_iUnits * m_iKeysSize) ||
         !values_grad.Init(m_iHeads, m_iUnits * m_iKeysSize))
         return false;
      //---  0A?@545;5=85 3@0485=B0 =0 Values
      for(int head = 0; head < m_iHeads; head++)
        {
         if(!querys[head].Reshape(m_iUnits, m_iKeysSize) ||
            !keys[head].Reshape(m_iUnits, m_iKeysSize) ||
            !values[head].Reshape(m_iUnits, m_iKeysSize))
            return false;
         MATRIX score;
         if(!score.Init(1, m_iUnits * m_iUnits) ||
            !score.Row(m_cScores.m_mMatrix.Row(head), 0) ||
            !score.Reshape(m_iUnits, m_iUnits))
            return false;
         MATRIX temp = score.Transpose().MatMul(gradients[head]);
         if(!temp.Reshape(1, m_iUnits * m_iKeysSize))
            return false;
         if(!values_grad.Row(temp.Row(0), head))
            return false;
         //---  0A?@545;5=85 3@0485=B0 =0 Querys 8 Keys
         gradients[head] = gradients[head].MatMul(values[head].Transpose());
         //---
         for(int r = 0; r < m_iUnits; r++)
           {
            temp.Init(m_iUnits, m_iUnits);
            temp.Identity();
            VECTOR s = score.Row(r);
            for(int c = 0; c < m_iUnits; c++)
               if(!temp.Col((temp.Col(c) - s), c))
                  return false;
            s = s.MatMul(temp);
            if(!gradients[head].Row(s * gradients[head].Row(r) / sqrt(m_iKeysSize), r))
               return false;
           }
         temp = gradients[head].MatMul(keys[head]);
         if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
            !querys_grad.Row(temp.Row(0), head))
            return false;
         temp = gradients[head].Transpose().MatMul(querys[head]);
         if(!temp.Reshape(1, m_iUnits * m_iKeysSize) ||
            !keys_grad.Row(temp.Row(0), head))
            return false;
        }
      //---
      if(!querys_grad.Reshape(1, m_cQuerys.GetGradients().Total()) ||
         !keys_grad.Reshape(1, m_cKeys.GetGradients().Total()) ||
         !values_grad.Reshape(1, m_cValues.GetGradients().Total()))
         return false;
      m_cQuerys.GetGradients().m_mMatrix = querys_grad;
      m_cKeys.GetGradients().m_mMatrix = keys_grad;
      m_cValues.GetGradients().m_mMatrix = values_grad;
     }
   else // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      if(m_cValues.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cValues.GetGradients().GetIndex() < 0)
         return false;
      if(m_cScores.GetIndex() < 0)
         return false;
      if(m_cAttentionOut.GetGradients().GetIndex() < 0)
         return false;
      if(m_cScoreGrad.GetIndex() < 0)
         return false;
      if(m_cScoreTemp.GetIndex() < 0)
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_outputs_grad, m_cAttentionOut.GetGradients().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores, m_cScores.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores_grad, m_cScoreGrad.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_scores_temp, m_cScoreTemp.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_values, m_cValues.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionScoreGradients, def_attscr_values_grad, m_cValues.GetGradients().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_AttentionScoreGradients, def_attscr_window, m_iKeysSize))
         return false;
      //--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0, 0};
      int NDRange[] = {m_iUnits, m_iHeads};
      if(!m_cOpenCL.Execute(def_k_AttentionScoreGradients, 2, off_set, NDRange))
         return false;
      //--- 03@C7:0 @57C;LB0B>2
      if(!m_cValues.GetGradients().BufferRead())
         return false;
      //--- !>740=85 1CD5@>2 40==KE
      if(m_cQuerys.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cQuerys.GetGradients().GetIndex() < 0)
         return false;
      if(m_cKeys.GetOutputs().GetIndex() < 0)
         return false;
      if(m_cKeys.GetGradients().GetIndex() < 0)
         return false;
      //--- 5@540G0 0@3C<5=B>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_keys, m_cKeys.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_keys_grad, m_cKeys.GetGradients().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_querys, m_cQuerys.GetOutputs().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_querys_grad, m_cQuerys.GetGradients().GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_AttentionHiddenGradients, def_atthgr_scores_grad, m_cScoreGrad.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_AttentionHiddenGradients, def_atthgr_key_size, m_iKeysSize))
         return false;
      //--- >AB0=>2:0 :5@=5;0 2 >G5@54L 2K?>;=5=8O
      if(!m_cOpenCL.Execute(def_k_AttentionHiddenGradients, 2, off_set, NDRange))
         return false;
      //--- 03@C7:0 @57C;LB0B>2
      if(!m_cQuerys.GetGradients().BufferRead())
         return false;
     }
//--- 5@5=>A 3@0485=B0 >H81:8 =0 ?@54K4CI89 A;>9
   attention_grad = m_cW0.GetGradients().m_mMatrix;
   if(!m_cValues.CalcHiddenGradient(prevLayer))
      return false;
   attention_grad += prevLayer.GetGradients().m_mMatrix;
   if(!m_cQuerys.CalcHiddenGradient(prevLayer))
      return false;
   attention_grad += prevLayer.GetGradients().m_mMatrix;
   if(!m_cKeys.CalcHiddenGradient(prevLayer))
      return false;
   prevLayer.GetGradients().m_mMatrix += attention_grad;
   if(m_cOpenCL && !prevLayer.GetGradients().BufferWrite())
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O 3@0485=B0 >H81:8 4> <0B@8F 25A>2             |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::CalcDeltaWeights(CNeuronBase *prevLayer)
  {
//--- ;>: :>=B@>;59
   if(CheckPointer(m_cFF2) == POINTER_INVALID)
      return false;
//--- K7K205< 0=0;>38G=K9 <5B>4 4;O 2A5E 2=CB@5==8E A;>Q2
   if(!m_cFF2.CalcDeltaWeights(m_cFF1))
      return false;
   if(!m_cFF1.CalcDeltaWeights(m_cW0))
      return false;
   if(!m_cW0.CalcDeltaWeights(m_cAttentionOut))
      return false;
   if(CheckPointer(m_cQuerys) == POINTER_INVALID)
      return false;
   if(!m_cQuerys.CalcDeltaWeights(prevLayer))
      return false;
   if(CheckPointer(m_cKeys) == POINTER_INVALID)
      return false;
   if(!m_cKeys.CalcDeltaWeights(prevLayer))
      return false;
   if(CheckPointer(m_cValues) == POINTER_INVALID)
      return false;
   if(!m_cValues.CalcDeltaWeights(prevLayer))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 >1=>2;5=8O <0B@8F 25A>2KE :>MDD8F85=B>2                    |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::UpdateWeights(int batch_size, double learningRate, double &Beta[], double &Lambda[])
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronAttention::UpdateWeights(batch_size, learningRate, Beta, Lambda))
      return false;
//--- ;>: :>=B@>;59
   if(CheckPointer(m_cW0) == POINTER_INVALID)
      return false;
//--- K7K205< 0=0;>38G=K9 <5B>4 4;O 2A5E 2=CB@5==8E A;>Q2
   if(!m_cW0.UpdateWeights(batch_size, learningRate, Beta, Lambda))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 A>E@0=5=8O M;5<5=B>2 :;0A0 2 D09;                          |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::Save(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronAttention::Save(file_handle))
      return false;
//--- !>E@0=O5< :>=AB0=BK
   if(FileWriteInteger(file_handle, m_iHeads) <= 0)
      return false;
//--- K7K205< 0=0;>38G=K9 <5B>4 4;O 2A5E 2=CB@5==8E A;>Q2
   if(CheckPointer(m_cW0) == POINTER_INVALID)
      return false;
   if(!m_cW0.Save(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 2>AAB0=>2;5=8O :;0AA0 87 A>E@0=Q==KE 40==KE                |
//+------------------------------------------------------------------+
bool CNeuronMHAttention::Load(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronAttention::Load(file_handle))
      return false;
//--- 03@C605< :>=AB0=BK
   m_iHeads = FileReadInteger(file_handle);
//--- K7K205< 0=0;>38G=K9 <5B>4 4;O 2A5E 2=CB@5==8E A;>Q2
   if(CheckPointer(m_cW0) == POINTER_INVALID)
     {
      m_cW0 = new CNeuronConv();
      if(CheckPointer(m_cW0) == POINTER_INVALID)
         return false;
     }
   if(FileReadInteger(file_handle) != defNeuronConv || !m_cW0.Load(file_handle))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+