NeuroNetworksBook/Include/realization/neurondropout.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+
//|                                                NeuronDropout.mqh |
//|                                  Copyright 2021, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
//+------------------------------------------------------------------+
//| >4:;NG05< 181;8>B5:8                                            |
//+------------------------------------------------------------------+
#include "neuronbase.mqh"
//+------------------------------------------------------------------+
//| Class CNeuronDropout                                             |
//| 07=0G5=85: ;0AA @50;870F88 <5B>40 Dropout                      |
//+------------------------------------------------------------------+
class CNeuronDropout    :  public CNeuronBase
  {
protected:
   double            m_dOutProbability;
   int               m_iOutNumber;
   double            m_dInitValue;

   CBufferDouble     *m_cDropOutMultiplier;

public:
                     CNeuronDropout(void);
                    ~CNeuronDropout(void);
   //---
   virtual bool      Init(CLayerDescription *description);
   virtual bool      FeedForward(CNeuronBase *prevLayer);
   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer);
   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer) { return true; }
   virtual bool      UpdateWeights(int batch_size, double learningRate,
                                   double &Beta[], double &Lambda[])
     {
      return true;
     }

   //--- 5B>4K @01>BK A D09;0<8
   virtual bool      Save(const int file_handle);
   virtual bool      Load(const int file_handle);
   //--- 5B>4 845=B8D8:0F88 >1J5:B0
   virtual int       Type(void)           const { return(defNeuronDropout); }
  };
//+------------------------------------------------------------------+
//| >=AB@C:B>@ :;0AA0                                               |
//+------------------------------------------------------------------+
CNeuronDropout::CNeuronDropout(void)   :  m_dInitValue(1.0),
   m_dOutProbability(0),
   m_iOutNumber(0)
  {
   m_cDropOutMultiplier = new CBufferDouble();
   m_bTrain = false;
  }
//+------------------------------------------------------------------+
//| 5AB@C:B>@ :;0AA0                                                |
//+------------------------------------------------------------------+
CNeuronDropout::~CNeuronDropout(void)
  {
   if(CheckPointer(m_cDropOutMultiplier) != POINTER_INVALID)
      delete m_cDropOutMultiplier;
  }
//+------------------------------------------------------------------+
//| 5B>4 8=8F80;870F88 :;0AA0                                       |
//+------------------------------------------------------------------+
bool CNeuronDropout::Init(CLayerDescription *description)
  {
//--- ;>: :>=B@>;59
   if(!CheckPointer(description) == POINTER_INVALID ||
      description.count != description.window)
      return false;
   description.window = 0;
   if(!CNeuronBase::Init(description))
      return false;
//---  0AGQB :>MDD8F85=B>2
   m_dOutProbability = MathMin(description.probability, 0.9);
   if(m_dOutProbability < 0)
      return false;
   m_iOutNumber = (int)(m_cOutputs.Total() * m_dOutProbability);
   m_dInitValue = 1.0 / (1.0 - m_dOutProbability);
   if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
     {
      m_cDropOutMultiplier = new CBufferDouble();
      if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
         return false;
     }
//--- =8F88@C5< 1CD5@ <0A:8@>20=8O
   if(!m_cDropOutMultiplier.BufferInit(m_cOutputs.Rows(), m_cOutputs.Cols(), m_dInitValue))
      return false;
   m_bTrain = true;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 ?@O<>3> ?@>E>40                                            |
//+------------------------------------------------------------------+
bool CNeuronDropout::FeedForward(CNeuronBase *prevLayer)
  {
//--- ;>: :>=B@>;59
   if(!prevLayer || !prevLayer.GetOutputs() || !m_cOutputs || !m_cDropOutMultiplier)
      return false;
//--- @>2525@:0 D;030 @568<0 @01>BK
   if(!m_bTrain)
     {
      m_cOutputs.m_mMatrix = prevLayer.GetOutputs().m_mMatrix;
      if(m_cOpenCL && !m_cOutputs.BufferWrite())
         return false;
      return true;
     }
//--- 5=5@8@C5< B5=7>@ <0A:8@>20=8O 40==KE
   uint total = m_cOutputs.Total();
   if(!m_cDropOutMultiplier.m_mMatrix.Fill(m_dInitValue))
      return false;
   for(int i = 0; i < m_iOutNumber; i++)
     {
      int pos = (int)(MathRand() * MathRand() / MathPow(32767.0, 2) * total);
      if(m_cDropOutMultiplier.m_mMatrix.Flat(pos) == 0)
        {
         i--;
         continue;
        }
      if(!m_cDropOutMultiplier.m_mMatrix.Flat(pos, 0))
         return false;
     }
//---   0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
   if(!m_cOpenCL)
     {
      m_cOutputs.m_mMatrix = prevLayer.GetOutputs().m_mMatrix * m_cDropOutMultiplier.m_mMatrix;
     }
   else  // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      CBufferDouble *inputs = prevLayer.GetOutputs();
      if(inputs.GetIndex() < 0)
         return false;
      if(m_cDropOutMultiplier.GetIndex() < 0 && !m_cDropOutMultiplier.BufferCreate(m_cOpenCL))
         return false;
      if(m_cOutputs.GetIndex() < 0)
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_inputs, inputs.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_mask, m_cDropOutMultiplier.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_outputs, m_cOutputs.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_MaskMult, def_mask_total, total))
         return false;
      //--- >AB0=>2:0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0};
      int NDRange[] = { (int)(total + 3) / 4};
      if(!m_cOpenCL.Execute(def_k_MaskMult, 1, off_set, NDRange))
         return false;
      //--- >;CG5=85 @57C;LB0B>2
      if(!m_cOutputs.BufferRead())
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O 3@0485=B0 G5@57 A:@KBK9 A;>9                 |
//+------------------------------------------------------------------+
bool CNeuronDropout::CalcHiddenGradient(CNeuronBase *prevLayer)
  {
//--- ;>: :>=B@>;59
   if(CheckPointer(prevLayer) == POINTER_INVALID ||
      CheckPointer(prevLayer.GetGradients()) == POINTER_INVALID ||
      CheckPointer(m_cGradients) == POINTER_INVALID ||
      CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
      return false;
//--- @>2525@:0 D;030 @568<0 @01>BK
   if(!m_bTrain)
     {
      prevLayer.GetGradients().m_mMatrix = m_cGradients.m_mMatrix;
      if(m_cOpenCL && !prevLayer.GetGradients().BufferWrite())
         return false;
     }
//---   0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
   uint total = m_cOutputs.Total();
   if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
     {
      prevLayer.GetGradients().m_mMatrix = m_cGradients.m_mMatrix * m_cDropOutMultiplier.m_mMatrix;
     }
   else  // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      CBufferDouble *prev = prevLayer.GetGradients();
      if(prev.GetIndex() < 0)
         return false;
      if(m_cDropOutMultiplier.GetIndex() < 0)
         return false;
      if(m_cGradients.GetIndex() < 0)
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_inputs, m_cGradients.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_mask, m_cDropOutMultiplier.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_outputs, prev.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_MaskMult, def_mask_total, total))
         return false;
      //--- >AB0=>2:0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0};
      int NDRange[] = { (int)(total + 3) / 4 };
      if(!m_cOpenCL.Execute(def_k_MaskMult, 1, off_set, NDRange))
         return false;
      //--- >;CG5=85 @57C;LB0B>2
      if(!prev.BufferRead())
         return false;
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 A>E@0=5=8O M;5<5=B>2 :;0AA0 2 D09;                         |
//+------------------------------------------------------------------+
bool CNeuronDropout::Save(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronBase::Save(file_handle))
      return false;
//--- !>E@0=O5< :>=AB0=BC 25@>OB=>AB8 "2K:84K20=8O" M;5<5=B>2
   if(FileWriteDouble(file_handle, m_dOutProbability) <= 0)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 2>AAB0=>2;5=8O :;0AA0 87 A>E@0=Q==KE 40==KE                |
//+------------------------------------------------------------------+
bool CNeuronDropout::Load(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronBase::Load(file_handle))
      return false;
//--- !G8BK20=85 8 2>AAB0=>2;5=85 :>=AB0=B
   m_dOutProbability = FileReadDouble(file_handle);
   m_iOutNumber = (int)(m_cOutputs.Total() * m_dOutProbability);
   m_dInitValue = 1.0 / (1.0 - m_dOutProbability);
//--- =8F80;870F8O 1CD5@0 <0A:8@>20=88O 40==KE
   if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
     {
      m_cDropOutMultiplier = new CBufferDouble();
      if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
         return false;
     }
   if(!m_cDropOutMultiplier.BufferInit(m_cOutputs.Rows(),m_cOutputs.Cols(), m_dInitValue))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+