Original_NNB/MQL5/Include/NeuroNetworksBook/realization/neurondropout.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+
//|                                                NeuronDropout.mqh |
//|                                  Copyright 2021, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
//+------------------------------------------------------------------+
//| >4:;NG05< 181;8>B5:8                                            |
//+------------------------------------------------------------------+
#include "neuronbase.mqh"
//+------------------------------------------------------------------+
//| Class CNeuronDropout                                             |
//| 07=0G5=85: ;0AA @50;870F88 <5B>40 Dropout                      |
//+------------------------------------------------------------------+
class CNeuronDropout    :  public CNeuronBase
  {
protected:
   double            m_dOutProbability;
   int               m_iOutNumber;
   double            m_dInitValue;

   CBufferDouble     *m_cDropOutMultiplier;

public:
                     CNeuronDropout(void);
                    ~CNeuronDropout(void);
   //---
   virtual bool      Init(CLayerDescription *description);
   virtual bool      FeedForward(CNeuronBase *prevLayer);
   virtual bool      CalcHiddenGradient(CNeuronBase *prevLayer);
   virtual bool      CalcDeltaWeights(CNeuronBase *prevLayer) { return true; }
   virtual bool      UpdateWeights(int batch_size, double learningRate,
                                   double &Beta[], double &Lambda[])
     {
      return true;
     }

   //--- 5B>4K @01>BK A D09;0<8
   virtual bool      Save(const int file_handle);
   virtual bool      Load(const int file_handle);
   //--- 5B>4 845=B8D8:0F88 >1J5:B0
   virtual int       Type(void)           const { return(defNeuronDropout); }
  };
//+------------------------------------------------------------------+
//| >=AB@C:B>@ :;0AA0                                               |
//+------------------------------------------------------------------+
CNeuronDropout::CNeuronDropout(void)   :  m_dInitValue(1.0),
                                          m_dOutProbability(0),
                                          m_iOutNumber(0)
  {
   m_cDropOutMultiplier = new CBufferDouble();
   m_bTrain = false;
  }
//+------------------------------------------------------------------+
//| 5AB@C:B>@ :;0AA0                                                |
//+------------------------------------------------------------------+
CNeuronDropout::~CNeuronDropout(void)
  {
   if(CheckPointer(m_cDropOutMultiplier) != POINTER_INVALID)
      delete m_cDropOutMultiplier;
  }
//+------------------------------------------------------------------+
//| 5B>4 8=8F80;870F88 :;0AA0                                       |
//+------------------------------------------------------------------+
bool CNeuronDropout::Init(CLayerDescription *description)
  {
//--- ;>: :>=B@>;59
   if(!CheckPointer(description) == POINTER_INVALID ||
      description.count != description.window)
      return false;
   description.window = 0;
   if(!CNeuronBase::Init(description))
      return false;
//---  0AGQB :>MDD8F85=B>2
   m_dOutProbability = MathMin(description.probability, 0.9);
   if(m_dOutProbability < 0)
      return false;
   m_iOutNumber = (int)(m_cOutputs.Total() * m_dOutProbability);
   m_dInitValue = 1.0 / (1.0 - m_dOutProbability);
   if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
     {
      m_cDropOutMultiplier = new CBufferDouble();
      if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
         return false;
     }
//--- =8F88@C5< 1CD5@ <0A:8@>20=8O
   if(!m_cDropOutMultiplier.BufferInit(m_cOutputs.Total(), m_dInitValue))
      return false;
   m_bTrain = true;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 ?@O<>3> ?@>E>40                                            |
//+------------------------------------------------------------------+
bool CNeuronDropout::FeedForward(CNeuronBase *prevLayer)
  {
//--- ;>: :>=B@>;59
   if(CheckPointer(prevLayer) == POINTER_INVALID ||
      CheckPointer(prevLayer.GetOutputs()) == POINTER_INVALID ||
      CheckPointer(m_cOutputs) == POINTER_INVALID ||
      CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
      return false;
//--- @>2525@:0 D;030 @568<0 @01>BK
   if(!m_bTrain)
      return m_cOutputs.AssignArray(prevLayer.GetOutputs());
//--- 5=5@8@C5< B5=7>@ <0A:8@>20=8O 40==KE
   int total = m_cOutputs.Total();
   if(!m_cDropOutMultiplier.BufferInit(total, m_dInitValue))
      return false;
   for(int i = 0; i < m_iOutNumber; i++)
     {
      int pos = (int)(MathRand() * MathRand() / MathPow(32767.0, 2) * total);
      if(m_cDropOutMultiplier[pos] == 0)
        {
         i--;
         continue;
        }
      if(!m_cDropOutMultiplier.Update(pos, 0))
         return false;
     }
//---   0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
   if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
     {
      double out[];
      double prev[];
      if(ArrayResize(out, total) < total)
         return false;
      if(!prevLayer.GetOutputs().GetData(prev, false) ||
         ArraySize(prev) != total)
         return false;
      for(int i = 0; i < total; i++)
         out[i] = prev[i] * m_cDropOutMultiplier[i];
      if(!m_cOutputs.AssignArray(out))
         return false;
     }
   else  // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      CBufferDouble *inputs = prevLayer.GetOutputs();
      if(inputs.GetIndex() < 0 && !inputs.BufferCreate(m_cOpenCL))
         return false;
      if(m_cDropOutMultiplier.GetIndex() < 0 && !m_cDropOutMultiplier.BufferCreate(m_cOpenCL))
         return false;
      if(m_cOutputs.GetIndex() < 0 && !m_cOutputs.BufferCreate(m_cOpenCL))
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_inputs, inputs.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_mask, m_cDropOutMultiplier.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_outputs, m_cOutputs.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_MaskMult, def_mask_total, total))
         return false;
      //--- >AB0=>2:0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0};
      int s = total;
      int d = s % 4;
      s = (s - d) / 4 + (d > 0 ? 1 : 0);
      int NDRange[] = {s};
      if(!m_cOpenCL.Execute(def_k_MaskMult, 1, off_set, NDRange))
         return false;
      //--- >;CG5=85 @57C;LB0B>2
      if(!m_cOutputs.BufferRead())
         return false;
      //--- G8AB:0 ?0<OB8 :5@=5;0
      inputs.BufferFree();
      m_cDropOutMultiplier.BufferFree();
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 @0A?@545;5=8O 3@0485=B0 G5@57 A:@KBK9 A;>9                 |
//+------------------------------------------------------------------+
bool CNeuronDropout::CalcHiddenGradient(CNeuronBase *prevLayer)
  {
//--- ;>: :>=B@>;59
   if(CheckPointer(prevLayer) == POINTER_INVALID ||
      CheckPointer(prevLayer.GetGradients()) == POINTER_INVALID ||
      CheckPointer(m_cGradients) == POINTER_INVALID ||
      CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
      return false;
//--- @>2525@:0 D;030 @568<0 @01>BK
   if(!m_bTrain)
      return prevLayer.GetGradients().AssignArray(m_cGradients);
//---   0725B2;5=85 0;3>@8B<0 2 7028A8<>AB8 >B CAB@>9AB0 2K?>;=5=88O >?5@0F89
   int total = m_cOutputs.Total();
   if(CheckPointer(m_cOpenCL) == POINTER_INVALID)
     {
      double gradient[];
      double prev[];
      if(ArrayResize(prev, total) < total)
         return false;
      if(!m_cGradients.GetData(gradient, false) ||
         ArraySize(gradient) != total)
         return false;
      for(int i = 0; i < total; i++)
         prev[i] = gradient[i] * m_cDropOutMultiplier[i];
      if(!prevLayer.GetGradients().AssignArray(prev))
         return false;
     }
   else  // ;>: OpenCL
     {
      //--- !>740=85 1CD5@>2 40==KE
      CBufferDouble *prev = prevLayer.GetGradients();
      if(prev.GetIndex() < 0 && !prev.BufferCreate(m_cOpenCL))
         return false;
      if(m_cDropOutMultiplier.GetIndex() < 0 && !m_cDropOutMultiplier.BufferCreate(m_cOpenCL))
         return false;
      if(m_cGradients.GetIndex() < 0 && !m_cGradients.BufferCreate(m_cOpenCL))
         return false;
      //--- 5@540G0 ?0@0<5B@>2 :5@=5;C
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_inputs, m_cGradients.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_mask, m_cDropOutMultiplier.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgumentBuffer(def_k_MaskMult, def_mask_outputs, prev.GetIndex()))
         return false;
      if(!m_cOpenCL.SetArgument(def_k_MaskMult, def_mask_total, total))
         return false;
      //--- >AB0=>2:0 2 >G5@54L 2K?>;=5=8O
      int off_set[] = {0};
      int s = total;
      int d = s % 4;
      s = (s - d) / 4 + (d > 0 ? 1 : 0);
      int NDRange[] = {s};
      if(!m_cOpenCL.Execute(def_k_MaskMult, 1, off_set, NDRange))
         return false;
      //--- >;CG5=85 @57C;LB0B>2
      if(!prev.BufferRead())
         return false;
      //--- G8AB:0 ?0<OB8 :5@=5;0
      m_cGradients.BufferFree();
      m_cDropOutMultiplier.BufferFree();
     }
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 A>E@0=5=8O M;5<5=B>2 :;0AA0 2 D09;                         |
//+------------------------------------------------------------------+
bool CNeuronDropout::Save(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronBase::Save(file_handle))
      return false;
//--- !>E@0=O5< :>=AB0=BC 25@>OB=>AB8 "2K:84K20=8O" M;5<5=B>2
   if(FileWriteDouble(file_handle, m_dOutProbability) <= 0)
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//| 5B>4 2>AAB0=>2;5=8O :;0AA0 87 A>E@0=Q==KE 40==KE                |
//+------------------------------------------------------------------+
bool CNeuronDropout::Load(const int file_handle)
  {
//--- K7>2 <5B>40 @>48B5;LA:>3> :;0AA0
   if(!CNeuronBase::Load(file_handle))
      return false;
//--- !G8BK20=85 8 2>AAB0=>2;5=85 :>=AB0=B
   m_dOutProbability = FileReadDouble(file_handle);
   m_iOutNumber = (int)(m_cOutputs.Total() * m_dOutProbability);
   m_dInitValue = 1.0 / (1.0 - m_dOutProbability);
//--- =8F80;870F8O 1CD5@0 <0A:8@>20=88O 40==KE
   if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
     {
      m_cDropOutMultiplier = new CBufferDouble();
      if(CheckPointer(m_cDropOutMultiplier) == POINTER_INVALID)
         return false;
     }
   if(!m_cDropOutMultiplier.BufferInit(m_cOutputs.Total(), m_dInitValue))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+