NeuroNetworksBook/Include/about_ai/activation/activations.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+
//|                                                  activations.mqh |
//|                                  Copyright 2021, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2021, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
//---
const double theta = 0;
const double a = 1.0;
const double b = 0.0;
//+------------------------------------------------------------------+
/// >@>3>20O DC=:F8O 0:B820F88
/// >=AB0=B0 'theta' >?@545;O5B C@>25=L 0:B820F88 =59@>=0.
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActStep(double x)
  {
   return (x >= theta ? 1 : 0);
  }
//+------------------------------------------------------------------+
/// 8=59=0O DC=:F8O 0:B820F88
/// >=AB0=B0 'a' >?@545;O5B C@>25=L =0:;>=0 ;8=88,
/// >=AB0=B0 'b' >?@545;O5B A<5I5=8 ;8=88 >B =0G0;0 :>>@48=0B
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActLinear(double x)
  {
   return (a * x + b);
  }
//+------------------------------------------------------------------+
/// @>872>4=0O ;8=59=>9 DC=:F88 0:B820F8
/// >=AB0=B0 'a' >?@545;O5B C@>25=L =0:;>=0 ;8=88,
/// 0@0<5B@  'y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
///  40==>< A;CG05 ?0@0<5B@ 4>102;5= 4;O 548=AB20 D>@<K DC=:F88 A 0=0;>38G=K<8 4;O 4@C38E DC=:F89 0:B820F88.
//+------------------------------------------------------------------+
double ActLinearDerivative(double y)
  {
   return a;
  }
//+------------------------------------------------------------------+
/// >38AB8G5A:0O DC=:F88 0:B820F88 (!83<>840)
/// >=AB0=B0 'a' >?@545;O5B 480?07>= 7=0G5=89 DC=:F88 >B '0' 4> 'a'
/// >=AB0=B0 'b' >?@545;O5B A<5I5=8 ;8=88 >B =0G0;0 :>>@48=0B
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActSigmoid(double x)
  {
   return (a / (1 + exp(-x)) - b);
  }
//+------------------------------------------------------------------+
/// @>872>4=0O ;>38AB8G5A:>9 DC=:F88
/// >=AB0=B0 'a' >?@545;O5B 480?07>= 7=0G5=89 DC=:F88 >B '0' 4> 'a'
/// >=AB0=B0 'b' >?@545;O5B A<5I5=8 ;8=88 >B =0G0;0 :>>@48=0B
/// 0@0<5B@  'y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
//+------------------------------------------------------------------+
double ActSigmoidDerivative(double y)
  {
   y = MathMax(MathMin(y + b, a), 0.0);
   return (y * (1 - y / a));
  }
//+------------------------------------------------------------------+
/// 8?5@1>;8G5A:89 B0=35=A
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActTanh(double x)
  {
   return tanh(x);
  }
//+------------------------------------------------------------------+
/// @>872>4=0O =8?5@1>;8G5A:>3> B0=35=A0
/// 0@0<5B@  'y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
//+------------------------------------------------------------------+
double ActTanhDerivative(double y)
  {
   y = MathMax(MathMin(y, 1.0), -1.0);
   return (1 - pow(y, 2));
  }
//+------------------------------------------------------------------+
/// $C=:F8O 0:B820F88 PReLU
/// >=AB0=B0 'a' >?@545;O5B :>MDD8F85=B "CB5G:8" (?@>1CA:0 >B@8F0B5;L=KE 7=0G5=89)
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActPReLU(double x)
  {
   return (x >= 0 ? x : a * x);
  }
//+------------------------------------------------------------------+
/// @>872>4=0O PReLU
/// >=AB0=B0 'a' >?@545;O5B :>MDD8F85=B "CB5G:8" (?@>1CA:0 >B@8F0B5;L=KE 7=0G5=89)
/// 0@0<5B@  'y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
///  40==>< A;CG05 ?0@0<5B@ 4>102;5= 4;O 548=AB20 D>@<K DC=:F88 A 0=0;>38G=K<8 4;O 4@C38E DC=:F89 0:B820F88.
//+------------------------------------------------------------------+
double ActPReLUDerivative(double y)
  {
   return (y >= 0 ? 1 : a);
  }
//+------------------------------------------------------------------+
/// $C=:F8O 0:B820F88 Swish
/// >=AB0=B0 'b' >?@545;O5B =5;8=59=>ABL DC=:F88.
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
double ActSwish(double x)
  {
   return (x / (1 + exp(-b * x)));
  }
//+------------------------------------------------------------------+
/// @>872>4=0O Swish
/// >=AB0=B0 'b' >?@545;O5B =5;8=59=>ABL DC=:F88.
/// 0@0<5B@ 'x' ?>;CG05B 2725I5==CN AC<<C 8AE>4=KE 40==KE.
/// 0@0<5B@  'y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
//+------------------------------------------------------------------+
double ActSwishDerivative(double x, double y)
  {
   if(x == 0)
      return 0.5;
   double by = b * y;
   return (by + (y * (1 - by)) / x);
  }
//+------------------------------------------------------------------+
/// $C=:F8O 0:B820F88 SoftMax
/// 0@0<5B@ 'X' ?>;CG05B <0AA82 2725I5==KE AC<< 8AE>4=KE 40==KE.
//+------------------------------------------------------------------+
bool SoftMax(double& X[], double& Y[])
  {
   uint total = X.Size();
   if(total == 0)
      return false;
   if(ArrayResize(Y, total) <= 0)
      return false;
//---  0AAGQB M:A?>=5=BK 4;O :064>3> M;5<5=B0 <0AA820
   double sum = 0;
   for(uint i = 0; i < total; i++)
      sum += Y[i] = exp(X[i]);
//--- >@<0;870F8O 40==KE 2 <0AA825
   for(uint i = 0; i < total; i++)
      Y[i] /= sum;
//---
   return true;
  }
//+------------------------------------------------------------------+
/// @>872>4=0O SoftMax
/// 0@0<5B@  'Y' ?>A;54=55 A>AB>O=85 DC=:F88 0:B280F88 (@57C;LB0B ?@O<>3> ?@>E>40),
/// 0@0<5B@  'G' 3@0485=B >H81:8, ?>;CG5==K9 >B ?>A;54CNI53> A;>O
/// $C=:F8O 2>72@0I05B <0AA82 A:>@@5:B8@>20==KE 3@0485=B>2 >H81:8 'D'
//+------------------------------------------------------------------+
bool ActSoftMaxDerivative(double& Y[], double& G[], double& D[])
  {
   uint total = Y.Size();
   if(total == 0 || ArrayResize(D, total) <= 0)
      return false;
   for(uint i = 0; i < total; i++)
     {
      double grad = 0;
      for(uint j = 0; j < total; j++)
         grad += Y[j] * ((int)(i == j) - Y[i]) * G[j];
      D[i] = grad;
     }
   return true;
  }
//+------------------------------------------------------------------+