NN_in_Trading/Experts/RL/EVD.mqh

//+------------------------------------------------------------------+
//|                                                          EVD.mqh |
//|                                                   Copyright DNG® |
//|                                https://www.mql5.com/ru/users/dng |
//+------------------------------------------------------------------+
#property copyright "Copyright DNG®"
#property link      "https://www.mql5.com/ru/users/dng"
#property version   "1.00"
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
#include "ICM.mqh"
//---
#define defEVD              0x7794   ///<Neuron Net \details Identified class #CICM
//+------------------------------------------------------------------+
//| Exploration via Disagreement                                     |
//+------------------------------------------------------------------+
class CEVD : protected CNet
  {
protected:
   uint              iMinBufferSize;
   uint              iStateEmbedingLayer;
   double            dPrevBalance;
   bool              bUseTargetNet;
   bool              bTrainMode;
   //---
   CNet              cTargetNet;
   CReplayBuffer     cReplay;
   CNet              cForwardNet;

   virtual bool      AddInputData(CArrayFloat *inputVals);

public:
                     CEVD();
                     CEVD(CArrayObj *Description, CArrayObj *Forward);
   bool              Create(CArrayObj *Description, CArrayObj *Forward);
                    ~CEVD();
   int               feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true); ///< Feed Forward method.@param[in] prevLayer Pointer to previos layer. @param[in] window Window of input data. @param[in] tem Use Time Embedding.
   bool              backProp(int batch, float discount = 0.999f);
   int               getAction(int state_size = 0);              ///< Method to get results of feed forward process.@param[out] resultVals Array of result values
   float             getRecentAverageError() { return recentAverageError; } ///< Method to check quality of study. @return Average error
   bool              Save(string file_name, bool common = true);
   bool              Save(string dqn, string forward, bool common = true);
   ///< Save method. @param[in] file_name File name to save @param[in] error Average error @param[in] undefine Undefined percent @param[in] Foecast percent @param[in] time Last study time @param[in] common Common flag
   virtual bool      Load(string file_name, bool common = true);
   bool              Load(string dqn, string forward, uint state_layer, bool common = true);
   ///< Load method. @param[in] file_name File name to save @param[out] error Average error @param[out] undefine Undefined percent @param[out] Foecast percent @param[out] time Last study time @param[in] common Common flag
   //---
   virtual int       Type(void)   const   {  return defEVD;   }///< Identificator of class.@return Type of class
   virtual bool      TrainMode(bool flag) { bTrainMode = flag; return (CNet::TrainMode(flag) && cForwardNet.TrainMode(flag)); } ///< Set Training Mode Flag
   virtual bool      GetLayerOutput(uint layer, CBufferFloat *&result) ///< Retutn Output data of layer. @param[in] layer Number of layer @param[out] return Buffer with data
     { return        CNet::GetLayerOutput(layer, result); }
   //---
   virtual void      SetStateEmbedingLayer(uint layer) { iStateEmbedingLayer = layer; }
   virtual void      SetBufferSize(uint min, uint max);
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CEVD::CEVD()   :  iMinBufferSize(100),
   bUseTargetNet(false),
   bTrainMode(true)
  {
   Create(NULL, NULL);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CEVD::~CEVD()
  {
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::Create(CArrayObj *Description, CArrayObj *Forward)
  {
   if(!CNet::Create(Description))
      return false;
   if(!cForwardNet.Create(Forward))
      return false;
   cTargetNet.Create(Description);
   bUseTargetNet = false;
   bTrainMode = true;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::backProp(int batch, float discount = 0.999000f)
  {
//---
   if(cReplay.Total() < (int)iMinBufferSize || !bTrainMode)
      return true;
//---
   CBufferFloat *state1, *state2, *targetVals = new CBufferFloat();
   vector<float> target, actions, st1, st2, result;
   matrix<float> forward;
   double reward;
   int action;
//--- цикл обучения в размере batch
   for(int i = 0; i < batch; i++)
     {
      //--- получаем случайное состояние и реплай буфера
      if(!cReplay.GetRendomState(state1, action, reward, state2))
         return false;
      //--- прямой проход обучаемой моделм ("текущее" состояие)
      if(!CNet::feedForward(state1, 1, false))
         return false;
      getResults(target);
      //--- выгружаем эмбединг состояния
      if(!GetLayerOutput(iStateEmbedingLayer, state1))
         return false;
      //--- прямой проход target net
      if(!cTargetNet.feedForward(state2, 1, false))
         return false;
      //--- корректировка вознаграждения
      if(bUseTargetNet)
        {
         cTargetNet.getResults(targetVals);
         reward += discount * targetVals.Maximum();
        }
      target[action] = (float)reward;
      if(!targetVals.AssignArray(target))
         return false;
      //--- обратный проход обучаемой модели
      CNet::backProp(targetVals);
      //--- прямой проход forward net - прогноз следующего состояния
      if(!cForwardNet.feedForward(state1, 1, false))
         return false;
      //--- выгружаем эмбединг "будущего" состояния
      if(!cTargetNet.GetLayerOutput(iStateEmbedingLayer, state2))
         return false;
      //--- подготовка целей forward net
      cForwardNet.getResults(result);
      forward.Init(1, result.Size());
      forward.Row(result, 0);
      forward.Reshape(result.Size() / state2.Total(), state2.Total());
      int ensemble = (int)(forward.Rows() / target.Size());
      //--- копируем целевое состояние в матрицу целей ансамбля
      state2.GetData(st2);
      for(int r = 0; r < ensemble; r++)
         forward.Row(st2, r * target.Size() + action);
      //--- обратный проход foward net
      targetVals.AssignArray(forward);
      if(!cForwardNet.backProp(targetVals))
         return false;
     }
//---
   delete state1;
   delete state2;
   delete targetVals;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CEVD::getAction(int state_size = 0)
  {
   CBufferFloat *temp;
//--- получаем результат обучаемой модели.
   CNet::getResults(temp);
   if(!temp)
      return -1;
//--- в режиме обучения делаем поправку на "любопытство"
   if(bTrainMode && state_size > 0)
     {
      vector<float> model;
      matrix<float> forward;
      cForwardNet.getResults(model);
      forward.Init(1, model.Size());
      forward.Row(model, 0);
      temp.GetData(model);
//      
cForwardNet.GetLayerOutput(1,temp);
      //---
      int actions = (int)model.Size();
      forward.Reshape(forward.Cols() / state_size, state_size);
      matrix<float> ensemble[];
      if(!forward.Hsplit(forward.Rows() / actions, ensemble))
         return -1;
      matrix<float> means = ensemble[0];
      int total = ArraySize(ensemble);
      for(int i = 1; i < total; i++)
         means += ensemble[i];
      means = means / total;
      for(int i = 0; i < total; i++)
         ensemble[i] -= means;
      means = MathPow(ensemble[0], 2.0);
      for(int i = 1; i < total; i++)
         means += MathPow(ensemble[i], 2.0);
      model += means.Sum(1) / total;
      temp.AssignArray(model);
     }
//---
   return temp.Argmax();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::AddInputData(CArrayFloat *inputVals)
  {
   if(!inputVals)
      return false;
   if(!inputVals.Add((float)AccountInfoDouble(ACCOUNT_BALANCE)))
      return false;
   if(!inputVals.Add((float)AccountInfoDouble(ACCOUNT_EQUITY)))
      return false;
   if(!inputVals.Add((float)AccountInfoDouble(ACCOUNT_MARGIN_FREE)))
      return false;
   if(!inputVals.Add((float)AccountInfoDouble(ACCOUNT_MARGIN_LEVEL)))
      return false;
   if(!inputVals.Add((float)AccountInfoDouble(ACCOUNT_PROFIT)))
      return false;
//---
   double buy_value = 0, sell_value = 0, buy_profit = 0, sell_profit = 0;
   int total = PositionsTotal();
   for(int i = 0; i < total; i++)
     {
      if(PositionGetSymbol(i) != _Symbol)
         continue;
      switch((int)PositionGetInteger(POSITION_TYPE))
        {
         case POSITION_TYPE_BUY:
            buy_value += PositionGetDouble(POSITION_VOLUME);
            buy_profit += PositionGetDouble(POSITION_PROFIT);
            break;
         case POSITION_TYPE_SELL:
            sell_value += PositionGetDouble(POSITION_VOLUME);
            sell_profit += PositionGetDouble(POSITION_PROFIT);
            break;
        }
     }
   if(!inputVals.Add((float)buy_value))
      return false;
   if(!inputVals.Add((float)sell_value))
      return false;
   if(!inputVals.Add((float)buy_profit))
      return false;
   if(!inputVals.Add((float)sell_profit))
      return false;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
int CEVD::feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true)
  {
   if(!AddInputData(inputVals))
      return -1;
//---
   if(!CNet::feedForward(inputVals, window, tem))
      return -1;
//---
   int action = -1;
   if(bTrainMode)
     {
      CBufferFloat *state;
      //if(!GetLayerOutput(1, state))
      //   return -1;
      if(!GetLayerOutput(iStateEmbedingLayer, state))
         return -1;
      if(!cForwardNet.feedForward(state, 1, false))
        {
         delete state;
         return -1;
        }
      double balance = AccountInfoDouble(ACCOUNT_BALANCE);
      double reward = (dPrevBalance == 0 ? 0 : balance - dPrevBalance);
      dPrevBalance = balance;
      action = getAction(state.Total());
      delete state;
      if(action < 0 || action > 3)
         return -1;
      if(!cReplay.AddState(inputVals, action, reward))
         return -1;
     }
   else
      action = getAction();
//---
   return action;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::Save(string file_name, bool common = true)
  {
   if(file_name == NULL)
      return false;
//---
   int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_WRITE);
   if(handle == INVALID_HANDLE)
      return false;
//---
   if(FileWriteInteger(handle, iMinBufferSize) <= 0 || FileWriteInteger(handle, iStateEmbedingLayer) <= 0)
     {
      FileClose(handle);
      return false;
     }
   bool result = true;
   if(!CNet::Save(handle) || !cForwardNet.Save(handle))
      result = false;
   FileFlush(handle);
   FileClose(handle);
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::Load(string file_name, bool common = true)
  {
   if(file_name == NULL)
      return false;
//---
   int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ);
   if(handle == INVALID_HANDLE)
      return false;
//---
   iMinBufferSize = (uint)FileReadInteger(handle);
   iStateEmbedingLayer = (uint)FileReadInteger(handle);
   bool result = true;
   if(!CNet::Load(handle))
      result = false;
   if(!cForwardNet.Load(handle))
      result = false;
   FileFlush(handle);
   FileClose(handle);
   float temp = 0;
   datetime dt = 0;
   if(CNet::Save(TargetNetFile, temp, temp, temp, dt, false))
      bUseTargetNet = cTargetNet.Load(TargetNetFile, temp, temp, temp, dt, false);
   TrainMode(true);
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::Save(string dqn, string forward, bool common = true)
  {
   if(dqn == NULL || forward == NULL)
      return false;
   bool result = true;
   if(!CNet::Save(dqn, getRecentAverageError(), 0, 0, 0, common) ||
      !cForwardNet.Save(forward, cForwardNet.getRecentAverageError(), 0, 0, 0, common))
      result = false;
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CEVD::Load(string dqn, string forward, uint state_layer, bool common = true)
  {
   if(dqn == NULL || forward == NULL)
      return false;
   bool result = true;
   float err, undef, forecast;
   datetime date;
   if(!CNet::Load(dqn, err, undef, forecast, date, common) ||
      !cForwardNet.Load(forward, err, undef, forecast, date, common) ||
      !cTargetNet.Load(dqn, err, undef, forecast, date, common))
      result = false;
   iStateEmbedingLayer = state_layer;
   TrainMode(true);
   bUseTargetNet = true;
//---
   return result;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CEVD::SetBufferSize(uint min, uint max)
  {
   iMinBufferSize = MathMin(min, max);
   cReplay.SetMaxSize(max);
  }
//+------------------------------------------------------------------+