okWarrior_EA/references/MQL5/Experts/DDPG/Study.mq5

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   StudyActor.mq5 |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//| Includes                                                         |

//+------------------------------------------------------------------+

#include "..\NeuroNet_DNG\NeuroNet.mqh"

#include <Trade\Trade.mqh>

#include <Trade\SymbolInfo.mqh>

#include <Indicators\Oscilators.mqh>

//+------------------------------------------------------------------+

//| Defines                                                          |

//+------------------------------------------------------------------+

#define                    HistoryBars  20            //Depth of history

#define                    BarDescr     12            //Elements for 1 bar description

#define                    AccountDescr 8             //Account description

#define                    FileName     "DDPG"

#define                    LatentLayer  6

//+------------------------------------------------------------------+

//| Input parameters                                                 |

//+------------------------------------------------------------------+

input ENUM_TIMEFRAMES      TimeFrame   =  PERIOD_H1;

input float                Tau         =  0.001f;

//---

input group                "---- RSI ----"

input int                  RSIPeriod   =  14;            //Period

input ENUM_APPLIED_PRICE   RSIPrice    =  PRICE_CLOSE;   //Applied price

//---

input group                "---- CCI ----"

input int                  CCIPeriod   =  14;            //Period

input ENUM_APPLIED_PRICE   CCIPrice    =  PRICE_TYPICAL; //Applied price

//---

input group                "---- ATR ----"

input int                  ATRPeriod   =  14;            //Period

//---

input group                "---- MACD ----"

input int                  FastPeriod  =  12;            //Fast

input int                  SlowPeriod  =  26;            //Slow

input int                  SignalPeriod =  9;            //Signal

input ENUM_APPLIED_PRICE   MACDPrice   =  PRICE_CLOSE;   //Applied price

input int                  Agent = 1;

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

CNet                 Actor;

CNet                 Critic;

CNet                 TargetActor;

CNet                 TargetCritic;

//---

float                dError;

datetime             dtStudied;

//---

CSymbolInfo          Symb;

CTrade               Trade;

//---

MqlRates             Rates[];

CiRSI                RSI;

CiCCI                CCI;

CiATR                ATR;

CiMACD               MACD;

//---

CBufferFloat         State;

CBufferFloat         Account;

CBufferFloat         PrevAccount;

CBufferFloat         Gradient;

CBufferFloat         *Result;

vector<float>        check;

vector<float>        ActorResult;

double               PrevBalance = 0;

double               PrevEquity = 0;

bool                 FirstBar = true;

//+------------------------------------------------------------------+

//| Expert initialization function                                   |

//+------------------------------------------------------------------+

int OnInit()

  {

//---

   if(!Symb.Name(_Symbol))

      return INIT_FAILED;

   Symb.Refresh();

//---

   if(!RSI.Create(Symb.Name(), TimeFrame, RSIPeriod, RSIPrice))

      return INIT_FAILED;

//---

   if(!CCI.Create(Symb.Name(), TimeFrame, CCIPeriod, CCIPrice))

      return INIT_FAILED;

//---

   if(!ATR.Create(Symb.Name(), TimeFrame, ATRPeriod))

      return INIT_FAILED;

//---

   if(!MACD.Create(Symb.Name(), TimeFrame, FastPeriod, SlowPeriod, SignalPeriod, MACDPrice))

      return INIT_FAILED;

   if(!RSI.BufferResize(HistoryBars) || !CCI.BufferResize(HistoryBars) ||

      !ATR.BufferResize(HistoryBars) || !MACD.BufferResize(HistoryBars))

     {

      PrintFormat("%s -> %d", __FUNCTION__, __LINE__);

      return INIT_FAILED;

     }

//---

   if(!Trade.SetTypeFillingBySymbol(Symb.Name()))

      return INIT_FAILED;

//--- load models

   float temp;

   if(!Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true) ||

      !Critic.Load(FileName + "Crt.nnw", temp, temp, temp, dtStudied, true) ||

      !TargetActor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true) ||

      !TargetCritic.Load(FileName + "Crt.nnw", temp, temp, temp, dtStudied, true))

     {

      CArrayObj *actor = new CArrayObj();

      CArrayObj *critic = new CArrayObj();

      if(!CreateDescriptions(actor, critic))

        {

         delete actor;

         delete critic;

         return INIT_FAILED;

        }

      if(!Actor.Create(actor) || !Critic.Create(critic) ||

         !TargetActor.Create(actor) || !TargetCritic.Create(critic))

        {

         delete actor;

         delete critic;

         return INIT_FAILED;

        }

      delete actor;

      delete critic;

      //---

     }

//---

   COpenCLMy *opencl = Actor.GetOpenCL();

   Critic.SetOpenCL(opencl);

   TargetActor.SetOpenCL(opencl);

   TargetCritic.SetOpenCL(opencl);

//---

   Actor.getResults(Result);

   if(Result.Total() != 6)

     {

      PrintFormat("The scope of the actor does not match the actions count (%d <> %d)", 6, Result.Total());

      return INIT_FAILED;

     }

   ActorResult = vector<float>::Zeros(6);

//---

   Actor.GetLayerOutput(0, Result);

   if(Result.Total() != (HistoryBars * BarDescr))

     {

      PrintFormat("Input size of Actor doesn't match state description (%d <> %d)", Result.Total(), (HistoryBars * BarDescr));

      return INIT_FAILED;

     }

//---

   Actor.GetLayerOutput(LatentLayer, Result);

   int latent_state = Result.Total();

   Critic.GetLayerOutput(0, Result);

   if(Result.Total() != latent_state)

     {

      PrintFormat("Input size of Critic doesn't match latent state Actor (%d <> %d)", Result.Total(), latent_state);

      return INIT_FAILED;

     }

//---

   PrevBalance = AccountInfoDouble(ACCOUNT_BALANCE);

   PrevEquity = AccountInfoDouble(ACCOUNT_EQUITY);

   FirstBar = true;

   Gradient.BufferInit(AccountDescr, 0);

   Gradient.BufferCreate(opencl);

//---

   return(INIT_SUCCEEDED);

  }

//+------------------------------------------------------------------+

//| Expert deinitialization function                                 |

//+------------------------------------------------------------------+

void OnDeinit(const int reason)

  {

//---

   TargetActor.WeightsUpdate(GetPointer(Actor), Tau);

   TargetCritic.WeightsUpdate(GetPointer(Critic), Tau);

   TargetActor.Save(FileName + "Act.nnw", Actor.getRecentAverageError(), 0, 0, TimeCurrent(), true);

   TargetCritic.Save(FileName + "Crt.nnw", Critic.getRecentAverageError(), 0, 0, TimeCurrent(), true);

   delete Result;

  }

//+------------------------------------------------------------------+

//| Expert tick function                                             |

//+------------------------------------------------------------------+

void OnTick()

  {

//---

   if(!IsNewBar())

      return;

//---

   int bars = CopyRates(Symb.Name(), TimeFrame, iTime(Symb.Name(), TimeFrame, 1), HistoryBars, Rates);

   if(!ArraySetAsSeries(Rates, true))

      return;

//---

   RSI.Refresh();

   CCI.Refresh();

   ATR.Refresh();

   MACD.Refresh();

   Symb.Refresh();

   Symb.RefreshRates();

//---

   MqlDateTime sTime;

   float atr = 0;

   State.Clear();

   for(int b = 0; b < (int)HistoryBars; b++)

     {

      float open = (float)Rates[b].open;

      TimeToStruct(Rates[b].time, sTime);

      float rsi = (float)RSI.Main(b);

      float cci = (float)CCI.Main(b);

      atr = (float)ATR.Main(b);

      float macd = (float)MACD.Main(b);

      float sign = (float)MACD.Signal(b);

      if(rsi == EMPTY_VALUE || cci == EMPTY_VALUE || atr == EMPTY_VALUE || macd == EMPTY_VALUE || sign == EMPTY_VALUE)

         continue;

      //---

      State.Add((float)Rates[b].close - open);

      State.Add((float)Rates[b].high - open);

      State.Add((float)Rates[b].low - open);

      State.Add((float)Rates[b].tick_volume / 1000.0f);

      State.Add((float)sTime.hour);

      State.Add((float)sTime.day_of_week);

      State.Add((float)sTime.mon);

      State.Add(rsi);

      State.Add(cci);

      State.Add(atr);

      State.Add(macd);

      State.Add(sign);

     }

//---

   vector<float> account = vector<float>::Zeros(AccountDescr - 1);

   account[0] = (float)AccountInfoDouble(ACCOUNT_BALANCE);

   account[1] = (float)AccountInfoDouble(ACCOUNT_EQUITY);

//---

   double buy_value = 0, sell_value = 0, buy_profit = 0, sell_profit = 0;

   double position_discount = 0;

   double multiplyer = 1.0 / (60.0 * 60.0 * 10.0);

   int total = PositionsTotal();

   datetime current = TimeCurrent();

   for(int i = 0; i < total; i++)

     {

      if(PositionGetSymbol(i) != Symb.Name())

         continue;

      double profit = PositionGetDouble(POSITION_PROFIT);

      switch((int)PositionGetInteger(POSITION_TYPE))

        {

         case POSITION_TYPE_BUY:

            buy_value += PositionGetDouble(POSITION_VOLUME);

            buy_profit += profit;

            break;

         case POSITION_TYPE_SELL:

            sell_value += PositionGetDouble(POSITION_VOLUME);

            sell_profit += profit;

            break;

        }

      position_discount += profit - (current - PositionGetInteger(POSITION_TIME)) * multiplyer * MathAbs(profit);

     }

   account[2] = (float)buy_value;

   account[3] = (float)sell_value;

   account[4] = (float)buy_profit;

   account[5] = (float)sell_profit;

   account[6] = (float)position_discount;

//---

   Account.Clear();

   Account.Add((float)((account[0] - PrevBalance) / PrevBalance));

   Account.Add((float)(account[1] / PrevBalance));

   Account.Add((float)((account[1] - PrevEquity) / PrevEquity));

   Account.Add(account[2]);

   Account.Add(account[3]);

   Account.Add((float)(account[4] / PrevBalance));

   Account.Add((float)(account[5] / PrevBalance));

   Account.Add((float)(account[6] / PrevBalance));

//---

   if(Account.GetIndex() >= 0)

      if(!Account.BufferWrite())

         return;

//---

   if(!FirstBar)

     {

      if(!TargetActor.feedForward(GetPointer(State), 1, false, GetPointer(Account)))

         return;

      if(!TargetCritic.feedForward(GetPointer(TargetActor), LatentLayer, GetPointer(TargetActor)))

         return;

      TargetCritic.getResults(Result);

      float reward = (float)(account[0] - PrevBalance + Result[0]);

      if(account[0] == PrevBalance)

         if((buy_value + sell_value) == 0)

            reward -= 1;

      Result.Update(0, reward);

      if(!Critic.backProp(Result, GetPointer(Actor)) || !Actor.backPropGradient(GetPointer(PrevAccount), GetPointer(Gradient)))