okWarrior_EA/references/MQL5/Experts/DWSL/Study.mq5

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                        Study.mq5 |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

#define Study

#include "Trajectory.mqh"

//+------------------------------------------------------------------+

//| Input parameters                                                 |

//+------------------------------------------------------------------+

input int                  Iterations     = 10000;

input float                Tau            = 0.01f;

input float                MaxErrorActorStudy   = 0.15f;

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

STrajectory          Buffer[];

CNet                 Actor;

CNet                 Critic1;

CNet                 Critic2;

CNet                 TargetCritic1;

CNet                 TargetCritic2;

CNet                 Convolution;

//---

float                dError;

datetime             dtStudied;

//---

CBufferFloat         State;

CBufferFloat         Account;

CBufferFloat         Actions;

CBufferFloat         Gradient;

CBufferFloat         *Result;

vector<float>        check;

int                  StartTargetIter;

//---

COpenCLMy           *OpenCL;

//+------------------------------------------------------------------+

//| Expert initialization function                                   |

//+------------------------------------------------------------------+

int OnInit()

  {

//---

   ResetLastError();

   if(!LoadTotalBase())

     {

      PrintFormat("Error of load study data: %d", GetLastError());

      return INIT_FAILED;

     }

//--- load models

   float temp;

   if(!Actor.Load(FileName + "Act.nnw", temp, temp, temp, dtStudied, true) ||

      !Critic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) ||

      !Critic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true) ||

      !Convolution.Load(FileName + "CNN.nnw", temp, temp, temp, dtStudied, true) ||

      !TargetCritic1.Load(FileName + "Crt1.nnw", temp, temp, temp, dtStudied, true) ||

      !TargetCritic2.Load(FileName + "Crt2.nnw", temp, temp, temp, dtStudied, true))

     {

      Print("Init new models");

      CArrayObj *actor = new CArrayObj();

      CArrayObj *critic = new CArrayObj();

      CArrayObj *convolution = new CArrayObj();

      if(!CreateDescriptions(actor, critic, convolution))

        {

         delete actor;

         delete critic;

         delete convolution;

         return INIT_FAILED;

        }

      if(!Actor.Create(actor) || !Critic1.Create(critic) || !Critic2.Create(critic) ||

         !Convolution.Create(convolution))

        {

         delete actor;

         delete critic;

         delete convolution;

         return INIT_FAILED;

        }

      if(!TargetCritic1.Create(critic) || !TargetCritic2.Create(critic))

        {

         delete actor;

         delete critic;

         delete convolution;

         return INIT_FAILED;

        }

      delete actor;

      delete critic;

      delete convolution;

      //---

      TargetCritic1.WeightsUpdate(GetPointer(Critic1), 1.0f);

      TargetCritic2.WeightsUpdate(GetPointer(Critic2), 1.0f);

      StartTargetIter = StartTargetIteration;

     }

   else

      StartTargetIter = 0;

//---

   OpenCL = Actor.GetOpenCL();

   Critic1.SetOpenCL(OpenCL);

   Critic2.SetOpenCL(OpenCL);

   TargetCritic1.SetOpenCL(OpenCL);

   TargetCritic2.SetOpenCL(OpenCL);

   Convolution.SetOpenCL(OpenCL);

//---

   Actor.getResults(Result);

   if(Result.Total() != NActions)

     {

      PrintFormat("The scope of the actor does not match the actions count (%d <> %d)", NActions, Result.Total());

      return INIT_FAILED;

     }

//---

   Actor.GetLayerOutput(0, Result);

   if(Result.Total() != (HistoryBars * BarDescr))

     {

      PrintFormat("Input size of Actor doesn't match state description (%d <> %d)", Result.Total(), (HistoryBars * BarDescr));

      return INIT_FAILED;

     }

//---

   Actor.GetLayerOutput(LatentLayer, Result);

   int latent_state = Result.Total();

   Critic1.GetLayerOutput(0, Result);

   if(Result.Total() != latent_state)

     {

      PrintFormat("Input size of Critic doesn't match latent state Actor (%d <> %d)", Result.Total(), latent_state);

      return INIT_FAILED;

     }

//---

   Gradient.BufferInit(AccountDescr, 0);

//---

   if(!EventChartCustom(ChartID(), 1, 0, 0, "Init"))

     {

      PrintFormat("Error of create study event: %d", GetLastError());

      return INIT_FAILED;

     }

//---

   return(INIT_SUCCEEDED);

  }

//+------------------------------------------------------------------+

//| Expert deinitialization function                                 |

//+------------------------------------------------------------------+

void OnDeinit(const int reason)

  {

//---

   TargetCritic1.WeightsUpdate(GetPointer(Critic1), Tau);

   TargetCritic2.WeightsUpdate(GetPointer(Critic2), Tau);

   Actor.Save(FileName + "Act.nnw", 0, 0, 0, TimeCurrent(), true);

   TargetCritic1.Save(FileName + "Crt1.nnw", Critic1.getRecentAverageError(), 0, 0, TimeCurrent(), true);

   TargetCritic2.Save(FileName + "Crt2.nnw", Critic2.getRecentAverageError(), 0, 0, TimeCurrent(), true);

   Convolution.Save(FileName + "CNN.nnw", 0, 0, 0, TimeCurrent(), true);

   delete Result;

  }

//+------------------------------------------------------------------+

//| ChartEvent function                                              |

//+------------------------------------------------------------------+

void OnChartEvent(const int id,

                  const long &lparam,

                  const double &dparam,

                  const string &sparam)

  {

//---

   if(id == 1001)

      Train();

  }

//+------------------------------------------------------------------+

//| Train function                                                   |

//+------------------------------------------------------------------+

void Train(void)

  {

   int total_tr = ArraySize(Buffer);

   uint ticks = GetTickCount();

//---

   int total_states = Buffer[0].Total;

   for(int i = 1; i < total_tr; i++)

      total_states += Buffer[i].Total;

   vector<float> temp, next;

   Convolution.getResults(temp);

   matrix<float> state_embedding = matrix<float>::Zeros(total_states, temp.Size());

   matrix<float> rewards = matrix<float>::Zeros(total_states, NRewards);

   matrix<float> actions = matrix<float>::Zeros(total_states, NActions);

   int state = 0;

   for(int tr = 0; tr < total_tr; tr++)

     {

      for(int st = 0; st < Buffer[tr].Total; st++)

        {

         State.AssignArray(Buffer[tr].States[st].state);

         float PrevBalance = Buffer[tr].States[MathMax(st - 1, 0)].account[0];

         float PrevEquity = Buffer[tr].States[MathMax(st - 1, 0)].account[1];

         State.Add((Buffer[tr].States[st].account[0] - PrevBalance) / PrevBalance);

         State.Add(Buffer[tr].States[st].account[1] / PrevBalance);

         State.Add((Buffer[tr].States[st].account[1] - PrevEquity) / PrevEquity);

         State.Add(Buffer[tr].States[st].account[2]);

         State.Add(Buffer[tr].States[st].account[3]);

         State.Add(Buffer[tr].States[st].account[4] / PrevBalance);

         State.Add(Buffer[tr].States[st].account[5] / PrevBalance);

         State.Add(Buffer[tr].States[st].account[6] / PrevBalance);

         double x = (double)Buffer[tr].States[st].account[7] / (double)(D'2024.01.01' - D'2023.01.01');

         State.Add((float)MathSin(x != 0 ? 2.0 * M_PI * x : 0));

         x = (double)Buffer[tr].States[st].account[7] / (double)PeriodSeconds(PERIOD_MN1);

         State.Add((float)MathCos(x != 0 ? 2.0 * M_PI * x : 0));

         x = (double)Buffer[tr].States[st].account[7] / (double)PeriodSeconds(PERIOD_W1);

         State.Add((float)MathSin(x != 0 ? 2.0 * M_PI * x : 0));

         x = (double)Buffer[tr].States[st].account[7] / (double)PeriodSeconds(PERIOD_D1);

         State.Add((float)MathSin(x != 0 ? 2.0 * M_PI * x : 0));

         if(!Convolution.feedForward(GetPointer(State), 1, false, NULL))

           {

            PrintFormat("%s -> %d", __FUNCTION__, __LINE__);

            ExpertRemove();

            return;

           }

         Convolution.getResults(temp);

         if(!state_embedding.Row(temp, state))

            continue;

            if(!temp.Assign(Buffer[tr].States[st].rewards) ||

                  !next.Assign(Buffer[tr].States[st + 1].rewards) ||

                  !rewards.Row(temp - next * DiscFactor, state))

               continue;

               if(!temp.Assign(Buffer[tr].States[st].action) ||

                     !actions.Row(temp, state))

                  continue;

                  state++;

                  if(GetTickCount() - ticks > 500)

                    {

                     string str = StringFormat("%-15s %6.2f%%", "Embedding ", state * 100.0 / (double)(total_states));

                     Comment(str);

                     ticks = GetTickCount();

                    }

        }

     }

   if(state != total_states)

     {

      rewards.Resize(state, NRewards);

      actions.Resize(state, NActions);

      state_embedding.Reshape(state, state_embedding.Cols());

      total_states = state;

     }

//---

   vector<float> rewards1, rewards2, target_reward;

   STarget target;

//---

   vector<float> probability = GetProbTrajectories(Buffer, 0.9);

   int bar = (HistoryBars - 1) * BarDescr;

   for(int iter = 0; (iter < Iterations && !IsStopped()); iter ++)

     {

      int tr = SampleTrajectory(probability);

      int i = (int)((MathRand() * MathRand() / MathPow(32767, 2)) * (Buffer[tr].Total - 2));

      if(i < 0)

        {

         iter--;

         continue;

        }

      target_reward = vector<float>::Zeros(NRewards);

      //--- Target

      if(iter >= StartTargetIter)

        {

         State.AssignArray(Buffer[tr].States[i + 1].state);

         float PrevBalance = Buffer[tr].States[i].account[0];

         float PrevEquity = Buffer[tr].States[i].account[1];

         Account.Clear();

         Account.Add((Buffer[tr].States[i + 1].account[0] - PrevBalance) / PrevBalance);

         Account.Add(Buffer[tr].States[i + 1].account[1] / PrevBalance);

         Account.Add((Buffer[tr].States[i + 1].account[1] - PrevEquity) / PrevEquity);

         Account.Add(Buffer[tr].States[i + 1].account[2]);

         Account.Add(Buffer[tr].States[i + 1].account[3]);

         Account.Add(Buffer[tr].States[i + 1].account[4] / PrevBalance);

         Account.Add(Buffer[tr].States[i + 1].account[5] / PrevBalance);

         Account.Add(Buffer[tr].States[i + 1].account[6] / PrevBalance);

         double x = (double)Buffer[tr].States[i + 1].account[7] / (double)(D'2024.01.01' - D'2023.01.01');

         Account.Add((float)MathSin(x != 0 ? 2.0 * M_PI * x : 0));