Warrior_EA/references/MQL5/Experts/DoC/Trajectory.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   Trajectory.mqh |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//| Rewards structure                                                |

//|   0     -  Delta Balance                                         |

//|   1     -  Delta Equity ( "-" Drawdown / "+" Profit)             |

//|   2     -  Penalty for no opened positions                       |

//+------------------------------------------------------------------+

#include "..\RL\FQF.mqh"

//---

#define        HistoryBars             100            //Depth of history

#define        BarDescr                9              //Elements for 1 bar description

#define        NBarInPattern           1              //Bars for 1 pattern description

#define        AccountDescr            8              //Account description

#define        NActions                6              //Number of possible Actions

#define        NRewards                3              //Number of rewards

#define        TimeDescription         4

#define        LatentCount             1024

#define        EmbeddingSize           32

#define        Sparse                  0.4f

#define        Buffer_Size             6500

#define        DiscFactor              0.99f

#define        FileName                "DoC"

#define        MaxReplayBuffer         500

#define        MaxSL                   1000

#define        MaxTP                   1000

#define        fCAGrad_C               0.3f

#define        iCAGrad_Iters           15

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory;

extern STrajectory          Buffer[];

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct SState

  {

   float             state[BarDescr*NBarInPattern];

   float             account[AccountDescr];

   float             action[NActions];

   float             rewards[NRewards];

   //---

                     SState(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

   //---

   void              Clear(void)

     {

      ArrayInitialize(state, 0);

      ArrayInitialize(account, 0);

      ArrayInitialize(action, 0);

      ArrayInitialize(rewards, 0);

     }

   //--- overloading

   void              operator=(const SState &obj)

     {

      ArrayCopy(state, obj.state);

      ArrayCopy(account, obj.account);

      ArrayCopy(action, obj.action);

      ArrayCopy(rewards, obj.rewards);

     }

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

SState::SState(void)

  {

   ArrayInitialize(state, 0);

   ArrayInitialize(account, 0);

   ArrayInitialize(action, 0);

   ArrayInitialize(rewards, 0);

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   int total = ArraySize(state);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, state[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(account);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, account[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(action);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, action[i]) < sizeof(float))

         return false;

   total = ArraySize(rewards);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, rewards[i]) < sizeof(float))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

   if(FileIsEnding(file_handle))

      return false;

//---

   int total = FileReadInteger(file_handle);

   if(total != ArraySize(state))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      state[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(account))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      account[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(action))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      action[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(rewards))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      rewards[i] = FileReadFloat(file_handle);

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory

  {

   SState            States[Buffer_Size];

   int               Total;

   float             DiscountFactor;

   bool              CumCounted;

   //---

                     STrajectory(void);

   //---

   bool              Add(SState &state);

   void              CumRevards(void);

   void              ClearFirstN(const int n);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

STrajectory::STrajectory(void)  :   Total(0),

   DiscountFactor(DiscFactor),

   CumCounted(false)

  {

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   if(!CumCounted)

      CumRevards();

   if(FileWriteInteger(file_handle, Total) < sizeof(int))

      return false;

   if(FileWriteFloat(file_handle, DiscountFactor) < sizeof(float))

      return false;

   for(int i = 0; i < Total; i++)

      if(!States[i].Save(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   Total = FileReadInteger(file_handle);

   if(FileIsEnding(file_handle) || Total >= ArraySize(States))

      return false;

   DiscountFactor = FileReadFloat(file_handle);

   CumCounted = true;

//---

   for(int i = 0; i < Total; i++)

      if(!States[i].Load(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::CumRevards(void)

  {

   if(CumCounted)

      return;

//---

   for(int i = Total - 2; i >= 0; i--)

      for(int r = 0; r < NRewards; r++)

         States[i].rewards[r] += States[i + 1].rewards[r] * DiscountFactor;

   CumCounted = true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Add(SState &state)

  {

   if(Total + 1 >= ArraySize(States))

      return false;

   States[Total] = state;

   Total++;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::ClearFirstN(const int n)

  {

   for(int i=0; i<Buffer_Size-n; i++)

      States[i]=States[i+n];

   Total=MathMax(0,Buffer_Size-n);

   for(int i=Total; i<Buffer_Size; i++)

      States[i].Clear();

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SaveTotalBase(void)

  {

   int total = ArraySize(Buffer);

   if(total < 0)

      return true;

   int handle = FileOpen(FileName + ".bd", FILE_WRITE | FILE_BIN | FILE_COMMON);

   if(handle < 0)

      return false;

   int start = MathMax(total - MaxReplayBuffer, 0);

   if(FileWriteInteger(handle, total - start) < INT_VALUE)

     {

      FileClose(handle);

      return false;

     }

   for(int i = start; i < total; i++)

      if(!Buffer[i].Save(handle))

        {

         FileClose(handle);

         return false;

        }

   FileFlush(handle);

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool LoadTotalBase(void)

  {

   int handle = FileOpen(FileName + ".bd", FILE_READ | FILE_BIN | FILE_COMMON | FILE_SHARE_READ);

   if(handle < 0)

      return false;

   int total = FileReadInteger(handle);

   if(total <= 0)

     {

      FileClose(handle);

      return false;

     }

   if(ArrayResize(Buffer, total) < total)

     {

      FileClose(handle);

      return false;

     }

   for(int i = 0; i < total; i++)

      if(!Buffer[i].Load(handle))

        {

         FileClose(handle);

         return false;

        }

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool CreateDescriptions(CArrayObj *agent, CArrayObj *rtg)

  {

//---

   CLayerDescription *descr;

//---

   if(!agent)

     {

      agent = new CArrayObj();

      if(!agent)

         return false;

     }

//--- Agent

   agent.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   int prev_count = descr.count = (NRewards + BarDescr*NBarInPattern + AccountDescr + TimeDescription + NActions);

   descr.activation = None;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBatchNormOCL;

   descr.count = prev_count;

   descr.batch = 1000;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 2

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronEmbeddingOCL;

   prev_count = descr.count = HistoryBars;

     {

      int temp[] = {BarDescr*NBarInPattern,AccountDescr,TimeDescription,NActions,NRewards};

      ArrayCopy(descr.windows,temp);

     }

   int prev_wout = descr.window_out = EmbeddingSize;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 3

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronMLMHSparseAttentionOCL;

   prev_count = descr.count = prev_count*5;

   descr.window = prev_wout;

   descr.step = 4;

   descr.window_out = 32;

   descr.layers = 8;

   descr.probability = Sparse;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 4

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConvOCL;