Warrior_EA/references/MQL5/Experts/BAC/Trajectory.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   Trajectory.mqh |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

#include "..\RL\FQF.mqh"

//---

#define                    HistoryBars    20            //Depth of history

#define                    BarDescr        9            //Elements for 1 bar description

#define                    AccountDescr   12            //Account description

#define                    NActions        6            //Number of possible Actions

#define                    Buffer_Size    6500

#define                    DiscFactor     0.99f

#define                    FileName       "BAC"

#define                    LatentLayer     6

#define                    LatentCount    256

#define                    MaxSL          1000

#define                    MaxTP          1000

#define                    PoliticAdjust  1.0e-1f

#define                    MaxReplayBuffer 500

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct SState

  {

   float             state[HistoryBars * BarDescr];

   float             account[AccountDescr - 4];

   float             action[NActions];

   //---

                     SState(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

   //--- overloading

   void              operator=(const SState &obj)

     {

      ArrayCopy(state, obj.state);

      ArrayCopy(account, obj.account);

      ArrayCopy(action, obj.action);

     }

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

SState::SState(void)

  {

   ArrayInitialize(state, 0);

   ArrayInitialize(account, 0);

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   int total = ArraySize(state);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, state[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(account);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, account[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(action);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, action[i]) < sizeof(float))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

   if(FileIsEnding(file_handle))

      return false;

//---

   int total = FileReadInteger(file_handle);

   if(total != ArraySize(state))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      state[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(account))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      account[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(action))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      action[i] = FileReadFloat(file_handle);

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory

  {

   SState            States[Buffer_Size];

   float             Revards[Buffer_Size];

   int               Total;

   float             DiscountFactor;

   bool              CumCounted;

   //---

                     STrajectory(void);

   //---

   bool              Add(SState &state, float reward);

   void              CumRevards(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

STrajectory::STrajectory(void)  :   Total(0),

   DiscountFactor(DiscFactor),

   CumCounted(false)

  {

   ArrayInitialize(Revards, 0);

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   if(!CumCounted)

      CumRevards();

   if(FileWriteInteger(file_handle, Total) < sizeof(int))

      return false;

   for(int i = 0; i < Total; i++)

     {

      if(!States[i].Save(file_handle))

         return false;

      if(FileWriteFloat(file_handle, Revards[i]) < sizeof(float))

         return false;

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   Total = FileReadInteger(file_handle);

   if(Total >= ArraySize(Revards))

      return false;

   CumCounted = true;

//---

   for(int i = 0; i < Total; i++)

     {

      if(!States[i].Load(file_handle))

         return false;

      Revards[i] = FileReadFloat(file_handle);

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::CumRevards(void)

  {

   if(CumCounted)

      return;

//---

   for(int i = Total - 2; i >= 0; i--)

      Revards[i] += Revards[i + 1] * DiscountFactor;

   CumCounted = true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Add(SState &state, float reward)

  {

   if(Total + 1 >= ArraySize(Revards))

      return false;

   States[Total] = state;

   if(Total > 0)

      Revards[Total - 1] = reward;

   Total++;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SaveTotalBase(void)

  {

   int total = ArraySize(Buffer);

   if(total < 0)

      return true;

   int handle = FileOpen(FileName + ".bd", FILE_WRITE | FILE_BIN | FILE_COMMON);

   if(handle < 0)

      return false;

   int start = MathMax(total - MaxReplayBuffer, 0);

   if(FileWriteInteger(handle, total - start) < INT_VALUE)

     {

      FileClose(handle);

      return false;

     }

   for(int i = start; i < total; i++)

      if(!Buffer[i].Save(handle))

        {

         FileClose(handle);

         return false;

        }

   FileFlush(handle);

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool LoadTotalBase(void)

  {

   int handle = FileOpen(FileName + ".bd", FILE_READ | FILE_BIN | FILE_COMMON | FILE_SHARE_READ);

   if(handle < 0)

      return false;

   int total = FileReadInteger(handle);

   if(total <= 0)

     {

      FileClose(handle);

      return false;

     }

   if(ArrayResize(Buffer, total) < total)

     {

      FileClose(handle);

      return false;

     }

   for(int i = 0; i < total; i++)

      if(!Buffer[i].Load(handle))

        {

         FileClose(handle);

         return false;

        }

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool CreateDescriptions(CArrayObj *actor, CArrayObj *critic, CArrayObj *autoencoder)

  {

//---

   CLayerDescription *descr;

//---

   if(!actor)

     {

      actor = new CArrayObj();

      if(!actor)

         return false;

     }

   if(!critic)

     {

      critic = new CArrayObj();

      if(!critic)

         return false;

     }

   if(!autoencoder)

     {

      autoencoder = new CArrayObj();

      if(!autoencoder)

         return false;

     }

//--- Actor

   actor.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   int prev_count = descr.count = (HistoryBars * BarDescr);

   descr.activation = None;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBatchNormOCL;

   descr.count = prev_count;

   descr.batch = 1000;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 2

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConvOCL;

   prev_count = descr.count = prev_count - 1;

   descr.window = 2;

   descr.step = 1;

   descr.window_out = 8;

   descr.activation = LReLU;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 3

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConvOCL;

   prev_count = descr.count = prev_count;

   descr.window = 8;

   descr.step = 8;

   descr.window_out = 8;

   descr.activation = LReLU;

   descr.optimization = ADAM;