Warrior_EA/references/MQL5/Experts/CT/Trajectory.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   Trajectory.mqh |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//| Rewards structure                                                |

//|   0     -  Delta Balance                                         |

//|   1     -  Delta Equity ( "-" Drawdown / "+" Profit)             |

//|   2     -  Penalty for no opened positions                       |

//+------------------------------------------------------------------+

#include "..\RL\FQF.mqh"

//---

#define        HistoryBars             100            //Depth of history

#define        ValueBars               10             //Depth of history for Value function

#define        BarDescr                9              //Elements for 1 bar description

#define        NBarInPattern           1              //Bars for 1 pattern description

#define        AccountDescr            8              //Account description

#define        NActions                6              //Number of possible Actions

#define        NRewards                3              //Number of rewards

#define        WorkerInput             512

#define        TimeDescription         4

#define        LatentCount             512

#define        LatentLayer             2

#define        EmbeddingSize           32

#define        Sparse                  0.4f

#define        Buffer_Size             6500

#define        DiscFactor              0.99f

#define        FileName                "CT"

#define        MaxReplayBuffer         500

#define        MaxSL                   1000

#define        MaxTP                   1000

#define        fCAGrad_C               0.3f

#define        iCAGrad_Iters           15

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory;

extern STrajectory          Buffer[];

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct SState

  {

   float             state[BarDescr * NBarInPattern];

   float             account[AccountDescr];

   float             action[NActions];

   float             rewards[NRewards];

   //---

                     SState(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

   //---

   void              Clear(void)

     {

      ArrayInitialize(state, 0);

      ArrayInitialize(account, 0);

      ArrayInitialize(action, 0);

      ArrayInitialize(rewards, 0);

     }

   //--- overloading

   void              operator=(const SState &obj)

     {

      ArrayCopy(state, obj.state);

      ArrayCopy(account, obj.account);

      ArrayCopy(action, obj.action);

      ArrayCopy(rewards, obj.rewards);

     }

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

SState::SState(void)

  {

   ArrayInitialize(state, 0);

   ArrayInitialize(account, 0);

   ArrayInitialize(action, 0);

   ArrayInitialize(rewards, 0);

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   int total = ArraySize(state);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, state[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(account);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, account[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(action);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, action[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(rewards);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, rewards[i]) < sizeof(float))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

   if(FileIsEnding(file_handle))

      return false;

//---

   int total = FileReadInteger(file_handle);

   if(total != ArraySize(state))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      state[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(account))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      account[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(action))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      action[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(rewards))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      rewards[i] = FileReadFloat(file_handle);

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory

  {

   SState            States[Buffer_Size];

   int               Total;

   float             DiscountFactor;

   bool              CumCounted;

   //---

                     STrajectory(void);

   //---

   bool              Add(SState &state);

   void              CumRevards(void);

   void              ClearFirstN(const int n);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

   //--- overloading

   void              operator=(const STrajectory &obj)

     {

      Total = obj.Total;

      DiscountFactor = obj.DiscountFactor;

      CumCounted = obj.CumCounted;

      for(int i = 0; i < Buffer_Size; i++)

         States[i] = obj.States[i];

     }

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

STrajectory::STrajectory(void)  :   Total(0),

   DiscountFactor(DiscFactor),

   CumCounted(false)

  {

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   if(!CumCounted)

      CumRevards();

   if(FileWriteInteger(file_handle, Total) < sizeof(int))

      return false;

   if(FileWriteFloat(file_handle, DiscountFactor) < sizeof(float))

      return false;

   for(int i = 0; i < Total; i++)

      if(!States[i].Save(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   Total = FileReadInteger(file_handle);

   if(FileIsEnding(file_handle) || Total >= ArraySize(States))

      return false;

   DiscountFactor = FileReadFloat(file_handle);

   CumCounted = true;

//---

   for(int i = 0; i < Total; i++)

      if(!States[i].Load(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::CumRevards(void)

  {

   if(CumCounted)

      return;

//---

   for(int i = Total - 2; i >= 0; i--)

      for(int r = 0; r < NRewards; r++)

         States[i].rewards[r] += States[i + 1].rewards[r] * DiscountFactor;

   CumCounted = true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Add(SState &state)

  {

   if(Total + 1 >= ArraySize(States))

      return false;

   States[Total] = state;

   Total++;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::ClearFirstN(const int n)

  {

   for(int i = 0; i < Buffer_Size - n; i++)

      States[i] = States[i + n];

   Total = MathMax(0, Buffer_Size - n);

   for(int i = Total; i < Buffer_Size; i++)

      States[i].Clear();

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

int SaveTotalBase(int min_bars)

  {

   int total = ArraySize(Buffer);

   if(total < 0)

      return 0;

   int handle = FileOpen(FileName + ".bd", FILE_WRITE | FILE_BIN | FILE_COMMON);

   if(handle < 0)

      return 0;

   int indexes[MaxReplayBuffer];

   int count = 0;

   for(int i = total - 1; i >= 0; i--)

     {

      if(Buffer[i].Total < min_bars)

         continue;

      indexes[count] = i;

      count++;

     }

   if(FileWriteInteger(handle, count) < INT_VALUE)

     {

      FileClose(handle);

      return 0;

     }

   for(int i = count - 1; i >= 0; i--)

      if(!Buffer[indexes[i]].Save(handle))

        {

         FileClose(handle);

         return (count - (i + 1));

        }

   FileFlush(handle);

   FileClose(handle);

//---

   return count;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool LoadTotalBase(void)

  {

   int handle = FileOpen(FileName + ".bd", FILE_READ | FILE_BIN | FILE_COMMON | FILE_SHARE_READ);

   if(handle < 0)

      return false;

   int total = FileReadInteger(handle);

   if(total <= 0)

     {

      FileClose(handle);

      return false;

     }

   if(ArrayResize(Buffer, total) < total)

     {

      FileClose(handle);

      return false;

     }

   for(int i = 0; i < total; i++)

      if(!Buffer[i].Load(handle))

        {

         FileClose(handle);

         return false;

        }

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool CreateDescriptions(CArrayObj *agent)

  {

//---

   CLayerDescription *descr;

//---

   if(!agent)

     {

      agent = new CArrayObj();

      if(!agent)

         return false;

     }

//--- Agent

   agent.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   int prev_count = descr.count = (BarDescr * NBarInPattern + AccountDescr + TimeDescription + NActions + NRewards);

   descr.activation = None;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBatchNormOCL;

   descr.count = prev_count;

   descr.batch = 1000;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 2

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronEmbeddingOCL;

   prev_count = descr.count = HistoryBars;

     {

      int temp[] = {BarDescr * NBarInPattern, AccountDescr, TimeDescription, NActions, NRewards};

      ArrayCopy(descr.windows, temp);

     }

   int prev_wout = descr.window_out = EmbeddingSize;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 3

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronMLMHSparseAttentionOCL;

   prev_count = descr.count = prev_count * 5;

   descr.window = EmbeddingSize;

   descr.step = 8;

   descr.window_out = 32;

   descr.layers = 4;

   descr.probability = Sparse;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 4

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConvOCL;

   descr.count = prev_count;

   descr.window = EmbeddingSize;

   descr.step = EmbeddingSize;

   descr.window_out = 16;

   descr.optimization = ADAM;

   descr.activation = LReLU;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 5

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   descr.count = LatentCount;

   descr.optimization = ADAM;

   descr.activation = LReLU;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 6

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   prev_count = descr.count = LatentCount;

   descr.activation = TANH;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 7

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   descr.count = LatentCount;

   descr.activation = LReLU;

   descr.optimization = ADAM;

   if(!agent.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 8

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   descr.count = WorkerInput;

   descr.activation = None;

   descr.optimization = ADAM;