okWarrior_EA/references/MQL5/Experts/ADAPT/Trajectory.mqh

<EFBFBD><EFBFBD>//+------------------------------------------------------------------+

//|                                                   Trajectory.mqh |

//|                                                   Copyright DNG<EFBFBD> |

//|                                https://www.mql5.com/en/users/dng |

//+------------------------------------------------------------------+

#property copyright "Copyright DNG<00>"

#property link      "https://www.mql5.com/en/users/dng"

#property version   "1.00"

//+------------------------------------------------------------------+

//| Rewards structure                                                |

//|   0     -  Delta Balance                                         |

//|   1     -  Delta Equity ( "-" Drawdown / "+" Profit)             |

//|   2     -  Penalty for no open positions                         |

//+------------------------------------------------------------------+

#include "..\NeuroNet_DNG\NeuroNet.mqh"

//---

#define        HistoryBars             1             //Depth of history

#define        GPTBars                 100           //Analitic Depth of history

#define        PrecoderBars            10            //Forecast Depth

#define        BarDescr                9             //Elements for 1 bar description

#define        AccountDescr            12            //Account description

#define        NActions                6             //Number of possible Actions

#define        NRewards                3             //Number of rewards

#define        NForecast               5             //Number of forecast

#define        Lenc                    3             //Number ADAPT Encoder blocks

#define        EmbeddingSize           32

#define        Buffer_Size             6500

#define        DiscFactor              0.99f

#define        FileName                "ADAPT"

#define        SignalFile(agent)       StringFormat("Signals\\Signal%d.csv",agent)

#define        LatentCount             512

#define        MaxSL                   1000

#define        MaxTP                   1000

#define        MaxReplayBuffer         500

#define        StartTargetIteration    200000

#define        STE_Multiplier          1.0f/2

#define        STE_Noise_Multiplier    1.0f/10        // <EFBFBD> defines the impact of noise prediction error

#define        STD_Delta_Multiplier    1.0f/20        // noise factor <EFBFBD>

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct SState

  {

   float             state[HistoryBars * BarDescr];

   float             account[AccountDescr - 4];

   float             action[NActions];

   float             rewards[NRewards];

   //---

                     SState(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

   //--- overloading

   void              operator=(const SState &obj)

     {

      ArrayCopy(state, obj.state);

      ArrayCopy(account, obj.account);

      ArrayCopy(action, obj.action);

      ArrayCopy(rewards, obj.rewards);

     }

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

SState::SState(void)

  {

   ArrayInitialize(state, 0);

   ArrayInitialize(account, 0);

   ArrayInitialize(action, 0);

   ArrayInitialize(rewards, 0);

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   int total = ArraySize(state);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, state[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(account);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, account[i]) < sizeof(float))

         return false;

//---

   total = ArraySize(action);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, action[i]) < sizeof(float))

         return false;

   total = ArraySize(rewards);

   if(FileWriteInteger(file_handle, total) < sizeof(int))

      return false;

   for(int i = 0; i < total; i++)

      if(FileWriteFloat(file_handle, rewards[i]) < sizeof(float))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SState::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

   if(FileIsEnding(file_handle))

      return false;

//---

   int total = FileReadInteger(file_handle);

   if(total != ArraySize(state))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      state[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(account))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      account[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(action))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      action[i] = FileReadFloat(file_handle);

     }

//---

   total = FileReadInteger(file_handle);

   if(total != ArraySize(rewards))

      return false;

//---

   for(int i = 0; i < total; i++)

     {

      if(FileIsEnding(file_handle))

         return false;

      rewards[i] = FileReadFloat(file_handle);

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

struct STrajectory

  {

   SState            States[Buffer_Size];

   int               Total;

   float             DiscountFactor;

   bool              CumCounted;

   //---

                     STrajectory(void);

   //---

   bool              Add(SState &state);

   void              CumRevards(void);

   //---

   bool              Save(int file_handle);

   bool              Load(int file_handle);

  };

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

STrajectory::STrajectory(void)  :   Total(0),

   DiscountFactor(DiscFactor),

   CumCounted(false)

  {

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Save(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   if(!CumCounted)

      CumRevards();

   if(FileWriteInteger(file_handle, Total) < sizeof(int))

      return false;

   if(FileWriteFloat(file_handle, DiscountFactor) < sizeof(float))

      return false;

   for(int i = 0; i < Total; i++)

      if(!States[i].Save(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Load(int file_handle)

  {

   if(file_handle == INVALID_HANDLE)

      return false;

//---

   Total = FileReadInteger(file_handle);

   if(FileIsEnding(file_handle) || Total >= ArraySize(States))

      return false;

   DiscountFactor = FileReadFloat(file_handle);

   CumCounted = true;

//---

   for(int i = 0; i < Total; i++)

      if(!States[i].Load(file_handle))

         return false;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

void STrajectory::CumRevards(void)

  {

   if(CumCounted)

      return;

//---

   for(int i = Total - 2; i >= 0; i--)

      for(int r = 0; r < NRewards; r++)

         States[i].rewards[r] += States[i + 1].rewards[r] * DiscountFactor;

   CumCounted = true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool STrajectory::Add(SState &state)

  {

   if(Total + 1 >= ArraySize(States))

      return false;

   States[Total] = state;

   Total++;

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool SaveTotalBase(void)

  {

   int total = ArraySize(Buffer);

   if(total < 0)

      return true;

   int handle = FileOpen(FileName + ".bd", FILE_WRITE | FILE_BIN | FILE_COMMON);

   if(handle < 0)

      return false;

   int start = MathMax(total - MaxReplayBuffer, 0);

   if(FileWriteInteger(handle, total - start) < INT_VALUE)

     {

      FileClose(handle);

      return false;

     }

   for(int i = start; i < total; i++)

      if(!Buffer[i].Save(handle))

        {

         FileClose(handle);

         return false;

        }

   FileFlush(handle);

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool LoadTotalBase(void)

  {

   int handle = FileOpen(FileName + ".bd", FILE_READ | FILE_BIN | FILE_COMMON | FILE_SHARE_READ);

   if(handle < 0)

      return false;

   int total = FileReadInteger(handle);

   if(total <= 0)

     {

      FileClose(handle);

      return false;

     }

   if(ArrayResize(Buffer, total) < total)

     {

      FileClose(handle);

      return false;

     }

   for(int i = 0; i < total; i++)

      if(!Buffer[i].Load(handle))

        {

         FileClose(handle);

         return false;

        }

   FileClose(handle);

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool CreateDescriptions(CArrayObj *actor, CArrayObj *end_encoder, CArrayObj *state_encoder)

  {

//---

   CLayerDescription *descr;

//---

   if(!actor)

     {

      actor = new CArrayObj();

      if(!actor)

         return false;

     }

   if(!end_encoder)

     {

      end_encoder = new CArrayObj();

      if(!end_encoder)

         return false;

     }

   if(!state_encoder)

     {

      state_encoder = new CArrayObj();

      if(!state_encoder)

         return false;

     }

//--- Endpoints Encoder

   end_encoder.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   int prev_count = descr.count = 3 * NForecast;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!end_encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConcatenate;

   descr.count = LatentCount;

   descr.window = prev_count;

   descr.step = NForecast;

   descr.optimization = ADAM;

   descr.activation = LReLU;

   if(!end_encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- State Encoder

   state_encoder.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   prev_count = descr.count = GPTBars * EmbeddingSize;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!state_encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConcatenate;

   descr.count = LatentCount;

   descr.window = prev_count;

   descr.step = AccountDescr;

   descr.optimization = ADAM;

   descr.activation = SIGMOID;

   if(!state_encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- Actor

   actor.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   prev_count = descr.count = LatentCount;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronConcatenate;

   descr.count = LatentCount;

   descr.window = prev_count;

   descr.step = LatentCount;

   descr.optimization = ADAM;

   descr.activation = LReLU;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 2

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   descr.count = LatentCount;

   descr.activation = SIGMOID;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 3

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   descr.count = 2 * NActions;

   descr.activation = None;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 4

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronVAEOCL;

   descr.count = NActions;

   descr.optimization = ADAM;

   if(!actor.Add(descr))

     {

      delete descr;

      return false;

     }

//---

   return true;

  }

//+------------------------------------------------------------------+

//|                                                                  |

//+------------------------------------------------------------------+

bool CreateTrajNetDescriptions(CArrayObj *encoder, CArrayObj *endpoints, CArrayObj *probability)

  {

//---

   CLayerDescription *descr;

//---

   if(!encoder)

     {

      encoder = new CArrayObj();

      if(!encoder)

         return false;

     }

   if(!endpoints)

     {

      endpoints = new CArrayObj();

      if(!endpoints)

         return false;

     }

   if(!probability)

     {

      probability = new CArrayObj();

      if(!probability)

         return false;

     }

//--- Encoder

   encoder.Clear();

//--- Input layer

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBaseOCL;

   int prev_count = descr.count = (HistoryBars * BarDescr);

   descr.activation = None;

   descr.optimization = ADAM;

   if(!encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 1

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronBatchNormOCL;

   descr.count = prev_count;

   descr.batch = MathMax(1000, GPTBars);

   descr.activation = None;

   descr.optimization = ADAM;

   if(!encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 2

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronEmbeddingOCL;

     {

      int temp[] = {prev_count};

      ArrayCopy(descr.windows, temp);

     }

   prev_count = descr.count = GPTBars;

   int prev_wout = descr.window_out = EmbeddingSize;

   if(!encoder.Add(descr))

     {

      delete descr;

      return false;

     }

//--- layer 3

   if(!(descr = new CLayerDescription()))

      return false;

   descr.type = defNeuronPEOCL;

   descr.count = prev_count;

   descr.window = prev_wout;

   if(!encoder.Add(descr))

     {

      delete descr;

      return false;

     }

   for(int l = 0; l < Lenc; l++)

     {

      //--- layer 4

      if(!(descr = new CLayerDescription()))

         return false;

      descr.type = defNeuronTransposeOCL;

      descr.count = prev_count;

      descr.window = prev_wout;

      if(!encoder.Add(descr))

        {

         delete descr;

         return false;

        }

      //--- layer 5

      if(!(descr = new CLayerDescription()))

         return false;

      descr.type = defNeuronMH2AttentionOCL;

      descr.count = prev_wout;

      descr.window = prev_count;

      descr.step = 8;

      descr.window_out = 16;

      descr.optimization = ADAM;