NN_in_Trading/Experts/Unsupervised/AE/ae_net.mq5

//+------------------------------------------------------------------+
//|                                                      pca_net.mq5 |
//|                                              Copyright 2022, DNG |
//|                                https://www.mql5.com/ru/users/dng |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, DNG"
#property link      "https://www.mql5.com/ru/users/dng"
#property version   "1.00"
//+------------------------------------------------------------------+
//| Includes                                                         |
//+------------------------------------------------------------------+
#include "..\..\NeuroNet_DNG\NeuroNet.mqh"
#include <Trade\SymbolInfo.mqh>
#include <Indicators\Oscilators.mqh>
//---
#define FileName        Symb.Name()+"_"+EnumToString((ENUM_TIMEFRAMES)Period())+"_"+IntegerToString(HistoryBars,3)+StringSubstr(__FILE__,0,StringFind(__FILE__,".",0))
#define FileName_AE     Symb.Name()+"_"+EnumToString((ENUM_TIMEFRAMES)Period())+"_ae"
#define CSV             __FILE__+".csv"
//---
enum ENUM_SIGNAL
  {
   Sell = -1,
   Undefine = 0,
   Buy = 1
  };
//+------------------------------------------------------------------+
//|   input parameters                                               |
//+------------------------------------------------------------------+
input int                  StudyPeriod =  2;            //Study period, years
/*input*/ uint                 HistoryBars =  40;            //Depth of history
ENUM_TIMEFRAMES            TimeFrame   =  PERIOD_CURRENT;
//---
input group                "---- RSI ----"
input int                  RSIPeriod   =  14;            //Period
input ENUM_APPLIED_PRICE   RSIPrice    =  PRICE_CLOSE;   //Applied price
//---
input group                "---- CCI ----"
input int                  CCIPeriod   =  14;            //Period
input ENUM_APPLIED_PRICE   CCIPrice    =  PRICE_TYPICAL; //Applied price
//---
input group                "---- ATR ----"
input int                  ATRPeriod   =  14;            //Period
//---
input group                "---- MACD ----"
input int                  FastPeriod  =  12;            //Fast
input int                  SlowPeriod  =  26;            //Slow
input int                  SignalPeriod =  9;            //Signal
input ENUM_APPLIED_PRICE   MACDPrice   =  PRICE_CLOSE;   //Applied price
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CSymbolInfo         *Symb;
MqlRates            Rates[];
CNet                *Net;
CBufferFloat        *TempData;
CiRSI               *RSI;
CiCCI               *CCI;
CiATR               *ATR;
CiMACD              *MACD;
CNet                *AE;
CBufferFloat         Fractals;

//---
float                dError;
float                dUndefine;
float                dForecast;
double               dPrevSignal;
datetime             dtStudied;
bool                 bEventStudy;
//+------------------------------------------------------------------+
//| Expert initialization function                                   |
//+------------------------------------------------------------------+
int OnInit()
  {
//---
   Symb = new CSymbolInfo();
   if(CheckPointer(Symb) == POINTER_INVALID || !Symb.Name(_Symbol))
      return INIT_FAILED;
   Symb.Refresh();
//---
   RSI = new CiRSI();
   if(CheckPointer(RSI) == POINTER_INVALID || !RSI.Create(Symb.Name(), TimeFrame, RSIPeriod, RSIPrice))
      return INIT_FAILED;
//---
   CCI = new CiCCI();
   if(CheckPointer(CCI) == POINTER_INVALID || !CCI.Create(Symb.Name(), TimeFrame, CCIPeriod, CCIPrice))
      return INIT_FAILED;
//---
   ATR = new CiATR();
   if(CheckPointer(ATR) == POINTER_INVALID || !ATR.Create(Symb.Name(), TimeFrame, ATRPeriod))
      return INIT_FAILED;
//---
   MACD = new CiMACD();
   if(CheckPointer(MACD) == POINTER_INVALID || !MACD.Create(Symb.Name(), TimeFrame, FastPeriod, SlowPeriod, SignalPeriod, MACDPrice))
      return INIT_FAILED;
//---
   AE = new CNet(NULL);
   if(CheckPointer(AE) == POINTER_INVALID)
      return INIT_FAILED;
   if(!AE.Load(FileName_AE + ".nnw", dError, dUndefine, dForecast, dtStudied, false))
      return INVALID_HANDLE;
//---
   if(!AE.GetLayerOutput(0, TempData))
      return INIT_FAILED;
   HistoryBars = TempData.Total() / 12;
//---
   Net = new CNet(NULL);
   ResetLastError();
   if(CheckPointer(Net) == POINTER_INVALID || !Net.Load(FileName + ".nnw", dError, dUndefine, dForecast, dtStudied, false))
     {
      printf("%s - %d -> Error of read %s prev Net %d", __FUNCTION__, __LINE__, FileName + ".nnw", GetLastError());
      CArrayObj *Topology = new CArrayObj();
      if(CheckPointer(Topology) == POINTER_INVALID)
         return INIT_FAILED;
      if(!AE.GetLayerOutput(5, TempData))
         return INIT_FAILED;
      //--- 0
      CLayerDescription *desc = new CLayerDescription();
      if(CheckPointer(desc) == POINTER_INVALID)
         return INIT_FAILED;
      desc.count = TempData.Total();
      desc.type = defNeuronBaseOCL;
      desc.optimization = ADAM;
      desc.activation = None;
      if(!Topology.Add(desc))
         return INIT_FAILED;
      //--- 1
      desc = new CLayerDescription();
      if(CheckPointer(desc) == POINTER_INVALID)
         return INIT_FAILED;
      desc.count = 300;
      desc.type = defNeuron;
      desc.activation = LReLU;
      desc.optimization = ADAM;
      if(!Topology.Add(desc))
         return INIT_FAILED;
      //--- 2
      desc = new CLayerDescription();
      if(CheckPointer(desc) == POINTER_INVALID)
         return INIT_FAILED;
      desc.count = 300;
      desc.type = defNeuron;
      desc.activation = TANH;
      desc.optimization = ADAM;
      if(!Topology.Add(desc))
         return INIT_FAILED;
      //--- 3
      desc = new CLayerDescription();
      if(CheckPointer(desc) == POINTER_INVALID)
         return INIT_FAILED;
      desc.count = 300;
      desc.type = defNeuron;
      desc.activation = TANH;
      desc.optimization = ADAM;
      if(!Topology.Add(desc))
         return INIT_FAILED;
      //--- 4
      desc = new CLayerDescription();
      if(CheckPointer(desc) == POINTER_INVALID)
         return INIT_FAILED;
      desc.count = 3;
      desc.type = defNeuron;
      desc.activation = SIGMOID;
      desc.optimization = ADAM;
      if(!Topology.Add(desc))
         return INIT_FAILED;
      delete Net;
      Net = new CNet(Topology);
      delete Topology;
      if(CheckPointer(Net) == POINTER_INVALID)
         return INIT_FAILED;
      dError = -1;
      dUndefine = 0;
      dForecast = 0;
      dtStudied = 0;
     }
//---
   bEventStudy = EventChartCustom(ChartID(), 1, (long)MathMax(0, MathMin(iTime(Symb.Name(), PERIOD_CURRENT, (int)(100 * Net.recentAverageSmoothingFactor * (dForecast >= 70 ? 1 : 10))), dtStudied)), 0, "Init");
//---
   return(INIT_SUCCEEDED);
  }
//+------------------------------------------------------------------+
//| Expert deinitialization function                                 |
//+------------------------------------------------------------------+
void OnDeinit(const int reason)
  {
//---
   if(CheckPointer(Symb) != POINTER_INVALID)
      delete Symb;
//---
   if(CheckPointer(AE) != POINTER_INVALID)
      delete AE;
//---
   if(CheckPointer(RSI) != POINTER_INVALID)
      delete RSI;
//---
   if(CheckPointer(CCI) != POINTER_INVALID)
      delete CCI;
//---
   if(CheckPointer(ATR) != POINTER_INVALID)
      delete ATR;
//---
   if(CheckPointer(MACD) != POINTER_INVALID)
      delete MACD;
//---
   if(CheckPointer(Net) != POINTER_INVALID)
      delete Net;
   if(CheckPointer(TempData) != POINTER_INVALID)
      delete TempData;
  }
//+------------------------------------------------------------------+
//| Expert tick function                                             |
//+------------------------------------------------------------------+
void OnTick()
  {
//---
   if(!bEventStudy && (dPrevSignal == -2 || dtStudied < SeriesInfoInteger(Symb.Name(), TimeFrame, SERIES_LASTBAR_DATE)))
      bEventStudy = EventChartCustom(ChartID(), 1, (long)MathMax(0, MathMin(iTime(Symb.Name(), PERIOD_CURRENT, (int)(100 * Net.recentAverageSmoothingFactor * (dForecast >= 70 ? 1 : 10))), dtStudied)), 0, "New Bar");
//---
   Comment(StringFormat("K7>2 A>1KB8O %s; PrevSignal %.5f; >45;L >1CG5=0 %s -> %s", (string)bEventStudy, dPrevSignal, TimeToString(dtStudied), TimeToString(SeriesInfoInteger(Symb.Name(), TimeFrame, SERIES_LASTBAR_DATE))));
  }
//+------------------------------------------------------------------+
//| Trade function                                                   |
//+------------------------------------------------------------------+
void OnTrade()
  {
//---
  }
//+------------------------------------------------------------------+
//| TradeTransaction function                                        |
//+------------------------------------------------------------------+
void OnTradeTransaction(const MqlTradeTransaction& trans,
                        const MqlTradeRequest& request,
                        const MqlTradeResult& result)
  {
//---
  }
//+------------------------------------------------------------------+
//| ChartEvent function                                              |
//+------------------------------------------------------------------+
void OnChartEvent(const int id,
                  const long &lparam,
                  const double &dparam,
                  const string &sparam)
  {
//---
   if(id == 1001)
     {
      Net.TrainMode(true);
      Train(lparam);
      bEventStudy = false;
      OnTick();
     }
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void Train(datetime StartTrainBar = 0)
  {
   int count = 0;
//---
   MqlDateTime start_time;
   TimeCurrent(start_time);
   start_time.year -= StudyPeriod;
   if(start_time.year <= 0)
      start_time.year = 1900;
   datetime st_time = StructToTime(start_time);
   dtStudied = MathMax(StartTrainBar, st_time);
   ulong last_tick = 0;
//---
   double prev_er = DBL_MAX;
   datetime bar_time = 0;
   bool stop = IsStopped();
   CArrayDouble *loss = new CArrayDouble();
//---
   int bars = CopyRates(Symb.Name(), TimeFrame, st_time, TimeCurrent(), Rates);
   if(!RSI.BufferResize(bars) || !CCI.BufferResize(bars) || !ATR.BufferResize(bars) || !MACD.BufferResize(bars))
     {
      ExpertRemove();
      return;
     }
   if(!ArraySetAsSeries(Rates, true))
     {
      ExpertRemove();
      return;
     }
   RSI.Refresh(OBJ_ALL_PERIODS);
   CCI.Refresh(OBJ_ALL_PERIODS);
   ATR.Refresh(OBJ_ALL_PERIODS);
   MACD.Refresh(OBJ_ALL_PERIODS);
//---
   MqlDateTime sTime;
   int total = (int)(bars - MathMax(HistoryBars, 0) - 300);
   do
     {
      prev_er = dError;
      stop = IsStopped();
      bool add_loop = false;
      //---
      for(int it = total; it >= 0 && !stop; it--)
        {
         int i = (int)((MathRand() * MathRand() / MathPow(32767, 2)) * (total)) + 300;
         TempData.Clear();
         int r = i + (int)HistoryBars;
         if(r > bars)
            continue;
         //---
         for(int b = 0; b < (int)HistoryBars; b++)
           {
            int bar_t = r - b;
            double open = Rates[bar_t].open;
            TimeToStruct(Rates[bar_t].time, sTime);
            double rsi = RSI.Main(bar_t);
            double cci = CCI.Main(bar_t);
            double atr = ATR.Main(bar_t);
            double macd = MACD.Main(bar_t);
            double sign = MACD.Signal(bar_t);
            if(rsi == EMPTY_VALUE || cci == EMPTY_VALUE || atr == EMPTY_VALUE || macd == EMPTY_VALUE || sign == EMPTY_VALUE)
               continue;
            //---
            if(!TempData.Add((float)(Rates[bar_t].close - open)) || !TempData.Add((float)(Rates[bar_t].high - open)) || !TempData.Add((float)(Rates[bar_t].low - open)) || !TempData.Add((float)Rates[bar_t].tick_volume / 1000.0f) ||
               !TempData.Add(sTime.hour) || !TempData.Add(sTime.day_of_week) || !TempData.Add(sTime.mon) ||
               !TempData.Add((float)rsi) || !TempData.Add((float)cci) || !TempData.Add((float)atr) || !TempData.Add((float)macd) || !TempData.Add((float)sign))
               break;
           }
         if(TempData.Total() < (int)HistoryBars * 12)
            continue;
         AE.feedForward(TempData, 12, true, (CBufferFloat*)NULL);
         if(!AE.GetLayerOutput(5, TempData))
            break;
         Net.feedForward(TempData, 2, true, (CBufferFloat*)NULL);
         Net.getResults(TempData);
         //---
         float sum = 0;
         for(int res = 0; res < 3; res++)
           {
            float temp = exp(TempData.At(res));
            sum += temp;
            TempData.Update(res, temp);
           }
         for(int res = 0; (res < 3 && sum > 0); res++)
            TempData.Update(res, TempData.At(res) / sum);
         //---
         switch(TempData.Argmax())
           {
            case 1:
               dPrevSignal = (TempData[1] != TempData[2] ? TempData[1] : 0);
               break;
            case 2:
               dPrevSignal = -TempData[2];
               break;
            default:
               dPrevSignal = 0;
               break;
           }
         if((GetTickCount64() - last_tick) >= 250)
           {
            string s = StringFormat("Study -> Era %d -> %.2f -> Undefine %.2f%% foracast %.2f%%\n %d of %d -> %.2f%% \nError %.2f\n%s -> %.2f ->> Buy %.5f - Sell %.5f - Undef %.5f", count, dError, dUndefine, dForecast, it + 1, total, (double)(it + 1.0) / (total) * 100, Net.getRecentAverageError(), EnumToString(DoubleToSignal(dPrevSignal)), dPrevSignal, TempData[1], TempData[2], TempData[0]);
            Comment(s);
            last_tick = GetTickCount64();
           }
         stop = IsStopped();
         if(!stop)
           {
            TempData.Clear();
            bool sell = (Rates[i - 1].high <= Rates[i].high && Rates[i + 1].high < Rates[i].high);
            bool buy = (Rates[i - 1].low >= Rates[i].low && Rates[i + 1].low > Rates[i].low);
            TempData.Add(!(buy || sell));
            TempData.Add(buy);
            TempData.Add(sell);
            Net.backProp(TempData, NULL, NULL);
            ENUM_SIGNAL signal = DoubleToSignal(dPrevSignal);
            if(signal != Undefine)
              {
               if((signal == Sell && sell) || (signal == Buy && buy))
                  dForecast += (100 - dForecast) / Net.recentAverageSmoothingFactor;
               else
                  dForecast -= dForecast / Net.recentAverageSmoothingFactor;
               dUndefine -= dUndefine / Net.recentAverageSmoothingFactor;
              }
            else
              {
               if(!(buy || sell))
                  dUndefine += (100 - dUndefine) / Net.recentAverageSmoothingFactor;
              }
           }
        }
      count++;
      for(int i = 0; i < 300; i++)
        {
         TempData.Clear();
         int r = i + (int)HistoryBars;
         if(r > bars)
            continue;
         //---
         for(int b = 0; b < (int)HistoryBars; b++)
           {
            int bar_t = r - b;
            double open = Rates[bar_t].open;
            TimeToStruct(Rates[bar_t].time, sTime);
            double rsi = RSI.Main(bar_t);
            double cci = CCI.Main(bar_t);
            double atr = ATR.Main(bar_t);
            double macd = MACD.Main(bar_t);
            double sign = MACD.Signal(bar_t);
            if(rsi == EMPTY_VALUE || cci == EMPTY_VALUE || atr == EMPTY_VALUE || macd == EMPTY_VALUE || sign == EMPTY_VALUE)
               continue;
            //---
            if(!TempData.Add((float)(Rates[bar_t].close - open)) || !TempData.Add((float)(Rates[bar_t].high - open)) || !TempData.Add((float)(Rates[bar_t].low - open)) || !TempData.Add((float)Rates[bar_t].tick_volume / 1000.0f) ||
               !TempData.Add(sTime.hour) || !TempData.Add(sTime.day_of_week) || !TempData.Add(sTime.mon) ||
               !TempData.Add((float)rsi) || !TempData.Add((float)cci) || !TempData.Add((float)atr) || !TempData.Add((float)macd) || !TempData.Add((float)sign))
               break;
           }
         if(TempData.Total() < (int)HistoryBars * 12)
            continue;
         AE.feedForward(TempData, 12, true, (CBufferFloat*)NULL);
         if(!AE.GetLayerOutput(5, TempData))
            break;
         Net.feedForward(TempData, 2, true, (CBufferFloat*)NULL);
         Net.getResults(TempData);
         //---
         float sum = 0;
         for(int res = 0; res < 3; res++)
           {
            float temp = exp(TempData.At(res));
            sum += temp;
            TempData.Update(res, temp);
           }
         for(int res = 0; (res < 3 && sum > 0); res++)
            TempData.Update(res, TempData.At(res) / sum);
         //---
         switch(TempData.Argmax())
           {
            case 1:
               dPrevSignal = (TempData[1] != TempData[2] ? TempData[1] : 0);
               break;
            case 2:
               dPrevSignal = (TempData[1] != TempData[2] ? -TempData[2] : 0);
               break;
            default:
               dPrevSignal = 0;
               break;
           }
         if(DoubleToSignal(dPrevSignal) == Undefine)
            DeleteObject(Rates[i].time);
         else
            DrawObject(Rates[i].time, dPrevSignal, Rates[i].high, Rates[i].low);
        }
      if(!stop)
        {
         dError = Net.getRecentAverageError();
         Net.Save(FileName + ".nnw", dError, dUndefine, dForecast, Rates[0].time, false);
         printf("Era %d -> error %.2f %% forecast %.2f", count, dError, dForecast);
         ChartScreenShot(0, FileName + IntegerToString(count) + ".png", 750, 400);
         int h = FileOpen(FileName + ".csv", FILE_READ | FILE_WRITE | FILE_CSV);
         if(h != INVALID_HANDLE)
           {
            FileSeek(h, 0, SEEK_END);
            FileWrite(h, lr, count, dError, dUndefine, dForecast);
            FileFlush(h);
            FileClose(h);
           }
        }
     }
   while(!(dError < 0.01 && (prev_er - dError) < 0.01) && !stop);
//---
   Comment("Write dinamic of error");
   int handle = FileOpen("ae_loss.csv", FILE_WRITE | FILE_CSV | FILE_ANSI, ",", CP_UTF8);
   if(handle == INVALID_HANDLE)
     {
      PrintFormat("Error of open loss file: %d", GetLastError());
      delete loss;
      return;
     }
   for(int i = 0; i < loss.Total(); i++)
      if(FileWrite(handle, loss.At(i)) <= 0)
         break;
   FileClose(handle);
   PrintFormat("The dynamics of the error change is saved to a file %s\\%s",
               TerminalInfoString(TERMINAL_DATA_PATH), "ae_loss.csv");
   delete loss;
   Comment("");
   ExpertRemove();
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
ENUM_SIGNAL DoubleToSignal(double value)
  {
   value = NormalizeDouble(value, 1);
   if(MathAbs(value) > 1 || MathAbs(value) <= 0)
      return Undefine;
   if(value > 0)
      return Buy;
   else
      return Sell;
//---
   return Undefine;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void DrawObject(datetime time, double signal, double high, double low)
  {
   double price = 0;
   int arrow = 0;
   color clr = 0;
   ENUM_ARROW_ANCHOR anch = ANCHOR_BOTTOM;
   switch(DoubleToSignal(signal))
     {
      case Buy:
         price = low;
         arrow = 217;
         clr = clrBlue;
         anch = ANCHOR_TOP;
         break;
      case Sell:
         price = high;
         arrow = 218;
         clr = clrRed;
         anch = ANCHOR_BOTTOM;
         break;
     }
   if(price == 0 || arrow == 0)
      return;
//---
   string name = TimeToString(time);
   if(ObjectFind(0, name) < 0)
     {
      ResetLastError();
      if(!ObjectCreate(0, name, OBJ_ARROW, 0, time, 0))
        {
         printf("Error of creating object %d", GetLastError());
         return;
        }
     }
//printf("%s - %d -> %s",__FUNCTION__,__LINE__,name);
   ObjectSetDouble(0, name, OBJPROP_PRICE, price);
   ObjectSetInteger(0, name, OBJPROP_ARROWCODE, arrow);
   ObjectSetInteger(0, name, OBJPROP_COLOR, clr);
   ObjectSetInteger(0, name, OBJPROP_ANCHOR, anch);
   ObjectSetString(0, name, OBJPROP_TOOLTIP, EnumToString(DoubleToSignal(signal)) + " " + DoubleToString(signal, 5));
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void DeleteObject(datetime time)
  {
   string name = TimeToString(time);
   if(ObjectFind(0, name) >= 0)
      ObjectDelete(0, name);
  }
//+------------------------------------------------------------------+