//+------------------------------------------------------------------+
//|                                                    Catch22Lab.mq5|
//|                                      MMQ — Muhammad Minhas Qamar |
//|                                   www.mql5.com/en/articles/23488 |
//+------------------------------------------------------------------+
#property copyright "MMQ — Muhammad Minhas Qamar"
#property link      "https://www.mql5.com/en/articles/23488"
#property version   "1.00"
#property strict
#property script_show_inputs

#include <Catch22\Catch22Features.mqh>
#include <Math\Alglib\alglib.mqh>

//--- data / labeling
input int      InpBars        = 40000;  // history bars to sample
input int      InpWindow      = 128;    // rolling window for catch22
input int      InpHorizon     = 10;     // forward window for realized-vol label (bars)
input int      InpStride      = 10;     // bars between samples (>=horizon => non-overlapping labels)
input double   InpTrainFrac   = 0.70;   // fraction used for in-sample training
input int      InpEmbargo     = 5;      // extra purge samples beyond the label horizon
//--- model
input int      InpTrees       = 300;    // trees per forest
input double   InpRndVarRatio = 0.0;    // 0 => ALGLIB auto (sqrt) feature subsample
input int      InpSeed        = 42;     // RNG seed
//--- output
input bool     InpExportModel = true;   // export the combined-arm model for the EA

//--- target: the realized-volatility REGIME of the NEXT window, split into
//--- terciles (LOW / MED / HIGH). catch22 was designed for time-series
//--- classification, and characterizing volatility structure is its home
//--- turf — a far more learnable task than predicting FX direction. The
//--- tercile thresholds are fitted on the TRAINING window only.
#define LAB_LOW  0
#define LAB_MED  1
#define LAB_HIGH 2
#define NCLASSES 3

//+------------------------------------------------------------------+
//| One fully-built sample: feature row describing the CURRENT window|
//| plus the raw realized volatility of the NEXT (forward) window.   |
//| Tercile 'label' is assigned later, once the train-only thresholds|
//| are known, so no test-set information leaks into the binning.    |
//+------------------------------------------------------------------+
struct SSample
  {
   double            f[];      // combined feature vector (classic + catch22)
   double            fwdvol;   // realized vol of the forward window (raw)
   int               label;    // LOW/MED/HIGH, filled after tercile fit
   datetime          time;     // bar time (diagnostics)
  };

SSample g_samples[];
int     g_nsamples=0;

//+------------------------------------------------------------------+
//| Realized volatility of the FORWARD window for the bar at 'shift':|
//| the standard deviation of the InpHorizon log returns that follow |
//| the current bar (bars shift-1 .. shift-InpHorizon, i.e. newer).  |
//| Returns -1 if the forward history is unavailable.                |
//|  NOTE shift indexing: larger shift = older bar. "Forward" means  |
//|  decreasing shift.                                               |
//+------------------------------------------------------------------+
double ForwardVol(const string sym,ENUM_TIMEFRAMES tf,int shift)
  {
   int need=InpHorizon+1;
   int startShift=shift-InpHorizon;         // oldest forward bar we read
   if(startShift<0)
      return -1.0;
   double close[];
   if(CopyClose(sym,tf,startShift,need,close)<need)
      return -1.0;
   ArraySetAsSeries(close,false);           // oldest -> newest

//--- log returns across the forward window, then their std-dev
   double r[];
   ArrayResize(r,InpHorizon);
   for(int i=0;i<InpHorizon;i++)
     {
      if(close[i]<=0.0 || close[i+1]<=0.0)
        {
         r[i]=0.0;
         continue;
        }
      r[i]=MathLog(close[i+1]/close[i]);
     }
   double mean=0.0;
   for(int i=0;i<InpHorizon;i++)
      mean+=r[i];
   mean/=InpHorizon;
   double var=0.0;
   for(int i=0;i<InpHorizon;i++)
     {
      double d=r[i]-mean;
      var+=d*d;
     }
   return MathSqrt(var/InpHorizon);
  }

//+------------------------------------------------------------------+
//| Build the full sample set once (combined features + labels). The |
//| three arms are later trained by slicing columns out of this.     |
//+------------------------------------------------------------------+
bool BuildSamples(void)
  {
   CCatch22FeatureBuilder fb;
   if(!fb.Init(_Symbol,_Period,InpWindow,ARM_COMBINED))
     {
      Print("feature builder init failed");
      return false;
     }

//--- warm-up: leave InpWindow+1 bars of history behind each sample and
//--- InpHorizon bars of forward room ahead of it. We step by InpStride
//--- bars so that (when stride >= horizon) no two samples share a forward
//--- window — avoiding overlapping-outcome correlation (AFML Ch.4).
   int step=(InpStride>0)?InpStride:1;
   int oldest=InpBars;                  // oldest shift we attempt
   int newest=InpHorizon+1;             // need forward room
   ArrayResize(g_samples,InpBars/step+1);
   g_nsamples=0;

   for(int shift=oldest;shift>=newest;shift-=step)
     {
      double vol=ForwardVol(_Symbol,_Period,shift);
      if(vol<0.0 || !MathIsValidNumber(vol))
         continue;

      double row[];
      if(!fb.Build(shift,row))
         continue;
      if(ArraySize(row)!=CCatch22FeatureBuilder::DimOf(ARM_COMBINED))
         continue;

      //--- guard NaN/Inf
      bool ok=true;
      for(int j=0;j<ArraySize(row);j++)
         if(!MathIsValidNumber(row[j]))
           {
            ok=false;
            break;
           }
      if(!ok)
         continue;

      ArrayResize(g_samples[g_nsamples].f,ArraySize(row));
      for(int j=0;j<ArraySize(row);j++)
         g_samples[g_nsamples].f[j]=row[j];
      g_samples[g_nsamples].fwdvol=vol;
      g_samples[g_nsamples].label =-1;    // assigned after tercile fit
      g_samples[g_nsamples].time  =iTime(_Symbol,_Period,shift);
      g_nsamples++;
     }

   ArrayResize(g_samples,g_nsamples);
   fb.Deinit();
   PrintFormat("Built %d samples (%d features each).",
               g_nsamples,CCatch22FeatureBuilder::DimOf(ARM_COMBINED));
   return g_nsamples>200;
  }

//+------------------------------------------------------------------+
//| Assign LOW/MED/HIGH labels from forward-vol terciles fitted on   |
//| the TRAINING samples only [0,trainEnd). Applying train thresholds|
//| to the test set is essential: fitting terciles on all samples    |
//| would leak the test-period volatility distribution into labels.  |
//+------------------------------------------------------------------+
void AssignVolTerciles(int trainEnd)
  {
   double v[];
   ArrayResize(v,trainEnd);
   for(int i=0;i<trainEnd;i++)
      v[i]=g_samples[i].fwdvol;
   ArraySort(v);
   double q1=v[(int)(0.3333*(trainEnd-1))];
   double q2=v[(int)(0.6667*(trainEnd-1))];

   for(int i=0;i<g_nsamples;i++)
     {
      double vol=g_samples[i].fwdvol;
      if(vol<=q1)
         g_samples[i].label=LAB_LOW;
      else
         if(vol<=q2)
            g_samples[i].label=LAB_MED;
         else
            g_samples[i].label=LAB_HIGH;
     }
   PrintFormat("Vol terciles (train-fit): q1=%.6f q2=%.6f",q1,q2);
  }

//+------------------------------------------------------------------+
//| Column offsets so an arm can be sliced out of the combined row.  |
//| Classic occupies [0,C22_NCLASSIC); catch22 occupies the rest.    |
//+------------------------------------------------------------------+
void ArmColumns(ENUM_C22_ARM arm,int &start,int &count)
  {
   if(arm==ARM_CLASSIC)
     {
      start=0;
      count=C22_NCLASSIC;
     }
   else
      if(arm==ARM_CATCH22)
        {
         start=C22_NCLASSIC;
         count=CATCH22_N;
        }
      else
        {
         start=0;
         count=C22_NCLASSIC+CATCH22_N;
        }
  }

//+------------------------------------------------------------------+
//| Fill an ALGLIB CMatrixDouble for one arm over a sample index     |
//| range [lo,hi). Last column holds the class label (ALGLIB dataset |
//| convention: nvars feature columns followed by the class index).  |
//+------------------------------------------------------------------+
void FillMatrix(CMatrixDouble &xy,ENUM_C22_ARM arm,int lo,int hi)
  {
   int start,count;
   ArmColumns(arm,start,count);
   int rows=hi-lo;
   xy.Resize(rows,count+1);
   for(int r=0;r<rows;r++)
     {
      for(int c=0;c<count;c++)
         xy.Set(r,c,g_samples[lo+r].f[start+c]);
      xy.Set(r,count,(double)g_samples[lo+r].label);
     }
  }

//+------------------------------------------------------------------+
//| Train one random forest on the PURGED range [0,trainEnd), then   |
//| evaluate on [split,N). trainEnd < split leaves a purge+embargo   |
//| gap so that no training label's forward horizon overlaps the test|
//| window (AFML Ch.7 purging). Prints OOS accuracy, per-class       |
//| precision/recall/F1, and the top feature importances.            |
//+------------------------------------------------------------------+
void RunArm(ENUM_C22_ARM arm,string armName,int trainEnd,int split,
            CDecisionForestShell &model)
  {
   int start,count;
   ArmColumns(arm,start,count);
   int N=g_nsamples;

//--- training matrix over the purged in-sample range only
   CMatrixDouble xytrain;
   FillMatrix(xytrain,arm,0,trainEnd);

//--- build the forest with permutation importance enabled
   CDecisionForestBuilder builder;
   CDFReportShell rep;
   CAlglib::DFBuilderCreate(builder);
   CAlglib::DFBuilderSetDataset(builder,xytrain,trainEnd,count,NCLASSES);
   if(InpRndVarRatio>0.0)
      CAlglib::DFBuilderSetRndVarsRatio(builder,InpRndVarRatio);
   else
      CAlglib::DFBuilderSetRndVarsAuto(builder);
   CAlglib::DFBuilderSetSubsampleRatio(builder,0.66);
   CAlglib::DFBuilderSetSeed(builder,InpSeed);
//--- Permutation (MDA) importance: measures the drop in predictive power
//--- when each variable is shuffled — the AFML-recommended method. It is
//--- the importance mode that populates reliably in this ALGLIB build
//--- (the OOB-Gini path returns zeros here). Values are ~0 for features
//--- the model does not actually rely on, which is itself informative.
   CAlglib::DFBuilderSetImportancePermutation(builder);
   CAlglib::DFBuilderBuildRandomForest(builder,InpTrees,model,rep);

//--- out-of-sample evaluation
   int conf[NCLASSES][NCLASSES];
   for(int a=0;a<NCLASSES;a++)
      for(int b=0;b<NCLASSES;b++)
         conf[a][b]=0;

   double x[],y[];
   ArrayResize(x,count);
   int correct=0,total=0;
   for(int i=split;i<N;i++)
     {
      for(int c=0;c<count;c++)
         x[c]=g_samples[i].f[start+c];
      CAlglib::DFProcess(model,x,y);       // y = class probabilities
      int pred=0;
      double best=y[0];
      for(int k=1;k<NCLASSES;k++)
         if(y[k]>best)
           {
            best=y[k];
            pred=k;
           }
      int truth=g_samples[i].label;
      conf[truth][pred]++;
      if(pred==truth)
         correct++;
      total++;
     }

   double acc=(total>0)?(double)correct/total:0.0;
   PrintFormat("---- ARM %s (%d features) ----",armName,count);
   PrintFormat("OOS samples: %d   accuracy: %.4f   OOB relclserr: %.4f",
               total,acc,rep.GetOOBRelClsError());

//--- per-class precision / recall / F1
   double f1sum=0.0;
   for(int k=0;k<NCLASSES;k++)
     {
      int tp=conf[k][k];
      int fp=0,fn=0;
      for(int j=0;j<NCLASSES;j++)
        {
         if(j!=k)
           {
            fp+=conf[j][k];
            fn+=conf[k][j];
           }
        }
      double prec=(tp+fp>0)?(double)tp/(tp+fp):0.0;
      double rec =(tp+fn>0)?(double)tp/(tp+fn):0.0;
      double f1  =(prec+rec>0)?2*prec*rec/(prec+rec):0.0;
      f1sum+=f1;
      string cname=(k==LAB_LOW)?"LOW":(k==LAB_MED)?"MED":"HIGH";
      PrintFormat("   %-4s  precision=%.3f recall=%.3f f1=%.3f",cname,prec,rec,f1);
     }
   PrintFormat("   macro-F1=%.4f",f1sum/NCLASSES);

//--- feature importances (permutation/MDA), top 10. m_varimportances is a
//--- CRowDouble (rating per variable) and m_topvars a CRowInt (variable
//--- indices already sorted by descending importance).
   CDFReport *r=rep.GetInnerObj();
   int nv  =r.m_varimportances.Size();
   int topv=r.m_topvars.Size();
   CCatch22FeatureBuilder namer;
   namer.Init(_Symbol,_Period,InpWindow,arm);
   PrintFormat("   top features (permutation/MDA importance):");
   int shown=0;
   for(int t=0;t<topv && shown<10;t++)
     {
      int idx=r.m_topvars[t];
      if(idx<0 || idx>=nv)
         continue;
      PrintFormat("     %2d. %-40s %.5f",shown+1,namer.ColumnName(idx),
                  r.m_varimportances[idx]);
      shown++;
     }
   namer.Deinit();
  }

//+------------------------------------------------------------------+
//| Export a trained forest to a flat file the EA can load.          |
//+------------------------------------------------------------------+
void ExportModel(CDecisionForestShell &model,ENUM_C22_ARM arm)
  {
   string s;
   CAlglib::DFSerialize(model,s);
//--- write as raw bytes so the EA's binary read round-trips exactly
//--- (text mode would translate line endings and corrupt the tokens).
//--- FILE_COMMON puts it in the shared Common\Files folder, which the
//--- Strategy Tester's sandboxed EA can also read (the tester has its own
//--- private Files\ and would not see a terminal-local write).
   string fn="Catch22\\model_combined.txt";
   int h=FileOpen(fn,FILE_WRITE|FILE_BIN|FILE_COMMON);
   if(h==INVALID_HANDLE)
     {
      PrintFormat("model export failed (err %d)",GetLastError());
      return;
     }
   uchar bytes[];
   int n=StringToCharArray(s,bytes,0,StringLen(s),CP_ACP);
   FileWriteArray(h,bytes,0,n);
   FileClose(h);
   PrintFormat("Exported model to MQL5\\Files\\%s (%d chars). Arm=%d window=%d",
               fn,StringLen(s),(int)arm,InpWindow);
  }

//+------------------------------------------------------------------+
//| Persist the run configuration and, crucially, the train/test     |
//| calendar boundaries to a manifest file next to the model. The EA |
//| backtest must start at the TEST-FROM date recorded here so it is |
//| evaluated only on out-of-sample data.                            |
//+------------------------------------------------------------------+
void WriteManifest(datetime trainFrom,datetime trainTo,
                   datetime testFrom,datetime testTo)
  {
   string fn="Catch22\\manifest.txt";
   int h=FileOpen(fn,FILE_WRITE|FILE_TXT|FILE_ANSI|FILE_COMMON);
   if(h==INVALID_HANDLE)
     {
      PrintFormat("manifest write failed (err %d)",GetLastError());
      return;
     }
   FileWriteString(h,"Catch22 Lab manifest\r\n");
   FileWriteString(h,StringFormat("symbol=%s\r\n",_Symbol));
   FileWriteString(h,StringFormat("timeframe=%s\r\n",EnumToString(_Period)));
   FileWriteString(h,StringFormat("window=%d horizon=%d stride=%d\r\n",
                                  InpWindow,InpHorizon,InpStride));
   FileWriteString(h,StringFormat("train_from=%s\r\n",TimeToString(trainFrom,TIME_DATE|TIME_MINUTES)));
   FileWriteString(h,StringFormat("train_to=%s\r\n",  TimeToString(trainTo,  TIME_DATE|TIME_MINUTES)));
   FileWriteString(h,StringFormat("test_from=%s\r\n", TimeToString(testFrom, TIME_DATE|TIME_MINUTES)));
   FileWriteString(h,StringFormat("test_to=%s\r\n",   TimeToString(testTo,   TIME_DATE|TIME_MINUTES)));
   FileWriteString(h,StringFormat("model=model_combined.txt (arm=COMBINED, %d features)\r\n",
                                  CCatch22FeatureBuilder::DimOf(ARM_COMBINED)));
   FileClose(h);
   PrintFormat("Wrote MQL5\\Files\\%s",fn);
  }

//+------------------------------------------------------------------+
//| Script entry: build the dataset once, split chronologically,     |
//| then train and report all three arms; export the combined model. |
//+------------------------------------------------------------------+
void OnStart()
  {
   PrintFormat("=== Catch22 Lab: %s %s, bars=%d window=%d ===",
               _Symbol,EnumToString(_Period),InpBars,InpWindow);

   if(!BuildSamples())
     {
      Print("Not enough samples — increase InpBars or history.");
      return;
     }

   int split=(int)(g_nsamples*InpTrainFrac);

//--- purge: a training sample's label is the realized vol of its forward
//--- window (InpHorizon bars). Samples are InpStride bars apart, so the
//--- number of training samples whose forward window can reach the test
//--- block is ceil(horizon/stride). Drop those, plus an embargo, so train
//--- and test share no information (AFML Ch.7 purging + embargo).
   int step=(InpStride>0)?InpStride:1;
   int purge=(InpHorizon+step-1)/step+InpEmbargo;
   int trainEnd=split-purge;
   if(trainEnd<200)
     {
      PrintFormat("Purged train set too small (%d). Increase InpBars or InpTrainFrac.",
                  trainEnd);
      return;
     }
   PrintFormat("Chronological split: train[0..%d) PURGE[%d..%d) test[%d..%d)",
               trainEnd,trainEnd,split,split,g_nsamples);
   PrintFormat("  purge gap = %d samples (horizon %d + embargo %d)",
               purge,InpHorizon,InpEmbargo);

//--- fit LOW/MED/HIGH vol terciles on the training block only, then label
   AssignVolTerciles(trainEnd);

//--- Report the CALENDAR ranges. g_samples[0] is the oldest bar and
//--- g_samples[N-1] the newest, so these times bound each block. The
//--- TEST-FROM date is what you type into the Strategy Tester's "From"
//--- field when backtesting Catch22EA on the exported model, so the EA
//--- is evaluated only on data the model never trained on.
   datetime trainFrom=g_samples[0].time;
   datetime trainTo  =g_samples[trainEnd-1].time;
   datetime testFrom =g_samples[split].time;
   datetime testTo   =g_samples[g_nsamples-1].time;
   Print("--- date ranges (use TEST-FROM in the Strategy Tester) ---");
   PrintFormat("  TRAIN : %s  ->  %s",
               TimeToString(trainFrom,TIME_DATE|TIME_MINUTES),
               TimeToString(trainTo,  TIME_DATE|TIME_MINUTES));
   PrintFormat("  TEST  : %s  ->  %s   <== set tester 'From' = %s",
               TimeToString(testFrom, TIME_DATE|TIME_MINUTES),
               TimeToString(testTo,   TIME_DATE|TIME_MINUTES),
               TimeToString(testFrom, TIME_DATE));
   WriteManifest(trainFrom,trainTo,testFrom,testTo);

//--- class balance sanity (train terciles => train is ~1/3 each; test
//--- may drift if the vol regime shifts, which is itself informative)
   int cnt[NCLASSES];
   ArrayInitialize(cnt,0);
   for(int i=split;i<g_nsamples;i++)
      cnt[g_samples[i].label]++;
   PrintFormat("test-set class counts: LOW=%d MED=%d HIGH=%d",
               cnt[LAB_LOW],cnt[LAB_MED],cnt[LAB_HIGH]);

   CDecisionForestShell mClassic,mCatch22,mCombined;
   RunArm(ARM_CLASSIC,"CLASSIC",trainEnd,split,mClassic);
   RunArm(ARM_CATCH22,"CATCH22",trainEnd,split,mCatch22);
   RunArm(ARM_COMBINED,"COMBINED",trainEnd,split,mCombined);

   if(InpExportModel)
      ExportModel(mCombined,ARM_COMBINED);

   Print("=== Lab done ===");
  }
//+------------------------------------------------------------------+