Warrior_EA/Expert/AIBase/AutoTune.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//| Genetic indicator auto-tuner (GA population, crossover, successi|
//|                                                                  |
//| PARTIAL IMPLEMENTATION FILE - not standalone.                     |
//| This holds CExpertSignalAIBase method BODIES only. The class      |
//| declaration lives in Expert\ExpertSignalAIBase.mqh, which         |
//| #includes this file at the bottom, after the declaration. Do not  |
//| include it anywhere else and do not compile it on its own.        |
//|                                                                  |
//| Split out purely to make the 8216-line original navigable; the    |
//| code inside was moved verbatim, not rewritten.                    |
//+------------------------------------------------------------------+
#ifndef WARRIOR_AIBASE_AUTOTUNE_MQH
#define WARRIOR_AIBASE_AUTOTUNE_MQH
//+------------------------------------------------------------------+
//| Successive-halving era budget for rung 0..GA_RUNGS-1: cheap 3-era |
//| screen first, then promote survivors to progressively longer,     |
//| more-truthful training. See TuneIndicatorsAndTrain / GA_RUNGS.     |
//+------------------------------------------------------------------+
int CExpertSignalAIBase::GaRungEras(int rung)
  {
   switch(rung)
     {
      case 0:  return 3;   // screening rung (the operator's "3 eras" - noisy but cheap)
      case 1:  return 8;   // survivors get more budget
      default: return 20;  // finalists validated at a realistic budget
     }
  }
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaExtract(int candIdx, double &out[])
  {
   ArrayResize(out, AD_TUNE_PARAM_COUNT);
   int base = candIdx * AD_TUNE_PARAM_COUNT;
   for(int k = 0; k < AD_TUNE_PARAM_COUNT; k++)
      out[k] = m_gaPop[base + k];
  }
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaStore(int candIdx, const double &in[])
  {
   int base = candIdx * AD_TUNE_PARAM_COUNT;
   for(int k = 0; k < AD_TUNE_PARAM_COUNT; k++)
      m_gaPop[base + k] = in[k];
  }
//+------------------------------------------------------------------+
//| Mutate a candidate IN its valid ranges by reusing the tuner's own |
//| PerturbRandom() (single source of truth for per-param bounds and  |
//| preset snapping) - Unflatten -> perturb N times -> Flatten back.  |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaMutate(double &cand[], int mutations)
  {
   m_indicatorTuner.Unflatten(cand);
   for(int m = 0; m < mutations; m++)
      m_indicatorTuner.PerturbRandom(m_useADCumulativeDelta, m_useADShorteningOfThrust, m_useADWyckoffEventStream,
                                     m_useADWyckoffFailedStructure, m_useADWyckoffSignificantBarInversion, m_useMA, m_useRSI,
                                     m_useMACD, m_useIchimoku);
   m_indicatorTuner.Flatten(cand);
  }
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaRandomCandidate(const double &base[], double &out[], int mutations)
  {
   ArrayResize(out, AD_TUNE_PARAM_COUNT);
   for(int k = 0; k < AD_TUNE_PARAM_COUNT; k++)
      out[k] = base[k];
   GaMutate(out, mutations);
  }
//+------------------------------------------------------------------+
//| Block (whole-indicator) uniform crossover: each indicator's param |
//| BLOCK is taken intact from one parent or the other, so a child    |
//| recombines settings ACROSS indicators (the interaction-aware part)|
//| while keeping each indicator's own params internally coherent.    |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaBlockCrossover(const double &pa[], const double &pb[], double &child[])
  {
   ArrayResize(child, AD_TUNE_PARAM_COUNT);
   //--- block boundaries in the ADIndicatorTuner Flatten() layout: CumDelta[0..6], SOT[7..9],
   //--- WES[10..17], WFS[18..25], WSBI[26..29], MA[30..31], RSI[32].
   int bounds[] = {0, 7, 10, 18, 26, 30, 32, AD_TUNE_PARAM_COUNT};
   int nblocks = ArraySize(bounds) - 1;
   for(int b = 0; b < nblocks; b++)
     {
      bool fromA = (MathRand() % 2 == 0);
      for(int k = bounds[b]; k < bounds[b + 1]; k++)
         child[k] = fromA ? pa[k] : pb[k];
     }
  }
//+------------------------------------------------------------------+
//| Sort the ALIVE candidate indices (m_gaAlive[0..m_gaAliveCount-1]) |
//| by their m_gaScore descending (simple insertion sort - counts are |
//| tiny). After this the best survivors sit at the front.            |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaSortAliveByScoreDesc(void)
  {
   for(int i = 1; i < m_gaAliveCount; i++)
     {
      int key = m_gaAlive[i];
      double keyScore = m_gaScore[key];
      int j = i - 1;
      while(j >= 0 && m_gaScore[m_gaAlive[j]] < keyScore)
        {
         m_gaAlive[j + 1] = m_gaAlive[j];
         j--;
        }
      m_gaAlive[j + 1] = key;
     }
  }
//+------------------------------------------------------------------+
//| Build the next generation: elite (global best) carried unchanged, |
//| the rest bred by block crossover of two survivors + mutation.     |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::GaBreedNextGeneration(void)
  {
   int stride = AD_TUNE_PARAM_COUNT;
   double next[];
   ArrayResize(next, m_gaPopSize * stride);
   //--- child 0 = elitism: the best candidate found so far, unchanged
   for(int k = 0; k < stride; k++)
      next[k] = m_gaBestParams[k];
   int parents = MathMax(1, m_gaAliveCount); // survivors of the just-finished generation (sorted best-first)
   for(int c = 1; c < m_gaPopSize; c++)
     {
      double pa[], pb[], child[];
      GaExtract(m_gaAlive[MathRand() % parents], pa);
      GaExtract(m_gaAlive[MathRand() % parents], pb);
      GaBlockCrossover(pa, pb, child);
      GaMutate(child, 1 + MathRand() % 3);
      for(int k = 0; k < stride; k++)
         next[c * stride + k] = child[k];
     }
   ArrayCopy(m_gaPop, next);
  }
//+------------------------------------------------------------------+
//| Outer loop around Train(). AutoTuneIndicators off (or nothing     |
//| tunable) = pure pass-through to Train(). Otherwise runs a genetic |
//| population search with successive-halving evaluation: candidates  |
//| (full indicator-param vectors, MA type included) are scored in a  |
//| THROWAWAY net (m_evalMode - the deployed weights are never touched |
//| by the search), fixed-seed and averaged over GA_SEEDS runs, ranked |
//| on BALANCED accuracy; block crossover recombines settings across  |
//| indicators (interaction-aware). Only the final winner gets one    |
//| full deploy-retrain. All state is resumable across chunked calls. |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::TuneIndicatorsAndTrain(datetime StartTrainBar = 0)
  {
   bool anyTunable = (m_useADCumulativeDelta || m_useADShorteningOfThrust || m_useADWyckoffEventStream ||
                       m_useADWyckoffFailedStructure || m_useADWyckoffSignificantBarInversion ||
                       m_useMA || m_useRSI || m_useMACD || m_useIchimoku);
   if(!m_autoTuneIndicators || !anyTunable)
     {
      Train(StartTrainBar);
      return;
     }
   //--- ---- initialize a fresh search ----
   if(!m_gaActive)
     {
      m_gaActive = true;
      m_gaFinalRetrain = false;
      m_evalMode = false;
      m_tuneStartTrainBar = StartTrainBar;
      m_gaPopSize = (int)MathMax(4, m_indicatorTuneTrials);
      m_gaMaxGen = GA_MAX_GENERATIONS;
      m_gaGen = 0;
      m_gaRung = 0;
      m_gaCandPos = 0;
      m_gaSeedIdx = 0;
      ArrayResize(m_gaPop, m_gaPopSize * AD_TUNE_PARAM_COUNT);
      ArrayResize(m_gaScore, m_gaPopSize);
      ArrayResize(m_gaScoreSum, m_gaPopSize);
      ArrayResize(m_gaAlive, m_gaPopSize);
      ArrayInitialize(m_gaScore, 0.0);
      ArrayInitialize(m_gaScoreSum, 0.0);
      //--- seed population: candidate 0 = the user's configured starting params (elite seed), the rest are
      //--- random mutations of it so generation 0 already spans the space
      double base[];
      m_indicatorTuner.Flatten(base);
      GaStore(0, base);
      for(int c = 1; c < m_gaPopSize; c++)
        {
         double rc[];
         GaRandomCandidate(base, rc, 5);
         GaStore(c, rc);
        }
      for(int a = 0; a < m_gaPopSize; a++)
         m_gaAlive[a] = a;
      m_gaAliveCount = m_gaPopSize;
      ArrayResize(m_gaBestParams, AD_TUNE_PARAM_COUNT);
      for(int k = 0; k < AD_TUNE_PARAM_COUNT; k++)
         m_gaBestParams[k] = base[k]; // fallback winner = base until a finalist beats it
      m_gaBestScore = -1;
      m_gaHaveBest = true;
      Print(ID + ": auto-tune GENETIC search started - population " + IntegerToString(m_gaPopSize) +
            ", " + IntegerToString(m_gaMaxGen) + " generations, " + IntegerToString(GA_RUNGS) +
            " successive-halving rungs, " + IntegerToString(GA_SEEDS) + "-seed averaged, ranked on balanced accuracy");
     }
   //--- ---- final full retrain on the winning params (deploys) ----
   if(m_gaFinalRetrain)
     {
      Train(m_tuneStartTrainBar);
      if(m_trainRunActive)
         return; // still training the winner - resume next call
      m_gaActive = false;
      m_gaFinalRetrain = false;
      m_tuneTrialIndex = -1;
      Print(ID + ": auto-tune complete - deployed winner (balanced acc " + DoubleToString(m_gaBestScore, 1) + "%) after full retrain");
      return;
     }
   //--- ---- evaluate the current candidate at the current rung/seed (sandboxed, throwaway net) ----
   int candIdx = m_gaAlive[m_gaCandPos];
   if(!m_trainRunActive)
     {
      double cand[];
      GaExtract(candIdx, cand);
      m_indicatorTuner.Unflatten(cand);
      ReInitADIndicators(m_indicatorsPtr); // also invalidates the feature cache (params changed)
      MathSrand(GA_SEED_BASE + m_gaSeedIdx); // fixed per-seed init -> reproducible, removes init luck
      BuildFreshTopology();                  // builds into the throwaway Net (overwritten each candidate)
      m_evalMode = true;
      m_evalEraBudget = GaRungEras(m_gaRung);
      dError = -1;
      dUndefine = 0;
      dForecast = 0;
      dOosForecast = 0;
      m_oosSamples = 0;
      dOosError = -1;
      SetStatusLabel(StringFormat(ID + " : auto-tune gen %d/%d, rung %d, cand %d/%d, seed %d/%d",
                                  m_gaGen + 1, m_gaMaxGen, m_gaRung, m_gaCandPos + 1, m_gaAliveCount,
                                  m_gaSeedIdx + 1, GA_SEEDS));
     }
   Train(m_tuneStartTrainBar);
   if(m_trainRunActive)
      return; // candidate's capped eval yielded mid-chunk - resume next call
   //--- this (candidate, seed) run finished: accumulate its balanced-accuracy score
   m_gaScoreSum[candIdx] += MathMax(0.0, m_bestBalancedOos);
   m_gaSeedIdx++;
   if(m_gaSeedIdx < GA_SEEDS)
      return; // run the next fixed seed for this same candidate
   //--- all seeds done for this candidate: finalize its averaged score
   m_gaScore[candIdx] = m_gaScoreSum[candIdx] / GA_SEEDS;
   m_gaScoreSum[candIdx] = 0.0;
   m_gaSeedIdx = 0;
   //--- only the FINAL rung (realistic budget) is allowed to set the deployable global best, so the winner
   //--- is never an unvalidated candidate that got a lucky 3-era screen
   if(m_gaRung == GA_RUNGS - 1 && m_gaScore[candIdx] > m_gaBestScore)
     {
      m_gaBestScore = m_gaScore[candIdx];
      GaExtract(candIdx, m_gaBestParams);
     }
   Print(ID + StringFormat(": auto-tune gen %d/%d rung %d - candidate %d/%d balanced acc %.1f%%",
                           m_gaGen + 1, m_gaMaxGen, m_gaRung, m_gaCandPos + 1, m_gaAliveCount, m_gaScore[candIdx]));
   m_gaCandPos++;
   if(m_gaCandPos < m_gaAliveCount)
      return; // next candidate at this rung
   //--- every alive candidate scored at this rung: rank them
   GaSortAliveByScoreDesc();
   if(m_gaRung < GA_RUNGS - 1)
     {
      //--- successive-halving: promote the top half to the next, longer rung
      m_gaAliveCount = (int)MathMax(2, m_gaAliveCount / 2);
      m_gaRung++;
      m_gaCandPos = 0;
      return;
     }
   //--- final rung done -> this generation is complete
   m_gaGen++;
   if(m_gaGen < m_gaMaxGen)
     {
      GaBreedNextGeneration();
      m_gaRung = 0;
      m_gaCandPos = 0;
      m_gaSeedIdx = 0;
      m_gaAliveCount = m_gaPopSize;
      for(int a = 0; a < m_gaPopSize; a++)
         m_gaAlive[a] = a;
      return;
     }
   //--- search exhausted -> set up the single full retrain on the winning params, which deploys normally
   m_evalMode = false;
   m_indicatorTuner.Unflatten(m_gaBestParams);
   ReInitADIndicators(m_indicatorsPtr);
   MathSrand(GetTickCount()); // fresh (non-fixed) init for the real deployed model
   BuildFreshTopology();
   dError = -1;
   dUndefine = 0;
   dForecast = 0;
   dOosForecast = 0;
   m_oosSamples = 0;
   dOosError = -1;
   m_gaFinalRetrain = true;
   Print(ID + ": auto-tune search done - retraining the winner to convergence for deployment");
  }
#endif // WARRIOR_AIBASE_AUTOTUNE_MQH