Warrior_EA/Expert/AIBase/FeatureScreen.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//| The MI evidence screen - "does this feature vector predict this  |
//| label at all", answered without training, topology or            |
//| convergence, and the label-alignment lookahead scan.              |
//+------------------------------------------------------------------+
#ifndef WARRIOR_AIBASE_FEATURESCREEN_MQH
#define WARRIOR_AIBASE_FEATURESCREEN_MQH
//+------------------------------------------------------------------+
//| SPLIT OUT OF AutoTune.mqh (2026-08-23): that file was two          |
//| responsibilities - SEARCH (TuneIndicatorsByFilter, coordinate-     |
//| descent over indicator settings) and MEASUREMENT (this file, which |
//| never mutates an indicator parameter). They share the MI engine    |
//| that stays in AutoTune.mqh (FeatureColumnMI/BuildMiSample/          |
//| ScoreMiSample) because BOTH consume it - a genuine shared           |
//| dependency, not a reason to keep the two responsibilities in one   |
//| file. TuneIndicatorsAndTrain(), the entry point that decides which |
//| of the two branches a given model runs this process, also stays in |
//| AutoTune.mqh: it is the coordinator, not a member of either side.  |
//|                                                                  |
//| Still body-only method definitions of CExpertSignalAIBase, exactly |
//| like every other Expert\AIBase\*.mqh file - MQL5 has no partial   |
//| classes, so this split is a FILE-organisation move (legibility,    |
//| SRP-per-file) rather than a coupling reduction. See                |
//| project_oop_module_pattern for what a REAL decoupling of this      |
//| domain would need (a view + adapter, MQL5's single-inheritance     |
//| tax) - not attempted here because nothing downstream yet needs one.|
//+------------------------------------------------------------------+
//+------------------------------------------------------------------+
//| "Do these features predict this label at all?" - answered        |
//| without training, topology or convergence, so unlike every       |
//| accuracy number in this codebase it cannot be confounded by an   |
//| optimizer or an objective.                                       |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::ReportFeatureLabelInformation(void)
  {
//--- THE LATCH IS AT THE END, NOT HERE, AND THAT WAS A BUG WORTH A SESSION. It used to be set on
//--- entry, so a COLD start - where the label cache is allocated before it is filled, BuildMiSample
//--- finds no row carrying a label, and returns 0 - disabled the screen for the whole run on the
//--- first attempt. The first ensemble member then propagated it to g_ensembleChartMiReportDone and
//--- silenced every other member on the chart. Warm starts never showed it.
//--- PERMUTATION TEST, done properly.
   double cols[];
   int labels[];
   int nSample = BuildMiSample(cols, labels);
   //--- NOT ENOUGH SAMPLE IS NOT A MEASUREMENT. Reporting it as one printed "-1.00000 nats over 0
   //--- permutations" beside a plausible-looking "strongest single feature 0.05979" - which was a
   //--- STALE m_miBestColumn left by an earlier scoring call, not a number measured here. Say what
   //--- happened, keep the flag clear so the next era retries, and print nothing that reads as data.
   if(nSample < MI_MIN_SAMPLES)
     {
      m_miReportAttempts++;
      bool giveUp = (m_miReportAttempts >= MI_REPORT_MAX_ATTEMPTS);
      if(giveUp)
         m_miReportDone = true;
      Print(ID + StringFormat(": feature keep-screen DEFERRED - the usable sample holds %d row(s)"
                              " and %d are required (label cache %d bars, %d of them carrying a"
                              " resolved label). This is the cold-start ordering case: the cache is"
                              " allocated before it is filled. Attempt %d of %d%s",
                              nSample, MI_MIN_SAMPLES, m_labelCacheBars, LabelCacheResolvedCount(),
                              m_miReportAttempts, MI_REPORT_MAX_ATTEMPTS,
                              giveUp ? " - GIVING UP for this run; no keep-screen will be measured."
                                     : " - retrying next era."));
      return;
     }
   double observed = ScoreMiSample(cols, labels, nSample, false);
   double signalBestCol = m_miBestColumn;
   double labelEntropy  = m_miLabelEntropy;
   //--- OBSERVED PER-COLUMN MI, copied before the null loop overwrites m_miColumn on every draw.
   double obsCol[];
   int    colN = (observed >= 0.0) ? ArraySize(m_miColumn) : 0;
   int    colAtLeast[];
   if(colN > 0)
     {
      ArrayResize(obsCol, colN);
      ArrayResize(colAtLeast, colN);
      ArrayInitialize(colAtLeast, 0);
      for(int f = 0; f < colN; f++)
         obsCol[f] = m_miColumn[f];
     }
   double floorSum = 0.0, floorSumSq = 0.0, floorBestColSum = 0.0;
   int draws = 0, atLeastMean = 0, atLeastBestCol = 0;
   uint tPerm = GetTickCount();
   for(int s = 0; observed >= 0.0 && s < MI_NOISE_PERMUTATIONS; s++)
     {
      //--- A few hundred full MI scorings, and nothing below can start until they finish. Abandon the
      //--- test outright rather than truncate it: fewer draws does not make a smaller null, it makes a
      //--- WRONG one (p shifts toward significance), and this null's spread prices the lookahead margin below.
      if(ShutdownRequested())
        {
         draws = 0;
         break;
        }
      double sc = ScoreMiSample(cols, labels, nSample, true);
      if(sc < 0.0)
         continue;
      floorSum += sc;
      floorSumSq += sc * sc;
      floorBestColSum += m_miBestColumn;
      if(sc >= observed)
         atLeastMean++;
      //--- The MAX over columns is compared against the null distribution OF THE MAX, which corrects for
      //--- testing 26 features at once by construction - no Bonferroni needed, and far less conservative.
      if(m_miBestColumn >= signalBestCol)
         atLeastBestCol++;
      //--- PER-COLUMN null, accumulated from the same draws. Each column gets its OWN null
      //--- distribution here, which is what a per-column p-value needs - distinct from the
      //--- null-of-the-max above, which answers the single family-wise question "is the strongest
      //--- column real" and is far too conservative to select a SET with.
      for(int f = 0; f < colN && f < ArraySize(m_miColumn); f++)
         if(m_miColumn[f] >= obsCol[f])
            colAtLeast[f]++;
      draws++;
     }
//--- Left the null early because the program is going away: everything from here on either prints a
//--- number derived from `draws` or starts another scan. Neither is worth a millisecond of the teardown
//--- budget.
   if(ShutdownRequested())
      return;
   double floorMean    = (draws > 0) ? floorSum / draws : -1.0;
   double floorVar     = (draws > 1) ? MathMax(0.0, floorSumSq / draws - floorMean * floorMean) : 0.0;
   double floorSd      = MathSqrt(floorVar * (draws > 1 ? (double)draws / (draws - 1) : 1.0));
   double floorBestCol = (draws > 0) ? floorBestColSum / draws : -1.0;
   double pMean    = PermutationPValue(atLeastMean, draws);
   double pBestCol = PermutationPValue(atLeastBestCol, draws);
//--- Two SEPARATE questions, because at these sample sizes a small p can accompany a worthless
//--- effect. (1) Is it real - the p-values. Both are printed; neither is collapsed into a verdict
//--- that hides the other.
   double excessShare = (labelEntropy > 1e-9 && floorMean >= 0.0)
                        ? 100.0 * (observed - floorMean) / labelEntropy : 0.0;
   string verdict = (draws > 0 && pMean <= 0.05)
                    ? "above the noise floor - a real association"
                    : "AT THE NOISE FLOOR - indistinguishable from shuffled labels";
//--- Name the feature vector this was measured on. Stating the width and the panel's presence makes
//--- that mismatch visible in the log instead of requiring a timestamp comparison.
   string vecNote = StringFormat("%d features/bar, cross-asset %s", m_neuronsCount,
                                 m_crossAsset.IsReady()
                                 ? "PRESENT"
                                 : "ABSENT (reference symbols unsynchronised - these numbers describe "
                                   "a NARROWER vector than training will use)");
   Print(ID + StringFormat(": feature/label information - %.5f nats/feature vs a shuffled-label null of "
                           "%.5f +/- %.5f over %d permutations, p=%.4f; strongest single feature %.5f vs "
                           "%.5f (null max, p=%.4f); excess is %.2f%% of the label's %.3f nats of entropy "
                           "(%d samples %d bars apart = %d independent blocks over a %.1f-bar mean label "
                           "resolution, %.1fs) [%s]. %s.",
                           observed, floorMean, floorSd, draws, pMean,
                           signalBestCol, floorBestCol, pBestCol, excessShare, labelEntropy,
                           nSample, m_miStrideBars, m_miNullBlocks, MeanLabelLifespan(),
                           (GetTickCount() - tPerm) / 1000.0, vecNote, verdict));
//--- WHICH columns carry it, not just whether one does. Report-only: nothing prunes on this yet, and
//--- the number it produces is exactly what decides whether pruning is worth doing - a cut that keeps
//--- 3 of 52 columns is not a feature set, and one that keeps 45 is not worth a fingerprint re-key.
   if(draws > 0 && colN > 0)
     {
      double pcol[];
      ArrayResize(pcol, colN);
      for(int f = 0; f < colN; f++)
         pcol[f] = PermutationPValue(colAtLeast[f], draws);
      double sorted[];
      ArrayResize(sorted, colN);
      ArrayCopy(sorted, pcol);
      ArraySort(sorted);
      //--- Benjamini-Hochberg: the largest k with p_(k) <= (k/m)*q; keep every column at or below it.
      int keep = 0;
      for(int k = 1; k <= colN; k++)
         if(sorted[k - 1] <= (double)k / colN * MI_KEEP_FDR_Q)
            keep = k;
      double cutP = (keep > 0) ? sorted[keep - 1] : 0.0;
      //--- THE KEPT SET AS A HEX BITMASK, column 0 = bit 0 of the first nibble-group. Emitted so the
      //--- masks of two charts can be compared by eye and by grep. That comparison is the open
      //--- question a mask cannot be built without: identical masks across the fleet mean ONE
      //--- fleet-wide mask keeps every chart in a single pool group, while divergent masks would
      //--- split six charts into six groups of one - and pooling is the only thing currently holding
      //--- the FX charts above the capacity floor, so a prune could cost more than it buys.
      string maskHex = "";
      for(int b = 0; b < colN; b += 4)
        {
         int nib = 0;
         for(int q = 0; q < 4 && b + q < colN; q++)
            if(pcol[b + q] <= cutP && keep > 0)
               nib |= (1 << q);
         maskHex += StringFormat("%X", nib);
        }
      //--- The whole point of the exercise, stated in the units the capacity budget uses.
      int widthNow  = m_historyBars * colN;
      int widthKept = m_historyBars * MathMax(keep, 1);
      double effN   = EffectiveSampleSize((double)EstimatedInSampleBarsRaw());
      Print(ID + StringFormat(": feature keep-screen (REPORT ONLY, nothing prunes yet) - %d of %d"
                              " columns clear Benjamini-Hochberg at q=%.2f (cut p<=%.4f). Input width"
                              " would go %d -> %d (%d bars x columns), and the first-layer budget"
                              " effN/(width+1) %.1f -> %.1f against a %d-wide floor.%s",
                              keep, colN, MI_KEEP_FDR_Q, cutP, widthNow, widthKept, m_historyBars,
                              effN / (widthNow + 1.0), effN / (widthKept + 1.0), FIRST_LAYER_MIN_WIDTH,
                              (m_crossAsset.IsReady() ? ""
                               : " CAVEAT: measured with cross-asset ABSENT, so these columns are a"
                                 " SUBSET of what training uses - a mask built from this would not"
                                 " cover the cross-asset block at all.")));
      Print(ID + ": feature keep-mask " + maskHex + " (hex, column 0 = low bit; compare across charts"
            " before building a real mask - divergent masks would fragment the training pool).");
     }
//--- LATCH ONLY ON A SAMPLE WORTH KEEPING. A thin sample still gets measured and printed - the
//--- numbers are real for what they are - but it does not close the question, so a later era with
//--- more resolved labels can supersede it. Bounded by the same attempt budget as the deferral path,
//--- so a chart that never reaches the target still ends up with its best available answer.
   int goodSample = (int)(MI_SAMPLE_BARS * MI_GOOD_SAMPLE_FRACTION);
   if(nSample < goodSample && m_miReportAttempts < MI_REPORT_MAX_ATTEMPTS - 1)
     {
      m_miReportAttempts++;
      Print(ID + StringFormat(": that screen is UNDERPOWERED and does NOT close the question - %d"
                              " samples against a %d target (%d of %d cached bars carry a resolved"
                              " label). Columns kept at this sample size track POWER as much as"
                              " information. Re-measuring on a later era (attempt %d of %d).",
                              nSample, goodSample, LabelCacheResolvedCount(), m_labelCacheBars,
                              m_miReportAttempts, MI_REPORT_MAX_ATTEMPTS));
      return;
     }
//--- THE MEASUREMENT EXISTS NOW, so latch. Everything below is commentary and a positive control,
//--- each with its own teardown escape - none of it changes whether this era measured the screen.
   m_miReportDone = true;
//--- POWER, stated up front. Saying so prevents the opposite error to the one this replaced -
//--- reading "not significant" as "no signal" when it means "not enough independent data to tell".
   if(m_miNullBlocks > 0 && m_miNullBlocks < 30)
      Print(ID + StringFormat(": NOTE - only %d independent label blocks in this sample (%.1f-bar mean "
                              "label resolution, %d-bar sampling stride). The rows overlap heavily, so "
                              "this test has little power: treat a non-significant result here as 'not "
                              "enough independent history to answer', not as 'no signal'. More history "
                              "is what would settle it.", m_miNullBlocks, MeanLabelLifespan(),
                              m_miStrideBars));
//--- Stated every time, not only on a bad result: this measure is MARGINAL and PER-BAR, while the network
//--- reads m_historyBars bars at once. It can therefore only ever prove that signal EXISTS, never that it
//--- does not - an interaction across features or across time is invisible to it by construction. Said
//--- out loud so a floor-level reading is not over-read into "this instrument is unpredictable".
   if(!(draws > 0 && pMean <= 0.05))
      Print(ID + ": NOTE - that measure is marginal (one feature at a time) and per-bar, whereas the "
            "network sees " + IntegerToString((int)m_historyBars) + " bars jointly. A floor-level reading "
            "rules out a simple per-feature edge; it cannot rule out one that only exists in combination "
            "or across time. It does mean no per-feature indicator retuning will help.");
   if(observed < 0.0)
      return;
//--- POSITIVE CONTROL. A floor reading is therefore worthless until the instrument is shown to
//--- respond to a signal that is KNOWN to be there. This one is free: the label of a NEIGHBOURING
//--- sample row.
   if(ShutdownRequested())
      return;
   double controlMi = -1.0, decorrMi = -1.0;
   int controlBars = 1;
   int decorrBars = (int)MathMax(1.0, MeanLabelLifespan() / 4.0);
     {
      //--- Rebuilt rather than reused because the permutation loop above destroyed the honest label
      //--- ordering, and controlling against a shuffled array would measure the floor twice.
      double c0[], cK[];
      int l0[], lK[];
      int n0 = BuildMiSample(c0, l0);
      if(n0 >= MI_MIN_SAMPLES)
        {
         int offs[2];
         offs[0] = controlBars;
         offs[1] = decorrBars;
         for(int oi = 0; oi < 2; oi++)
           {
            int nK = BuildMiSample(cK, lK, offs[oi]);
            //--- Both builds are padded by the SAME fixed amount, so they enumerate the same bars with
            //--- the same stride and row k of one is row k of the other. Sized from what actually came
            //--- back, never from the caller's count.
            int nc = MathMin(n0, nK);
            if(nc < MI_MIN_SAMPLES)
               continue;
            double neighbourLabel[];
            int selfLabels[];
            ArrayResize(neighbourLabel, nc);
            ArrayResize(selfLabels, nc);
            for(int k = 0; k < nc; k++)
              {
               selfLabels[k]     = l0[k];
               neighbourLabel[k] = (double)lK[k];
              }
            double v = FeatureColumnMI(neighbourLabel, selfLabels, nc);
            if(oi == 0)
               controlMi = v;
            else
               decorrMi = v;
           }
        }
     }
   Print(ID + StringFormat(": MI positive control - the ADJACENT bar's label (mean resolution %.1f bars) "
                           "scores %.5f nats against the ~%.5f noise floor; by a quarter of that (%d bars) "
                           "it is already down to %.5f, which is how fast this target decorrelates. %s",
                           MeanLabelLifespan(),
                           controlMi, floorMean, decorrBars, decorrMi,
                           (controlMi > floorMean * 5.0)
                           ? "The estimator detects a known association on this exact data, so a "
                             "floor-level reading above is a real finding and not a broken measurement."
                           : "WARNING - the estimator FAILED to detect an association that must be there. "
                             "Every mutual-information figure above is void; fix this before drawing any "
                             "conclusion from them."));
//--- ALIGNMENT SCAN. Both destroy the information before any topology sees it, and both look
//--- identical in every accuracy number this EA prints - which is exactly why four different
//--- architectures all landed on the same precision.
   int offsets[] = { -5, -3, -2, -1, 0, 1, 2, 3, 5 };
   string profile = "";
   double atZero = -1.0, worstFuture = -1.0, farPast = -1.0;
   int worstFutureK = 0;
   for(int oi = 0; oi < ArraySize(offsets); oi++)
     {
      //--- Nine sample builds. A partial profile cannot be read - the lookahead test compares the k<0
      //--- side against k=0 and both must exist - so a stop abandons the scan rather than printing a row
      //--- with holes in it that would look like a null result at the missing offsets.
      if(ShutdownRequested())
         return;
      double oc[];
      int ol[];
      int on = BuildMiSample(oc, ol, offsets[oi]);
      double os = (on >= MI_MIN_SAMPLES) ? ScoreMiSample(oc, ol, on, false) : -1.0;
      profile += StringFormat("%s%+d:%.5f", (oi > 0 ? " " : ""), offsets[oi], os);
      if(offsets[oi] == 0)
         atZero = os;
      else
         if(offsets[oi] < 0 && os > worstFuture)
           {
            worstFuture = os;
            worstFutureK = offsets[oi];
           }
         else
            if(offsets[oi] > 0)
               farPast = os; // offsets ascend, so this ends on the largest k
     }
//--- A MARGIN, not a bare comparison. Every one of these offsets is an estimate with the same noise
//--- as the headline statistic, so "k=-3 came out above k=0" is meaningless when the gap is smaller
//--- than the null's own spread.
   double lookaheadMargin = 3.0 * floorSd;
   string alignVerdict;
   if(worstFuture > atZero + lookaheadMargin)
      alignVerdict = StringFormat(" | LOOKAHEAD - k=%d (a label whose swing leg opens AFTER these "
                                  "features exist) scores %.5f against %.5f at k=0, clearing the %.5f "
                                  "margin (3 sd of the null). The features can only score there by "
                                  "containing future information. Fix that before trusting any accuracy "
                                  "number this EA prints.", worstFutureK, worstFuture, atZero, lookaheadMargin);
   else
      alignVerdict = StringFormat(" | clean: no future label (k<0) beats k=0, so there is no lookahead. "
                                  "The rise on the k>0 side is expected - those legs are already under "
                                  "way, so the features hold part of the answer - and its size is the "
                                  "finding: %.5f at k=+5 against %.5f at k=0, i.e. ~%.1fx more is "
                                  "knowable 5 bars into a leg than at the entry the model actually trades.",
                                  farPast, atZero, (atZero > 1e-9 ? farPast / atZero : 0.0));
   Print(ID + ": MI label-alignment scan (label from bar i+k; higher index = OLDER bar, so k<0 is the "
         "future) - " + profile + alignVerdict);
  }
#endif // WARRIOR_AIBASE_FEATURESCREEN_MQH