Warrior_EA/Expert/AIBase/ChartUI.mqh

//+------------------------------------------------------------------+
//|                                                       Warrior_EA |
//|                                                     AnimateDread |
//|                                                                  |
//| Chart arrows, arrow persistence/restore, status panel, chart cle|
//|                                                                  |
//| PARTIAL IMPLEMENTATION FILE - not standalone.                     |
//| This holds CExpertSignalAIBase method BODIES only. The class      |
//| declaration lives in Expert\ExpertSignalAIBase.mqh, which         |
//| #includes this file at the bottom, after the declaration. Do not  |
//| include it anywhere else and do not compile it on its own.        |
//|                                                                  |
//| Split out purely to make the 8216-line original navigable; the    |
//| code inside was moved verbatim, not rewritten.                    |
//+------------------------------------------------------------------+
#ifndef WARRIOR_AIBASE_CHARTUI_MQH
#define WARRIOR_AIBASE_CHARTUI_MQH
//+------------------------------------------------------------------+
//| Erase this model's drawn arrows, its .arrows sidecar, and any     |
//| deferred restore still in flight.                                 |
//|                                                                  |
//| Must run on EVERY path that discards or replaces the trained      |
//| weights, because arrows outlive the model that drew them in two   |
//| separate ways:                                                    |
//|                                                                  |
//|  1. The chart objects themselves are saved in the CHART, not in   |
//|     the sidecar, so they survive a remove/re-add, a recompile, a  |
//|     terminal restart and a fresh deploy regardless of what        |
//|     happens to any file on disk.                                  |
//|  2. SaveChartSignals() rebuilds the sidecar by SCANNING the chart |
//|     for SIG_ARROW_PREFIX objects. So a stale arrow left on screen |
//|     is not merely cosmetic - the next save writes it back out     |
//|     under the CURRENT model's filename, laundering a dead model's |
//|     calls into the new model's history where nothing can tell     |
//|     them apart afterwards.                                        |
//|                                                                  |
//| Keyed on m_fileName + FILE_COMMON, NOT m_activeFileName/flags:    |
//| SaveChartSignals always writes the shared, chart-owned file and   |
//| refuses to run in a backtest, so the same tester guard is applied |
//| here rather than the m_activeFileCommon indirection the weight    |
//| files use.                                                        |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::ClearPersistedChartSignals(const string reason)
  {
   if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD))
      return;
//--- Cancel any deferred restore FIRST. Otherwise its already-parsed queue keeps drawing the old
//--- model's arrows from the next timer slice onward - after the file was deleted - and the next
//--- save writes them straight back out. See AdvanceChartSignalRestore.
   m_arrowRestorePending = false;
   m_arrowRestoreIndex = 0;
   ArrayFree(m_arrowRestoreTime);
   ArrayFree(m_arrowRestoreCode);
   ArrayFree(m_arrowRestorePrice);
   ArrayFree(m_arrowRestoreTf);
   string arrows = m_fileName + ".arrows";
   ResetLastError();
   if(FileIsExist(arrows, FILE_COMMON) && !FileDelete(arrows, FILE_COMMON))
      Print(ID + ": ERROR - failed to delete " + arrows + ", error " + IntegerToString(GetLastError()));
   ResetLastError();
//--- Clear them off the chart too, so the reset is visibly complete instead of leaving stale arrows
//--- on screen until the next restart. Namespaced delete - user drawings are untouched.
   ObjectsDeleteAll(0, SIG_ARROW_PREFIX);
   ChartRedraw(0);
   PrintVerbose(ID + ": cleared drawn signal arrows and " + arrows + " - " + reason);
  }
//+------------------------------------------------------------------+
//| Persist the currently-drawn directional arrows to a sidecar file  |
//| so they survive an EA remove/re-add/recompile without a retrain.  |
//| Captures time, arrow code (217 Buy / 218 Sell), price, and hide   |
//| state; chart-only (no persistent chart in a backtest). Skips      |
//| writing when there is nothing to save so it never clobbers a good |
//| file with an empty one (e.g. a transient no-arrow state).         |
//|                                                                    |
//| pruneChartObjects=false says "the caller is about to bulk-delete   |
//| every arrow anyway" - see the prune block at the end for why that  |
//| distinction is what keeps OnDeinit inside its budget.              |
//|                                                                    |
//| RETURNS true when the sidecar provably holds the current arrow set |
//| - including the "chart has no arrows" case, where there is nothing |
//| to lose. Only a genuine write failure returns false. The shutdown  |
//| path (PersistAndClearChartSignals) keys the chart purge on this:   |
//| arrows may only be removed from the chart once they exist on disk, |
//| because for a CONVERGED model the chart objects are the ONLY copy  |
//| (nothing redraws them - PruneDirectionalClusters runs per training |
//| era, and a deployed model has no eras left).                       |
//+------------------------------------------------------------------+
bool CExpertSignalAIBase::SaveChartSignals(bool pruneChartObjects = true)
  {
   if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD))
      return true;
   long   times[];
   int    codes[];
   double prices[];
   long   tfs[];
   int    n = 0;
   int    total = ObjectsTotal(0);
   //--- Pre-size the arrays ONCE to an upper bound (all chart objects), then fill and trim. The old
   //--- ArrayResize(n+1)-per-arrow reallocated the buffers on every match -> O(n^2), which on a chart
   //--- with thousands of accumulated arrows took long enough to blow MT5's OnDeinit time budget and
   //--- force-terminate shutdown before the chart got cleaned. This keeps the whole scan O(n).
   //--- room for the chart's own arrows PLUS any still sitting in the deferred-restore queue (see below)
   int queueRemaining = m_arrowRestorePending ? (ArraySize(m_arrowRestoreTime) - m_arrowRestoreIndex) : 0;
   if(queueRemaining < 0)
      queueRemaining = 0;
   ArrayResize(times, total + queueRemaining);
   ArrayResize(codes, total + queueRemaining);
   ArrayResize(prices, total + queueRemaining);
   ArrayResize(tfs, total + queueRemaining);
   for(int i = 0; i < total; i++)
     {
      string nm = ObjectName(0, i);
      if(StringFind(nm, SIG_ARROW_PREFIX) != 0)
         continue;
      if(ObjectGetInteger(0, nm, OBJPROP_TYPE) != OBJ_ARROW)
         continue;
      times[n]  = (long)ObjectGetInteger(0, nm, OBJPROP_TIME);
      codes[n]  = (int)ObjectGetInteger(0, nm, OBJPROP_ARROWCODE);
      prices[n] = ObjectGetDouble(0, nm, OBJPROP_PRICE);
      tfs[n]    = g_signalsVisible ? OBJ_ALL_PERIODS : OBJ_NO_PERIODS;
      n++;
     }
   //--- CRITICAL for init/deinit sync: a deferred restore may still be in flight (see
   //--- AdvanceChartSignalRestore), meaning part of the saved history has NOT been drawn yet and so is
   //--- invisible to the chart scan above. Saving at that moment - which OnDeinit does, via
   //--- ShutdownChartCleanup - would persist only the drawn subset and silently truncate the user's
   //--- history. Fold the undrawn remainder of the queue back in so the file always represents
   //--- chart-plus-queue, whatever point the restore reached. These entries are by construction not yet on
   //--- the chart, so no de-duplication is needed (and a same-bar duplicate would merely be overwritten on
   //--- the next load, since the object name is derived from the bar time).
   for(int q = m_arrowRestoreIndex; q < m_arrowRestoreIndex + queueRemaining; q++)
     {
      times[n]  = (long)m_arrowRestoreTime[q];
      codes[n]  = m_arrowRestoreCode[q];
      prices[n] = m_arrowRestorePrice[q];
      tfs[n]    = g_signalsVisible ? OBJ_ALL_PERIODS : OBJ_NO_PERIODS;
      n++;
     }
   if(n <= 0)
      return true; // nothing to save - leave any existing file intact (and nothing on the chart to lose)
   //--- Keep only the MAX_PERSISTED_ARROWS most recent, by TIME (see the define's comment for why scan
   //--- order can't be used). Selection is threshold-based rather than a full parallel-array sort: sort a
   //--- COPY of the times, read off the cut-off, then compact the four arrays in one pass. Compaction is
   //--- safe in place because the write index never runs ahead of the read index.
   long pruned[];
   int prunedCount = 0;
   if(n > MAX_PERSISTED_ARROWS)
     {
      long sortedTimes[];
      ArrayResize(sortedTimes, n);
      ArrayCopy(sortedTimes, times, 0, 0, n);
      ArraySort(sortedTimes); // ascending
      long cutoff = sortedTimes[n - MAX_PERSISTED_ARROWS];
      ArrayResize(pruned, n);
      int w = 0;
      for(int i = 0; i < n; i++)
        {
         //--- ">= cutoff" can match more than the cap when several arrows share the cut-off timestamp;
         //--- the w < cap test keeps the kept set at exactly the cap in that case.
         if(times[i] >= cutoff && w < MAX_PERSISTED_ARROWS)
           {
            times[w]  = times[i];
            codes[w]  = codes[i];
            prices[w] = prices[i];
            tfs[w]    = tfs[i];
            w++;
           }
         else
            pruned[prunedCount++] = times[i];
        }
      n = w;
     }
   //--- Staged through a temp file + atomic rename (System\AtomicFile.mqh), so an interrupted write
   //--- keeps the previous arrow set instead of truncating it, and never blocks LoadChartSignals()
   //--- on another instance.
   string arrowsName = m_fileName + ".arrows";
   string arrowsTmpName = "";
   int handle = AtomicWriteBegin(arrowsName, FILE_COMMON, arrowsTmpName);
   if(handle == INVALID_HANDLE)
     {
      //--- Was a silent `return`. Silence here is what made the "arrows stay on the chart" report
      //--- unresolvable for three sessions: every failure mode of this function looked identical to
      //--- success from the outside, and the only externally visible symptom (a littered chart) is two
      //--- calls downstream. Name the file and the error so the next occurrence is one grep.
      Print(ID + ": ERROR - could not open " + arrowsTmpName + " to persist " + IntegerToString(n) +
            " chart signal arrows, error " + IntegerToString(GetLastError()) +
            " - the arrows stay on the chart (they are the only copy).");
      return false;
     }
   //--- Results ARE checked now: the pruning step below is documented as running only after a
   //--- successful write, but nothing used to verify that, so a failed/partial write still deleted
   //--- the chart objects and lost those arrows in both places.
   bool ok = (FileWriteInteger(handle, 0x57534152) > 0); // 'WSAR' magic
   if(ok && FileWriteInteger(handle, n) <= 0)
      ok = false;
   for(int i = 0; ok && i < n; i++)
     {
      if(FileWriteLong(handle, times[i]) <= 0 || FileWriteInteger(handle, codes[i]) <= 0 ||
         FileWriteDouble(handle, prices[i]) <= 0 || FileWriteLong(handle, tfs[i]) <= 0)
         ok = false;
     }
   if(!AtomicWriteEnd(handle, arrowsName, arrowsTmpName, FILE_COMMON, ok, __FUNCTION__))
      return false; // AtomicWriteEnd already logged which half failed
   //--- Only after the file is safely written: drop the pruned arrows from the chart too, so the chart and
   //--- the sidecar stay in agreement and the object count stops growing without bound. Done last so a
   //--- failed write above leaves both the file AND the chart untouched.
   //---
   //--- SKIPPED ON SHUTDOWN (pruneChartObjects == false), and that is not an optimisation - it is the fix
   //--- for a reproducible "Abnormal termination" that left three of four charts littered on 2026-08-01.
   //--- This loop is O(prunedCount) ObjectDelete calls, and ObjectDelete is itself O(objects) on a
   //--- crowded chart, so it degrades to O(n^2). That was tolerable while the model called a direction on
   //--- ~6% of bars; after the triple-barrier relabel the models call on 83-94% of bars, the chart carries
   //--- many thousands of arrows, and the loop blew MetaTrader's OnDeinit budget - measured at 4.2-4.7 s
   //--- against a PAI chart that survived in 3.75 s. MetaTrader then force-terminated OnDeinit BEFORE
   //--- PurgeChart() ran, so the arrows stayed on screen: the slow tidy-up starved the fast one.
   //--- At shutdown the caller purges every arrow with a single bulk ObjectsDeleteAll immediately after
   //--- this returns, so deleting them one at a time first is work whose only effect is to prevent the
   //--- bulk delete from happening at all.
   if(prunedCount > 0 && pruneChartObjects)
     {
      for(int i = 0; i < prunedCount; i++)
         ObjectDelete(0, SIG_ARROW_PREFIX + TimeToString((datetime)pruned[i]));
      ChartRedraw(0);
      PrintVerbose(ID + ": pruned " + IntegerToString(prunedCount) + " old chart signal arrows (keeping the " +
                   IntegerToString(MAX_PERSISTED_ARROWS) + " most recent)");
     }
   m_lastArrowsSaved = n;
   return true;
  }
//+------------------------------------------------------------------+
//| Shutdown path: hand the drawn arrows over to the save/restore     |
//| mechanism and take them OFF the chart, rather than abandoning     |
//| them there for the next EA (or the user) to find.                 |
//|                                                                   |
//| Reported 2026-08-01: the panel and status label disappeared on    |
//| removal but the signal arrows stayed. Leaving them is wrong in    |
//| both directions - a removed EA that keeps drawing on the chart    |
//| looks broken, and the leftovers are not inert: SaveChartSignals() |
//| rebuilds the sidecar by SCANNING the chart, so the next model to  |
//| attach adopts the dead one's calls as its own history (see        |
//| ClearPersistedChartSignals' header).                              |
//|                                                                   |
//| ORDER MATTERS, and it is write-then-clear, never the reverse. For |
//| a converged model the chart objects are the only copy of its      |
//| signal history: nothing redraws them, because the renderer        |
//| (PruneDirectionalClusters) runs once per TRAINING era and a       |
//| deployed model has none left. So the purge is conditional on the  |
//| sidecar write having actually succeeded - a chart that stays      |
//| littered because the disk write failed is a far better outcome    |
//| than one that is clean because the history was destroyed, and the |
//| failure now says so in the log instead of looking like this bug.  |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::PersistAndClearChartSignals(void)
  {
   if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD))
     {
      PurgeChart(); // tester charts are throwaway - nothing to persist, just leave nothing behind
      return;
     }
   m_lastArrowsSaved = 0;
   //--- pruneChartObjects=false: the purge below removes every arrow in one bulk call, so deleting the
   //--- over-cap ones individually first is pure cost - and at O(objects) per ObjectDelete it is the cost
   //--- that overran MT5's deinit budget on 2026-08-01 and skipped the cleanup entirely.
   bool saved = SaveChartSignals(false);
   if(!saved)
     {
      Print(ID + ": WARNING - could not persist the drawn signal arrows, so they are being LEFT on the chart "
            "rather than discarded (they are the only copy). Clear them with the panel's reset-weights, or "
            "fix the write error logged above.");
      ClearStatusLabel();
      return;
     }
   int removed = PurgeChart();
   Print(ID + ": chart signals - persisted " + IntegerToString(m_lastArrowsSaved) + " arrow(s) to " +
         m_fileName + ".arrows and cleared " + IntegerToString(removed) +
         " from the chart; they are restored automatically the next time this configuration is attached.");
  }
//+------------------------------------------------------------------+
//| Recreate the arrows saved by SaveChartSignals(). Runs at init so  |
//| a re-added / recompiled / restarted chart shows the deployed      |
//| model's signals again without retraining. Restores the saved hide |
//| state, so the show/hide toggle survives too.                      |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::LoadChartSignals(void)
  {
   if(MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_FORWARD))
      return;
//--- ARROWS BELONG TO A MODEL. No weights loaded for this configuration means this chart is starting from
//--- era 0, so it must start visually clean: a fresh run that inherits a previous model's arrows shows
//--- calls it never made, and the first save would then write them back out as its OWN history (this
//--- function's sidecar is rebuilt by SCANNING the chart - see SaveChartSignals). The fresh-topology
//--- branch in InitNeuralNetwork already clears them for the ordinary case; this is the same rule stated
//--- once, at the single point where arrows are brought back, so it holds no matter which of that
//--- function's several early exits actually ran.
   if(!m_modelLoadedFromDisk)
     {
      ClearPersistedChartSignals("no saved model for this configuration - starting with a clean chart");
      return;
     }
   string fn = m_fileName + ".arrows";
   if(!FileIsExist(fn, FILE_COMMON))
      return;
   //--- share flags: read-only, see CopySharedFile().
   int handle = FileOpen(fn, FILE_COMMON | FILE_BIN | FILE_READ | FILE_SHARE_READ | FILE_SHARE_WRITE);
   if(handle == INVALID_HANDLE)
      return;
   if(FileReadInteger(handle) != 0x57534152)
     {
      FileClose(handle);
      return;
     }
   int n = FileReadInteger(handle);
   //--- Sanity-cap a corrupt/garbage count so a bad header can never queue millions of object creations.
   if(n < 0 || n > MAX_RESTORED_ARROWS)
     {
      Print(ID + ": .arrows header declares " + IntegerToString(n) + " arrows - refusing to restore (limit " +
            IntegerToString(MAX_RESTORED_ARROWS) + "); the file looks corrupt. Chart signals will rebuild as the model runs.");
      FileClose(handle);
      return;
     }
   ArrayResize(m_arrowRestoreTime, n);
   ArrayResize(m_arrowRestoreCode, n);
   ArrayResize(m_arrowRestorePrice, n);
   ArrayResize(m_arrowRestoreTf, n);
   int parsed = 0;
   for(int i = 0; i < n && !FileIsEnding(handle); i++)
     {
      m_arrowRestoreTime[parsed]  = (datetime)FileReadLong(handle);
      m_arrowRestoreCode[parsed]  = FileReadInteger(handle);
      m_arrowRestorePrice[parsed] = FileReadDouble(handle);
      m_arrowRestoreTf[parsed]    = FileReadLong(handle);
      parsed++;
     }
   FileClose(handle);
   //--- Trim to what actually parsed (a truncated file yields fewer records than the header claimed).
   if(parsed != n)
     {
      ArrayResize(m_arrowRestoreTime, parsed);
      ArrayResize(m_arrowRestoreCode, parsed);
      ArrayResize(m_arrowRestorePrice, parsed);
      ArrayResize(m_arrowRestoreTf, parsed);
     }
   //--- Apply the retention cap on the way IN as well, so a pre-cap file (one run had 2896 arrows) is
   //--- trimmed on its first load instead of waiting for the next save - otherwise the very restore this
   //--- cap exists to bound would still rebuild every one of them. Identical selection rule to
   //--- SaveChartSignals: most recent by TIME, since file order is chart-scan order, i.e. arbitrary.
   if(parsed > MAX_PERSISTED_ARROWS)
     {
      long sortedTimes[];
      ArrayResize(sortedTimes, parsed);
      for(int i = 0; i < parsed; i++)
         sortedTimes[i] = (long)m_arrowRestoreTime[i];
      ArraySort(sortedTimes); // ascending
      long cutoff = sortedTimes[parsed - MAX_PERSISTED_ARROWS];
      int w = 0;
      for(int i = 0; i < parsed; i++)
         if((long)m_arrowRestoreTime[i] >= cutoff && w < MAX_PERSISTED_ARROWS)
           {
            m_arrowRestoreTime[w]  = m_arrowRestoreTime[i];
            m_arrowRestoreCode[w]  = m_arrowRestoreCode[i];
            m_arrowRestorePrice[w] = m_arrowRestorePrice[i];
            m_arrowRestoreTf[w]    = m_arrowRestoreTf[i];
            w++;
           }
      Print(ID + ": .arrows sidecar holds " + IntegerToString(parsed) + " arrows - restoring the " +
            IntegerToString(w) + " most recent (retention cap " + IntegerToString(MAX_PERSISTED_ARROWS) +
            "); the file is rewritten capped on the next save.");
      parsed = w;
      ArrayResize(m_arrowRestoreTime, parsed);
      ArrayResize(m_arrowRestoreCode, parsed);
      ArrayResize(m_arrowRestorePrice, parsed);
      ArrayResize(m_arrowRestoreTf, parsed);
     }
   m_arrowRestoreIndex = 0;
   m_arrowRestorePending = (parsed > 0);
   m_arrowRestoreStartMs = GetTickCount();
   //--- Deliberately does NOT draw anything here - see AdvanceChartSignalRestore's declaration comment for
   //--- why this must not block OnInit. Drawing starts on the next timer/tick slice.
   if(m_arrowRestorePending)
      PrintVerbose(ID + ": queued " + IntegerToString(parsed) + " chart signal arrows for progressive restore");
  }
//+------------------------------------------------------------------+
//| See the declaration comment - draws a time-boxed slice of the     |
//| arrows queued by LoadChartSignals(), so a chart carrying          |
//| thousands of them fills in progressively instead of freezing the  |
//| terminal during init.                                             |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::AdvanceChartSignalRestore(void)
  {
   if(!m_arrowRestorePending)
      return;
   int total = ArraySize(m_arrowRestoreTime);
   uint sliceStart = GetTickCount();
   int drawnThisSlice = 0;
   while(m_arrowRestoreIndex < total)
     {
      //--- check the budget every 64 objects rather than every one: GetTickCount() itself is not free, and
      //--- at ~microseconds per object a per-iteration clock read would dominate the work being measured.
      if(drawnThisSlice > 0 && (drawnThisSlice & 63) == 0 && GetTickCount() - sliceStart >= ARROW_RESTORE_BUDGET_MS)
         break;
      datetime t   = m_arrowRestoreTime[m_arrowRestoreIndex];
      int    code  = m_arrowRestoreCode[m_arrowRestoreIndex];
      double price = m_arrowRestorePrice[m_arrowRestoreIndex];
      long   tf    = m_arrowRestoreTf[m_arrowRestoreIndex];
      m_arrowRestoreIndex++;
      drawnThisSlice++;
      string nm = SIG_ARROW_PREFIX + TimeToString(t);
      //--- Deliberately NO ObjectFind() pre-check. ObjectFind scans the chart's object list, so calling it
      //--- per arrow made the restore O(n^2) - with ~2900 arrows that is ~4.2M name comparisons, which is
      //--- what actually froze the terminal (observed 2026-07-26; previously masked because the filename
      //--- bug made LoadChartSignals always miss its file). ObjectCreate already returns false when the
      //--- name exists, and re-applying the properties below is harmless and exactly what a refresh does.
      ObjectCreate(0, nm, OBJ_ARROW, 0, t, 0);
      ObjectSetDouble(0, nm, OBJPROP_PRICE, price);
      ObjectSetInteger(0, nm, OBJPROP_ARROWCODE, code);
      ObjectSetInteger(0, nm, OBJPROP_COLOR, code == 217 ? clrBlue : clrRed);
      ObjectSetInteger(0, nm, OBJPROP_ANCHOR, code == 217 ? ANCHOR_TOP : ANCHOR_BOTTOM);
      ObjectSetInteger(0, nm, OBJPROP_TIMEFRAMES, tf); // restore the saved show/hide state
     }
   ChartRedraw(0);
   if(m_arrowRestoreIndex < total)
      return; // more slices to come
   //--- done - release the buffers and report once
   m_arrowRestorePending = false;
   Print(ID + ": restored " + IntegerToString(total) + " chart signal arrows in " +
         IntegerToString((int)(GetTickCount() - m_arrowRestoreStartMs)) + " ms (progressive, non-blocking)");
   ArrayFree(m_arrowRestoreTime);
   ArrayFree(m_arrowRestoreCode);
   ArrayFree(m_arrowRestorePrice);
   ArrayFree(m_arrowRestoreTf);
  }
//+------------------------------------------------------------------+
//| Time-boxed slice of the per-bar re-inference queued by            |
//| StartChartSignalRescan(). Same doctrine as                        |
//| AdvanceChartSignalRestore() above: one feedForward per bar over up |
//| to SIGNAL_RESCAN_LOOKBACK_BARS bars is real, non-trivial compute   |
//| (not a cheap object-property write like the arrow restore), so    |
//| the slice is bounded by elapsed time rather than a fixed count -   |
//| checked every bar since a single feedForward can itself approach   |
//| the whole budget on a slow (CPU-DLL, no OpenCL) backend.           |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::AdvanceChartSignalRescan(void)
  {
   if(!m_rescanPending)
      return;
   uint sliceStart = GetTickCount();
   EnsureShadowNet();
   CNet *deployNet = (CheckPointer(m_shadowNet) != POINTER_INVALID) ? m_shadowNet : Net;
   while(m_rescanIndex < m_rescanHi)
     {
      if(GetTickCount() - sliceStart >= ARROW_RESTORE_BUDGET_MS)
         break;
      int i = m_rescanIndex;
      m_rescanIndex++;
      //--- Same builder the live signal uses, so a restored arrow means what the deployed model would
      //--- actually have said on that bar rather than a lookalike assembled by a parallel loop.
      if(!BuildFeatureWindow(i))
         continue;
      deployNet.feedForward(TempData);
      deployNet.getResults(TempData);
      if(m_outputNeuronsCount == 1)
         m_arrowSignalCache[i] = TempData[0];
      else
        {
         double rawSignal = ApplyClassificationSoftmax();
         switch(DoubleToSignal(rawSignal))
           {
            case Buy:
               m_rescanRawBuy++;
               break;
            case Sell:
               m_rescanRawSell++;
               break;
            default:
               m_rescanRawNeutral++;
               break;
           }
         m_arrowSignalCache[i] = AdjustedSignalFromSoftmax();
        }
     }
   if(m_rescanIndex < m_rescanHi)
      return; // more slices to come
   //--- done - render the declustered set once, same as the old blocking pass did
   PruneDirectionalClusters(m_rescanBarsNow);
   SaveChartSignals();
   m_rescanPending = false;
   //--- Tally what the model actually called BEFORE decluttering, so an empty-looking chart is
   //--- distinguishable in the log between "the model called Neutral almost everywhere" (a real,
   //--- if extreme, calibration outcome) and "arrows were computed but never rendered" (a bug) -
   //--- both look identical on the chart otherwise.
   int buyCount = 0, sellCount = 0, neutralCount = 0, unscored = 0;
   for(int k = 0; k < m_rescanHi; k++)
     {
      if(m_arrowSignalCache[k] == -2.0)
        {
         unscored++;
         continue;
        }
      switch(DoubleToSignal(m_arrowSignalCache[k]))
        {
         case Buy:
            buyCount++;
            break;
         case Sell:
            sellCount++;
            break;
         default:
            neutralCount++;
            break;
        }
     }
   Print(ID + ": rescanned " + IntegerToString(m_rescanBarsNow) + " bars against the deployed model and rebuilt the chart signal arrows in " +
         IntegerToString((int)(GetTickCount() - m_rescanStartMs)) + " ms (progressive, non-blocking) - post-calibration calls (pre-decluttering): " +
         IntegerToString(buyCount) + " Buy, " + IntegerToString(sellCount) + " Sell, " + IntegerToString(neutralCount) + " Neutral, " +
         IntegerToString(unscored) + " unscored" +
         (m_outputNeuronsCount == 3 ? " | RAW network argmax (before logit-prior correction): " +
          IntegerToString(m_rescanRawBuy) + " Buy, " + IntegerToString(m_rescanRawSell) + " Sell, " +
          IntegerToString(m_rescanRawNeutral) + " Neutral" : ""));
  }
//+------------------------------------------------------------------+
//| Compounded/persistent accuracy line for the simple panels.       |
//| See the declaration comment (class body) for why this exists.    |
//+------------------------------------------------------------------+
string CExpertSignalAIBase::ComputeCompoundedAccuracyLine(void)
  {
   if(m_cumIsTotal <= 0 && m_cumOosTotal <= 0)
     {
      //--- These counters are LIFETIME and persisted (they are what the panel presents as the product's
      //--- accuracy), so they only advance on bars the model CALLED. They sit at zero in two very
      //--- different situations and the panel must not describe both as a wait.
      //--- This previously read "measuring..." forever, and the 2026-08-01 report was "they seem to be
      //--- signaling but the label stays stuck" - at the time an auto-tune GA was running every era as a
      //--- throwaway candidate, so nothing was counted for hours while the chart filled with arrows. That
      //--- particular gap is closed at the source: tuning is now a filter pass that finishes in seconds
      //--- (see TuneIndicatorsByFilter), so a run reaches real, counted eras almost immediately.
      //--- Before era 1 nothing has been scored, so "measuring..." is honest; after eras have run, zero
      //--- directional calls is a FINDING (a Neutral collapse) and must not read as a wait.
      if(m_eraCount > 0)
         return "Buy/Sell calls correct:  no directional calls yet";
      return "Buy/Sell calls correct:  measuring...";
     }
   //--- OUT-OF-SAMPLE only on the panel. The in-sample figure is the model graded on bars it trained on,
   //--- so it is always the flattering one and is never what forward trading delivers - showing both
   //--- invites a buyer to read the higher number as the product's accuracy. The IS/OOS pair is genuinely
   //--- useful (the gap between them is the over-fitting read), so it is not discarded, just moved to the
   //--- journal under DebuggingMode where diagnosing it belongs. Falls back to in-sample, explicitly
   //--- labelled, only in the brief window before the first out-of-sample era has been scored.
   if(m_cumOosTotal > 0)
     {
      //--- Labelled "lifetime" deliberately. These counters are monotonic and never reset per era (only
      //--- on a fresh model / reset-weights), and they persist across restarts via .stats, so this is the
      //--- average over EVERY era ever trained - not the current era's performance. At era 200+ a single
      //--- new era shifts it by a fraction of a percent, so a model that started badly and has since
      //--- recovered still reads low here.
      return "Buy/Sell calls correct:  " +
             IntegerToString((int)MathRound(m_cumOosCorrect * 100.0 / m_cumOosTotal)) +
             "%   (unseen data, lifetime)";
     }
   return "Buy/Sell calls correct:  " +
          IntegerToString((int)MathRound(m_cumIsCorrect * 100.0 / m_cumIsTotal)) +
          "%   (training data so far)";
  }
//+------------------------------------------------------------------+
//| See the declaration comment (class body) for why this exists.    |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::UpdateTrainingStatusLabel(const string &progressLine, double neuron0, double neuron1, double neuron2, double signalValue, bool forceRefresh)
  {
   // Cache regardless of whether this particular call actually redraws below - see
   // m_lastDisplayNeuron0's declaration comment for why the era-end forced refresh needs these.
   m_lastDisplayNeuron0 = neuron0;
   m_lastDisplayNeuron1 = neuron1;
   m_lastDisplayNeuron2 = neuron2;
   m_lastDisplaySignal = signalValue;
   // Throttle: SetStatusLabel()'s ChartRedraw() is real work that gets slower as more chart objects
   // accumulate over a long backtest - calling it every single bar across three passes (instead of
   // pass 1 alone, the original frequency) is what actually stalled a run for 1.5+ hours without
   // finishing era 0, not the shuffle itself. ~5 updates/sec is still visually live. forceRefresh
   // bypasses this - see this method's declaration comment for why the era-end call needs to.
   // 2026-07-30 raised 200 -> 400. The second half of the same responsiveness complaint the training
   // chunk budget addresses, and on a long-running chart the larger half: ChartRedraw() repaints the
   // WHOLE chart, so its cost scales with the accumulated signal arrows, and firing it five times a
   // second is what makes dragging the panel stutter rather than any single call being slow. Nothing
   // on the simple panel changes fast enough to need 5 Hz - era and accuracy move once per era, and
   // the progress percentage is smooth at 2.5 Hz. The era-end call passes forceRefresh and bypasses
   // this entirely, so no state transition is ever delayed by it.
   const uint STATUS_LABEL_THROTTLE_MS = 400;
   uint nowTick = GetTickCount();
   if(!forceRefresh && m_lastStatusLabelUpdateTick != 0 && nowTick - m_lastStatusLabelUpdateTick < STATUS_LABEL_THROTTLE_MS)
      return;
   m_lastStatusLabelUpdateTick = nowTick;
   //--- Simple panel (default, VerboseMode off): only what a non-technical user can act on - how far
   //--- training has progressed, how reliable its LIVE Buy/Sell calls are (hit-rate = live precision:
   //--- the same bars the deployed EA would actually trade, so it is a true forward-trading expectation,
   //--- not the Neutral-inflated headline accuracy), and what it is signalling right now. Every raw NN
   //--- internal (in-sample error, MSE, softmax neurons, confusion counts, recall) is diagnostic and is
   //--- shown only under VerboseMode in the detailed block below.
   if(!VerboseMode)
     {
      ENUM_SIGNAL curSig = DoubleToSignal(signalValue);
      string sigPlain = (curSig == Buy) ? "Buy" : (curSig == Sell) ? "Sell" : "Neutral (no trade)";
      int progressPct = (int)((double)(m_isTrainCursor + 1.0) / MathMax(m_isTrainQueueCount, 1) * 100.0);
      //--- Compounded, persistent DIRECTIONAL win-rate (Buy/Sell only, Neutral excluded - see
      //--- m_cumIsCorrect) - a stable number that keeps refining across eras and restarts, not the noisy
      //--- per-era metric that sat at "measuring", and not the Neutral-inflated all-class rate. IS =
      //--- in-sample (bars it trained on), OOS = out-of-sample (held-out; the honest generalization read -
      //--- the gap between them shows over/under-fitting).
      string accLine = ComputeCompoundedAccuracyLine();
      //--- The checkpoint/deploy internals - best score so far, whether it clears the deploy gate,
      //--- eras-since-best, plateau-ladder stage - used to occupy two more panel lines. They are
      //--- development diagnostics: a buyer cannot act on "ladder stage 2/3", and the panel is the
      //--- product's face. Emitted to the journal under DebuggingMode instead, so nothing is lost while
      //--- working on the EA - but from the ERA-END summary (Training.mqh), not from here: this method is
      //--- throttled to ~5 calls/second, so a Print() at this site would put five lines per second into
      //--- the journal, and the state it reports only changes once per era anyway. Deliberately not
      //--- folded into VerboseMode either: that switch is the supported power-user view, and this is
      //--- internal state whose meaning shifts as the selection rule does (it described a per-class
      //--- recall floor that no longer decides anything).
      //--- Three lines, and each one answers a question an owner actually has: what is it doing and how
      //--- far along, how well has it been calling direction, and what is it saying right now.
      string simple = StringFormat(
                         DisplayName() + "  -  learning  (era %d, %d%%)\n" +
                         "%s\n" +
                         "Current signal:  %s",
                         m_eraCount, progressPct, accLine, sigPlain);
      SetStatusLabel(simple);
      return;
     }
   // Recall/precision-by-class and the continual-learning OOS sim are still tracked (used for the
   // convergence gate elsewhere) but dropped from the on-chart status text - it made the panel too
   // tall/wordy for a one-line-per-metric display; the counts below are enough at a glance.
   string classLine = StringFormat(
                          "Predicted -> Buy: %d  Sell: %d  Neutral: %d\n" +
                          "Actual    -> Buy: %d  Sell: %d  Neutral: %d",
                          m_countBuySignals, m_countSellSignals, m_countNeutralSignals,
                          m_trueBuyCount, m_trueSellCount, m_trueNeutralCount
                       );
   string oosErrStr = (m_oosSamples > 0 ? DoubleToString(dOosError, 2) : "N/A");
   string s;
   if(m_outputNeuronsCount == 1)
      s = StringFormat(
             ID + " : Study -> Era %d\n" +
             "%s\n" +
             "IS %d%% Acc: %.2f%%  MSE: %.2f  AvgErr: %.2f\n" +
             "OOS %d%% Acc: %.2f%%  Mismatch: %s  Samples: %d\n" +
             "Signal Neuron: %.5f\n" +
             "Forecast: %s -> %.2f\n" +
             "%s",
             m_eraCount, progressLine,
             100 - m_oosSplitPct, dForecast,
             dError,
             Net.getRecentAverageError(),
             m_oosSplitPct, dOosForecast, oosErrStr, m_oosSamples,
             neuron0,
             EnumToString(DoubleToSignal(signalValue)), signalValue,
             classLine
          );
   else
      if(m_outputNeuronsCount == 3)
        {
         //--- Headline OOS accuracy above blends in Neutral (usually the majority class, and "don't
         //--- trade" rather than a call that can be right or wrong the way Buy/Sell are) - a model can
         //--- score well on it while its actual Buy/Sell calls are unreliable. This is the number that
         //--- predicts trading performance, so it gets its own line rather than staying log-only.
         string oosDirRecall = "OOS recall  Buy: " + (m_lastBuyRecallPct < 0 ? "n/a" : IntegerToString(m_lastBuyRecallPct) + "%") +
                                "   Sell: " + (m_lastSellRecallPct < 0 ? "n/a" : IntegerToString(m_lastSellRecallPct) + "%");
         //--- Live-trade precision line: precision on ONLY the calls that clear the confidence floor under
         //--- the live/prior-corrected rule, with the fire count in parentheses. This is the metric that
         //--- matches forward trading - the deployed EA takes exactly these bars (see AdjustedSignalFromSoftmax).
         string oosLivePrec = "Live win rate   Buy: " + (m_lastBuyFiredPrecPct < 0 ? "n/a" : IntegerToString(m_lastBuyFiredPrecPct) + "%") +
                              " (" + IntegerToString(m_lastBuyFired) + ")   Sell: " + (m_lastSellFiredPrecPct < 0 ? "n/a" : IntegerToString(m_lastSellFiredPrecPct) + "%") +
                              " (" + IntegerToString(m_lastSellFired) + ")";
         s = StringFormat(
                ID + " : Study -> Era %d\n" +
                "%s\n" +
                "IS %d%% Acc: %.2f%%  MSE: %.2f  AvgErr: %.2f\n" +
                "OOS %d%% Acc: %.2f%%  Mismatch: %s  Samples: %d\n" +
                "%s\n" +
                "%s\n" +
                "Buy: %.5f  Sell: %.5f  Neutral: %.5f\n" +
                "Forecast: %s -> %.2f\n" +
                "%s",
                m_eraCount, progressLine,
                100 - m_oosSplitPct, dForecast,
                dError,
                Net.getRecentAverageError(),
                m_oosSplitPct, dOosForecast, oosErrStr, m_oosSamples,
                oosDirRecall,
                oosLivePrec,
                neuron0, neuron1, neuron2,
                EnumToString(DoubleToSignal(signalValue)), signalValue,
                classLine
             );
        }
      else
         s = "Invalid neuron count!";
   SetStatusLabel(s);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::DrawObject(datetime time, double signal, double high, double low)
  {
   double price = 0;
   int arrow = 0;
   color clr = 0;
   ENUM_ARROW_ANCHOR anch = ANCHOR_BOTTOM;
   switch(DoubleToSignal(signal))
     {
      case Buy:
         price = low;
         arrow = 217;
         clr = clrBlue;
         anch = ANCHOR_TOP;
         break;
      case Sell:
         price = high;
         arrow = 218;
         clr = clrRed;
         anch = ANCHOR_BOTTOM;
         break;
     }
   if(price == 0 || arrow == 0)
      return;
   string name = SIG_ARROW_PREFIX + TimeToString(time);
   //--- Deliberately NO ObjectFind() pre-check - same fix as AdvanceChartSignalRestore's ObjectCreate
   //--- call (see that method's comment): ObjectFind scans the ENTIRE chart object list, so calling it
   //--- once per drawn arrow makes PruneDirectionalClusters's era-end sweep O(n^2) in the arrow count -
   //--- exactly the pattern that froze the terminal once already (2026-07-26, arrow restore). This
   //--- sweep runs ONCE PER ERA, completely unchunked (no TRAIN_TIME_BUDGET_MS yield), so its cost
   //--- directly stalls the panel/chart for however long it takes - raising the training budget can't
   //--- fix that, only cutting this O(n^2) cost can. ObjectCreate already returns false harmlessly when
   //--- the name exists (ignored below, same as the restore path); re-applying the properties is exactly
   //--- what a refresh does regardless of whether the object is new or already there.
   ObjectCreate(0, name, OBJ_ARROW, 0, time, 0);
   ObjectSetDouble(0, name, OBJPROP_PRICE, price);
   ObjectSetInteger(0, name, OBJPROP_ARROWCODE, arrow);
   ObjectSetInteger(0, name, OBJPROP_COLOR, clr);
   ObjectSetInteger(0, name, OBJPROP_ANCHOR, anch);
   ObjectSetInteger(0, name, OBJPROP_TIMEFRAMES, g_signalsVisible ? OBJ_ALL_PERIODS : OBJ_NO_PERIODS);
   ObjectSetString(0, name, OBJPROP_TOOLTIP, EnumToString(DoubleToSignal(signal)) + " " + DoubleToString(signal, 5));
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::DeleteObject(datetime time)
  {
   string name = SIG_ARROW_PREFIX + TimeToString(time);
   //--- No ObjectFind() pre-check either - same O(n^2)-avoidance reasoning as DrawObject() above.
   //--- ObjectDelete() already returns false (silently, no dialog) when the name doesn't exist.
   ObjectDelete(0, name);
  }
//+------------------------------------------------------------------+
//| End-of-era renderer + non-max suppression over m_arrowSignalCache.|
//| This is the SOLE place directional arrows are drawn during        |
//| training (the scan passes only RECORD predictions into the cache -|
//| they never draw), so the chart only ever shows the declustered    |
//| set, never the raw mid-era clusters. Walks strictly oldest ->     |
//| newest (bar index high -> low). For each scored bar:              |
//|  - Neutral: delete any arrow.                                     |
//|  - Same-direction within m_signalClusterWindow of the previous    |
//|    SEEN same-direction bar: suppress (collapses a whole contiguous |
//|    run to its first bar - advance last-seen either way).          |
//|  - Cross-direction within the window of the last KEPT opposite    |
//|    signal (flicker at one turn zone; real opposite pivots are a   |
//|    leg apart): keep only the higher-confidence side.              |
//|  - Otherwise: KEEP -> draw the arrow.                             |
//| Order-independent by design: pass 2 records IS bars in SHUFFLED   |
//| order, so this cache pass is the only place NMS/rendering can be  |
//| applied correctly. See m_signalClusterWindow.                     |
//+------------------------------------------------------------------+
void CExpertSignalAIBase::PruneDirectionalClusters(int bars)
  {
   if(m_signalClusterWindow <= 0)                // NMS off: the passes drew inline, nothing to render here
      return;
   int cacheSize = ArraySize(m_arrowSignalCache);
   if(cacheSize <= 0)
      return;
   int hi = MathMin(bars, cacheSize);
   int lastBuyIdx = -1;         // last SEEN bar per direction (same-direction contiguous collapse)
   int lastSellIdx = -1;
   int keptIdx = -1;            // last KEPT bar of EITHER direction (cross-direction resolution)
   double keptConf = 0.0;
   ENUM_SIGNAL keptDir = Neutral;
   for(int idx = hi - 1; idx >= 0; idx--)       // high index = oldest bar -> iterate forward in time
     {
      double sv = m_arrowSignalCache[idx];
      if(sv == -2.0)                             // not scored this era: leave whatever's there
         continue;
      datetime t = m_Time.GetData(idx);
      ENUM_SIGNAL sig = DoubleToSignal(sv);
      if(sig != Buy && sig != Sell)              // scored Neutral: ensure no arrow
        {
         DeleteObject(t);
         continue;
        }
      // 1) Same-direction contiguous collapse: suppress if within the window of the previous SEEN
      // same-direction bar; advance last-seen either way so a whole run collapses to its first bar.
      int lastSame = (sig == Buy) ? lastBuyIdx : lastSellIdx;
      bool sameContinuation = (lastSame >= 0 && (lastSame - idx) <= m_signalClusterWindow);
      if(sig == Buy)
         lastBuyIdx = idx;
      else
         lastSellIdx = idx;
      if(sameContinuation)
        {
         DeleteObject(t);
         continue;
        }
      // 2) Cross-direction resolution: a fresh cluster within the window of the last KEPT opposite
      // signal is flicker at one turn zone (real opposite pivots are a whole leg apart) - keep only the
      // higher-confidence side. Confidence = |signed signal| = the winning softmax probability.
      double conf = MathAbs(sv);
      if(keptIdx >= 0 && keptDir != sig && (keptIdx - idx) <= m_signalClusterWindow)
        {
         if(conf > keptConf)
            DeleteObject(m_Time.GetData(keptIdx));   // this bar is stronger: drop the weaker opposite arrow
         else
           {
            DeleteObject(t);                          // the kept opposite is stronger: suppress this bar
            continue;
           }
        }
      // KEEP: this is the render step - the arrow is drawn here, not by the scan passes.
      DrawObject(t, sv, m_High.GetData(idx), m_Low.GetData(idx));
      keptIdx = idx;                                  // first bar of a fresh, window-clear cluster
      keptConf = conf;
      keptDir = sig;
     }
  }
//+------------------------------------------------------------------+
//| PurgeChart - Removes this EA's own visual objects from the chart. |
//| Returns how many signal arrows it actually removed.               |
//+------------------------------------------------------------------+
int CExpertSignalAIBase::PurgeChart(void)
  {
   long chartID = 0;
//--- Delete ONLY what this EA created: our namespaced signal arrows plus the status-label objects. A
//--- blanket ObjectsDeleteAll(chartID) here (the old behavior) also wiped the user's own manual chart
//--- drawings and any other indicator's objects - unacceptable on a client's chart. Called from the
//--- destructor (clean removal); NOT from InitIndicators anymore, so arrows survive re-inits.
   int removed = ObjectsDeleteAll(chartID, SIG_ARROW_PREFIX);
   if(removed < 0)
      removed = 0;
//--- VERIFY, don't assume. This one call was the entire cleanup, its return value was discarded, and
//--- nothing downstream ever looked at the chart again - so "the arrows are still there after removal"
//--- was indistinguishable from "the arrows were never there", which is exactly why that report survived
//--- three sessions. The sweep below is a typed scan (OBJ_ARROW only, so it walks a handful of objects on
//--- a normal chart, not the whole object list) and finds nothing whenever the bulk delete did its job,
//--- which is the overwhelmingly common case. When it DOES find something, it finishes the job and says
//--- so - naming the failure instead of leaving it to be re-reported as a visual symptom.
//--- Names are collected before any deletion: deleting while enumerating by index would renumber the very
//--- list being walked, and object commands are queued on the chart rather than applied inline.
   string leftovers[];
   int found = 0;
   int arrowsTotal = ObjectsTotal(chartID, -1, OBJ_ARROW);
   if(arrowsTotal > 0)
     {
      ArrayResize(leftovers, arrowsTotal);
      for(int i = 0; i < arrowsTotal; i++)
        {
         string nm = ObjectName(chartID, i, -1, OBJ_ARROW);
         if(StringFind(nm, SIG_ARROW_PREFIX) == 0)
            leftovers[found++] = nm;
        }
     }
   for(int i = 0; i < found; i++)
      ObjectDelete(chartID, leftovers[i]);
   if(found > 0)
      Print(ID + ": WARNING - ObjectsDeleteAll(\"" + SIG_ARROW_PREFIX + "\") reported " + IntegerToString(removed) +
            " removed but left " + IntegerToString(found) + " signal arrow(s) on the chart; deleted them by name. " +
            "The bulk prefix delete is not doing its job on this terminal build - that is the root cause of any " +
            "'arrows stay on the chart' report, not the shutdown ordering.");
   ClearStatusLabel();
   ChartRedraw(chartID);
   return removed + found;
  }
#endif // WARRIOR_AIBASE_CHARTUI_MQH