forked from animatedread/Warrior_EA
6c2959dc32
A cold fleet start sizes every model BEFORE any chart has published a pool
file, so the first layer is budgeted as if the chart trains alone and then
pinned to .cfg. This is not a rare race - it is what happens EVERY time the
feature layout changes, because that invalidates the pool and forces a wipe.
Correcting it by hand needs a two-phase start: run the fleet to fill the pool,
stop, wipe the weights while KEEPING the pool, restart so derivation sees it.
That is not something an unattended fleet can do for itself, and getting it
wrong is silent - the models simply stay narrow.
TuneIndicatorsAndTrain now notices that the pool has appeared and re-derives
once, reusing ResetWeights() - the existing tested path that re-measures all
four sizes, rebuilds and rewrites the .cfg. No second copy of that logic.
Bounded on every axis that could make it a loop:
- once per model (the flag is set BEFORE the reset, because ResetWeights
zeroes m_eraCount and the model would otherwise re-qualify forever)
- only while era <= CAPACITY_RESIZE_MAX_ERA, so the discarded eras are worth
nothing
- only on CAPACITY_RESIZE_MIN_GROWTH real growth
- only if the recomputed width actually differs; if it does not, the check
settles itself rather than re-running the census every era
Safe against the one thing that would make it self-defeating: the derived width
is NOT part of BuildModelFingerprint, so a model that resizes does not leave
the pool it resized for.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
669 lines
35 KiB
MQL5
669 lines
35 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| Warrior_EA |
|
|
//| AnimateDread |
|
|
//| |
|
|
//| Filter-based indicator auto-tuner (mutual information scoring). |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef WARRIOR_AIBASE_AUTOTUNE_MQH
|
|
#define WARRIOR_AIBASE_AUTOTUNE_MQH
|
|
//--- ONCE-PER-CHART gate for the MI diagnostic suite on a multi-member ensemble. The first member
|
|
//--- to reach it runs it; the rest log one line and skip. Solo charts are untouched.
|
|
bool g_ensembleChartMiReportDone = false;
|
|
//--- ONCE-PER-CHART share of the indicator auto-tune SWEEP on a multi-member ensemble, same doctrine as
|
|
//--- the MI gate above: the sweep scores candidate indicator settings by feature/label MI, and every
|
|
//--- ensemble member holds identical indicators, identical cached features and identical labels, so all
|
|
//--- N sweeps are the same deterministic calculation (verified 2026-08-16 on SP500 H4: four members,
|
|
//--- byte-identical scores, spans and selection p). Worse, the sweep ends in the full MI diagnostic
|
|
//--- suite (ReportFeatureLabelInformation at its tail), which the MI gate above never intercepts on the
|
|
//--- sweep path - so each duplicate sweep also duplicated the ~200-draw permutation nulls, the slowest
|
|
//--- single block of "getting ready". The first member runs the sweep and publishes its outcome here;
|
|
//--- the rest apply the outcome (install the winner, or keep the configured settings the sweep restored)
|
|
//--- and skip both the sweep and the report. Same caveat as the MI gate: any winner ADOPTION is made by
|
|
//--- the donor and applied to every member via the flattened settings below, which is the consistent
|
|
//--- choice - members training on divergent feature vectors would not be an ensemble. Solo charts are
|
|
//--- untouched.
|
|
bool g_ensembleChartTuneDone = false;
|
|
bool g_ensembleChartTuneInstalled = false; // did the donor's sweep clear the family-wise gate and install?
|
|
double g_ensembleChartTuneSettings[]; // CADIndicatorTuner::Flatten() of the donor's final settings
|
|
//--- The genetic + successive-halving helpers that used to live here (GaRungEras, GaExtract,
|
|
//--- GaStore, GaMutate, GaRandomCandidate, GaBlockCrossover, GaSortAliveByScoreDesc,
|
|
//--- GaBreedNextGeneration) were deleted on 2026-08-01 together with the search they served.
|
|
//+------------------------------------------------------------------+
|
|
//| MUTUAL INFORMATION between one cached feature column and the |
|
|
//| swing label, in nats, over a sample of in-sample bars. |
|
|
//+------------------------------------------------------------------+
|
|
double CExpertSignalAIBase::FeatureColumnMI(const double &vals[], const int &labels[], int n)
|
|
{
|
|
if(n < MI_MIN_SAMPLES)
|
|
return 0.0;
|
|
double sorted[];
|
|
ArrayResize(sorted, n);
|
|
ArrayCopy(sorted, vals, 0, 0, n);
|
|
ArraySort(sorted);
|
|
//--- A column that never varies carries no information; short-circuit so the log below is never
|
|
//--- reached with a degenerate single-bin histogram.
|
|
if(sorted[0] == sorted[n - 1])
|
|
return 0.0;
|
|
int joint[]; ArrayResize(joint, MI_BINS * 3); ArrayInitialize(joint, 0);
|
|
int px[]; ArrayResize(px, MI_BINS); ArrayInitialize(px, 0);
|
|
int py[]; ArrayResize(py, 3); ArrayInitialize(py, 0);
|
|
for(int i = 0; i < n; i++)
|
|
{
|
|
//--- rank via binary search on the sorted copy; ties land in the same bin, which is correct
|
|
int lo = 0, hi = n - 1, rank = 0;
|
|
while(lo <= hi)
|
|
{
|
|
int mid = (lo + hi) / 2;
|
|
if(sorted[mid] < vals[i])
|
|
{
|
|
rank = mid + 1;
|
|
lo = mid + 1;
|
|
}
|
|
else
|
|
hi = mid - 1;
|
|
}
|
|
int bx = (int)((double)rank * MI_BINS / n);
|
|
if(bx >= MI_BINS)
|
|
bx = MI_BINS - 1;
|
|
int by = labels[i];
|
|
if(by < 0 || by > 2)
|
|
continue;
|
|
joint[bx * 3 + by]++;
|
|
px[bx]++;
|
|
py[by]++;
|
|
}
|
|
double mi = 0.0;
|
|
for(int b = 0; b < MI_BINS; b++)
|
|
{
|
|
if(px[b] <= 0)
|
|
continue;
|
|
for(int c = 0; c < 3; c++)
|
|
{
|
|
int j = joint[b * 3 + c];
|
|
if(j <= 0 || py[c] <= 0)
|
|
continue;
|
|
double pxy = (double)j / n;
|
|
mi += pxy * MathLog(pxy / (((double)px[b] / n) * ((double)py[c] / n)));
|
|
}
|
|
}
|
|
return (mi > 0.0) ? mi : 0.0;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Scores the CURRENT indicator parameters by how much the |
|
|
//| resulting feature vector tells us about the label - the mean |
|
|
//| per-column mutual information over a stratified sample of in- |
|
|
//| sample bars. |
|
|
//+------------------------------------------------------------------+
|
|
int CExpertSignalAIBase::BuildMiSample(double &cols[], int &labels[], int labelBarOffset = 0)
|
|
{
|
|
int bars = m_labelCacheBars;
|
|
if(bars <= 0 || m_neuronsCount <= 0)
|
|
{
|
|
//--- INSTRUMENTED 2026-08-26. This function has five distinct -1 exits and the caller can only
|
|
//--- see that the MI report collapsed to "-1.00000 nats over 0 permutations". On a COLD start
|
|
//--- the auto-tuner scored MI fine (0.00843) and the report seconds later returned -1 on the
|
|
//--- same chart, and three plausible explanations each failed to survive the log. Naming the
|
|
//--- exit costs one throttled line and ends the guessing.
|
|
TCLog("mi-sample-bars:" + ID,
|
|
StringFormat("%s: BuildMiSample abandoned - label cache holds %d bars and the feature"
|
|
" vector is %d wide; both must be positive.", ID, bars, m_neuronsCount));
|
|
return -1;
|
|
}
|
|
//--- Sample the IS region only. The OOS window must not influence which indicator settings ship, or
|
|
//--- the holdout has been used for selection and stops being a holdout at all.
|
|
int oosCutoff = (int)(MathMax(0, MathMin(100, m_oosSplitPct)) / 100.0
|
|
* MathMax(bars - MathMax(m_historyBars, 0), 0));
|
|
int lo = MathMax(oosCutoff, 2);
|
|
int hi = bars - MathMax(m_historyBars, 0) - 1;
|
|
//--- Keep the OFFSET label lookup inside the same bounds as the features, so a shifted scan
|
|
//--- measures a shift and not an edge effect. THE PAD IS FIXED, NOT |labelBarOffset|.
|
|
int shiftPad = MiShiftPad();
|
|
if(MathAbs(labelBarOffset) > shiftPad)
|
|
{
|
|
TCLog("mi-sample-shift:" + ID,
|
|
StringFormat("%s: BuildMiSample abandoned - caller asked for a %d-bar label shift but the"
|
|
" pad only covers %d.", ID, labelBarOffset, shiftPad));
|
|
return -1; // caller asked for a shift the pad does not cover
|
|
}
|
|
lo += shiftPad;
|
|
hi -= shiftPad;
|
|
if(hi - lo < MI_MIN_SAMPLES)
|
|
{
|
|
//--- THE LIKELY ONE, and the numbers say which term collapsed the window: the OOS cutoff, the
|
|
//--- history window, or a shift pad that scales with the measured label resolution.
|
|
TCLog("mi-sample-window:" + ID,
|
|
StringFormat("%s: BuildMiSample abandoned - usable IS window is %d rows (lo %d, hi %d) but"
|
|
" %d are required. bars=%d, oosSplit=%d%%, historyBars=%d, shiftPad=%d"
|
|
" (label resolution %d bars).",
|
|
ID, hi - lo, lo, hi, MI_MIN_SAMPLES, bars, m_oosSplitPct, m_historyBars,
|
|
shiftPad, LabelResolutionBars()));
|
|
return -1;
|
|
}
|
|
int stride = (int)MathMax(1, (hi - lo) / MI_SAMPLE_BARS);
|
|
//--- Published so the positive control can say how many BARS apart two sample rows are without
|
|
//--- recomputing this arithmetic at the call site, where it would silently drift out of agreement.
|
|
m_miStrideBars = stride;
|
|
int cap = (hi - lo) / stride + 1;
|
|
ArrayResize(cols, cap * m_neuronsCount);
|
|
ArrayResize(labels, cap);
|
|
int n = 0;
|
|
for(int i = lo; i < hi && n < cap; i += stride)
|
|
{
|
|
//--- Features come from bar i; the LABEL may be taken from a neighbouring bar (labelBarOffset != 0)
|
|
//--- so the caller can scan for a feature/label misalignment - see the alignment scan in
|
|
//--- ReportFeatureLabelInformation(). Both bars must carry a valid label for the row to count.
|
|
int li = i + labelBarOffset;
|
|
if(i >= ArraySize(m_labelCacheHasValue) || !m_labelCacheHasValue[i])
|
|
continue;
|
|
if(li < 0 || li >= ArraySize(m_labelCacheHasValue) || !m_labelCacheHasValue[li])
|
|
continue;
|
|
//--- BufferTempData(), NOT BufferTempDataCompute(). The Compute variant APPENDS the bar's
|
|
//--- features to TempData and never touches m_featureCache - only the caching wrapper writes
|
|
//--- that array.
|
|
TempData.Clear();
|
|
if(!BufferTempData(i) || TempData.Total() < m_neuronsCount)
|
|
continue;
|
|
for(int f = 0; f < m_neuronsCount; f++)
|
|
cols[n * m_neuronsCount + f] = TempData.At(f);
|
|
labels[n] = m_labelCacheBuy[li] ? 0 : (m_labelCacheSell[li] ? 1 : 2);
|
|
n++;
|
|
}
|
|
TempData.Clear();
|
|
return n;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| BLOCK PERMUTATION of a label column, in place. Fisher-Yates over |
|
|
//| BLOCK ORDER, within-block order untouched - that is what |
|
|
//| preserves the local dependence overlapping labels carry. A free |
|
|
//| shuffle would destroy it and report a null far too tight. |
|
|
//| |
|
|
//| THE RAGGED TAIL: when blockRows does not divide n the LAST block |
|
|
//| is short, and a version that wrote fixed-length blocks clamped |
|
|
//| its overrun to labels[n-1], duplicating one label and skewing |
|
|
//| every p-value toward significance. Each block now contributes |
|
|
//| exactly its own length, and the class-count invariance is CHECKED |
|
|
//| rather than asserted - a failed draw returns false so the caller |
|
|
//| skips it instead of poisoning the null. |
|
|
//+------------------------------------------------------------------+
|
|
bool BlockPermuteLabels(int &labels[], const int n, const int blockRows, int &blocksOut)
|
|
{
|
|
blocksOut = 0;
|
|
if(n <= 0 || blockRows <= 0 || ArraySize(labels) < n)
|
|
return false;
|
|
int rows = (blockRows > n) ? n : blockRows;
|
|
int blocks = (n + rows - 1) / rows;
|
|
blocksOut = blocks;
|
|
if(blocks <= 1)
|
|
return true; // one block: any permutation of it is itself
|
|
int before[3] = {0, 0, 0};
|
|
for(int i = 0; i < n; i++)
|
|
if(labels[i] >= 0 && labels[i] < 3)
|
|
before[labels[i]]++;
|
|
int order[];
|
|
ArrayResize(order, blocks);
|
|
for(int b = 0; b < blocks; b++)
|
|
order[b] = b;
|
|
for(int b = blocks - 1; b > 0; b--)
|
|
{
|
|
//--- ShuffleRandomIndex, not MathRand() modulo: with blockRows == 1 the block count equals
|
|
//--- the row count, which can exceed MathRand()'s 15-bit range - the same bias that was
|
|
//--- fixed in the pass-2 training queue shuffle.
|
|
int j = ShuffleRandomIndex(b + 1);
|
|
int t = order[b];
|
|
order[b] = order[j];
|
|
order[j] = t;
|
|
}
|
|
int shuffled[];
|
|
ArrayResize(shuffled, n);
|
|
int lastLen = n - (blocks - 1) * rows; // > 0 by construction of blocks
|
|
int w = 0;
|
|
for(int b = 0; b < blocks; b++)
|
|
{
|
|
int src = order[b] * rows;
|
|
int len = (order[b] == blocks - 1) ? lastLen : rows;
|
|
for(int q = 0; q < len; q++)
|
|
shuffled[w++] = labels[src + q];
|
|
}
|
|
//--- w == n unless the length arithmetic above is wrong, and a partial copy would leave the tail
|
|
//--- of labels[] holding the PREVIOUS draw - a null quietly correlated with the one before it.
|
|
if(w != n)
|
|
return false;
|
|
for(int i = 0; i < n; i++)
|
|
labels[i] = shuffled[i];
|
|
int after[3] = {0, 0, 0};
|
|
for(int i = 0; i < n; i++)
|
|
if(labels[i] >= 0 && labels[i] < 3)
|
|
after[labels[i]]++;
|
|
return (before[0] == after[0] && before[1] == after[1] && before[2] == after[2]);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Score an already-extracted sample. Split out from the extraction |
|
|
//| above so the permutation test can reuse ONE sample across every |
|
|
//| draw: feature extraction dominates the cost, and re-running it |
|
|
//| per shuffle is what would have made a few hundred permutations |
|
|
//| unaffordable. |
|
|
//+------------------------------------------------------------------+
|
|
double CExpertSignalAIBase::ScoreMiSample(const double &cols[], int &labels[], int n, bool shuffleLabels)
|
|
{
|
|
if(n < MI_MIN_SAMPLES)
|
|
return -1.0;
|
|
//--- PERMUTATION BASELINE. So a raw MI figure is uninterpretable on its own: 0.004 nats could be
|
|
//--- a genuine weak signal or could be pure noise.
|
|
//--- BLOCK permutation, not a free one, and the difference is the whole validity of the test. That
|
|
//--- was label autocorrelation leaking through an independence assumption, not an edge. It is Lopez
|
|
//--- de Prado ch.
|
|
if(shuffleLabels)
|
|
{
|
|
int blockRows = (m_miStrideBars > 0)
|
|
? (int)MathCeil(MathMax(MeanLabelLifespan(), 1.0) / m_miStrideBars) : 1;
|
|
if(blockRows < 1)
|
|
blockRows = 1;
|
|
if(blockRows > n)
|
|
blockRows = n;
|
|
if(!BlockPermuteLabels(labels, n, blockRows, m_miNullBlocks))
|
|
return -1.0;
|
|
}
|
|
//--- H(Y) over the sampled labels, so the caller can express MI as a fraction of the information the
|
|
//--- label actually contains. Computed AFTER any shuffle, which leaves it unchanged by construction:
|
|
//--- a permutation preserves the class counts. That invariance is now genuinely CHECKED, inside
|
|
//--- BlockPermuteLabels, which returns false if it fails - this comment used to claim the invariance was
|
|
//--- "itself a check on the shuffle" while nothing anywhere compared the counts, and the shuffle it
|
|
//--- was vouching for had in fact been breaking it whenever blockRows did not divide n.
|
|
int classCount[3] = {0, 0, 0};
|
|
for(int k = 0; k < n; k++)
|
|
classCount[labels[k]]++;
|
|
m_miLabelEntropy = 0.0;
|
|
for(int c = 0; c < 3; c++)
|
|
{
|
|
if(classCount[c] <= 0)
|
|
continue;
|
|
double pc = (double)classCount[c] / n;
|
|
m_miLabelEntropy -= pc * MathLog(pc);
|
|
}
|
|
double colVals[];
|
|
ArrayResize(colVals, n);
|
|
//--- Retained per column, not just summed - see m_miColumn's declaration. Costs one array write
|
|
//--- per column per call and nothing else; the MI itself was always computed here.
|
|
if(ArraySize(m_miColumn) != m_neuronsCount)
|
|
ArrayResize(m_miColumn, m_neuronsCount);
|
|
double total = 0.0;
|
|
m_miBestColumn = 0.0;
|
|
for(int f = 0; f < m_neuronsCount; f++)
|
|
{
|
|
for(int k = 0; k < n; k++)
|
|
colVals[k] = cols[k * m_neuronsCount + f];
|
|
double mi = FeatureColumnMI(colVals, labels, n);
|
|
m_miColumn[f] = mi;
|
|
total += mi;
|
|
if(mi > m_miBestColumn)
|
|
m_miBestColumn = mi;
|
|
}
|
|
return total / m_neuronsCount;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Extract + score in one call - the form the coordinate sweep uses, |
|
|
//| where each candidate genuinely needs a fresh extraction because |
|
|
//| the indicator settings (and therefore the features) just changed. |
|
|
//+------------------------------------------------------------------+
|
|
double CExpertSignalAIBase::ScoreCurrentParamsByMI(bool shuffleLabels = false)
|
|
{
|
|
double cols[];
|
|
int labels[];
|
|
int n = BuildMiSample(cols, labels);
|
|
if(n < MI_MIN_SAMPLES)
|
|
return -1.0;
|
|
return ScoreMiSample(cols, labels, n, shuffleLabels);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| FILTER-BASED indicator tuning. Replaced the genetic + |
|
|
//| successive- halving search on 2026-08-01. |
|
|
//+------------------------------------------------------------------+
|
|
void CExpertSignalAIBase::TuneIndicatorsByFilter(void)
|
|
{
|
|
double best[];
|
|
m_indicatorTuner.Flatten(best);
|
|
double bestScore = ScoreCurrentParamsByMI();
|
|
if(bestScore < 0.0)
|
|
{
|
|
Print(ID + ": auto-tune skipped - not enough labelled in-sample bars to score indicator settings");
|
|
return;
|
|
}
|
|
double startScore = bestScore;
|
|
int evaluated = 0;
|
|
uint t0 = GetTickCount();
|
|
//--- SPREAD OF THE CANDIDATE SCORES. Without it "no improvement" is ambiguous between two
|
|
//--- readings that want opposite responses: INERT (trial scores identical to the incumbent
|
|
//--- because the parameter change never reaches the scored features, so `sc > bestScore` can
|
|
//--- never fire) versus LIVE and genuinely finding nothing.
|
|
double candMin = DBL_MAX, candMax = -DBL_MAX;
|
|
int readyMin = INT_MAX;
|
|
//--- The configured settings, kept so a winner that fails the gate below can be handed back. best[] is
|
|
//--- mutated in place by the descent, so it cannot serve as the restore point.
|
|
double configured[];
|
|
ArrayCopy(configured, best);
|
|
for(int pass = 0; pass < MI_TUNE_PASSES; pass++)
|
|
{
|
|
bool improvedThisPass = false;
|
|
for(int p = 0; p < AD_TUNE_PARAM_COUNT; p++)
|
|
{
|
|
//--- skip parameters whose indicator is switched off - they cannot affect the feature vector
|
|
int owner = m_indicatorTuner.ParamOwner(p);
|
|
//--- OWNER 5 IS THE MA FEATURE and is the only one left: owners 0-4 (the AD/Wyckoff family)
|
|
//--- and 6-8 (RSI, MACD, Ichimoku) lost their feature groups on 2026-08-24. The owner
|
|
//--- NUMBERING is deliberately unchanged - CADIndicatorTuner's flat parameter array is
|
|
//--- persisted inside every .nnw, so renumbering it would silently discard the tuned MA
|
|
//--- period of every model already on disk (Unflatten refuses a size mismatch and falls
|
|
//--- back to constructor defaults). Dead owners simply never match now.
|
|
bool on = (owner == 5 && m_useMA);
|
|
if(!on)
|
|
continue;
|
|
double cands[];
|
|
int nc = m_indicatorTuner.ParamCandidates(p, cands);
|
|
double keep = best[p];
|
|
for(int c = 0; c < nc; c++)
|
|
{
|
|
//--- The longest uninterruptible stretch in the EA: every candidate re-creates handles,
|
|
//--- refreshes, and scores a full MI sample. Asked per candidate so a stop request costs at
|
|
//--- most one candidate rather than the rest of the descent - see ShutdownRequested().
|
|
if(ShutdownRequested())
|
|
{
|
|
//--- Hand the OPERATOR's settings back before leaving. best[] is mutated in place by
|
|
//--- the descent and the tuner object currently carries the LAST TRIAL's parameters,
|
|
//--- which nothing chose and which the .cfg would otherwise persist as if it had
|
|
//--- been selected.
|
|
m_indicatorTuner.Unflatten(configured);
|
|
PrintFormat("%s: auto-tune ABANDONED after %d candidates - stop requested. Configured"
|
|
" indicator settings restored; nothing installed.", ID, evaluated);
|
|
return;
|
|
}
|
|
if(cands[c] == keep)
|
|
continue; // already scored as the incumbent
|
|
double trial[];
|
|
ArrayCopy(trial, best);
|
|
trial[p] = cands[c];
|
|
m_indicatorTuner.Unflatten(trial);
|
|
ReInitTunableIndicators(m_indicatorsPtr); // also invalidates the feature cache (params changed)
|
|
//--- REFRESH, or the re-init changes nothing that the scorer can see. Without this the
|
|
//--- buffers still hold values copied from the PREVIOUS handle, so every candidate is
|
|
//--- scored on identical features.
|
|
RefreshData();
|
|
int ready = TunableBarsCalculated();
|
|
if(ready >= 0)
|
|
readyMin = (int)MathMin(readyMin, ready);
|
|
double sc = ScoreCurrentParamsByMI();
|
|
evaluated++;
|
|
if(sc >= 0.0)
|
|
{
|
|
candMin = MathMin(candMin, sc);
|
|
candMax = MathMax(candMax, sc);
|
|
}
|
|
if(sc > bestScore)
|
|
{
|
|
bestScore = sc;
|
|
keep = cands[c];
|
|
improvedThisPass = true;
|
|
}
|
|
}
|
|
best[p] = keep;
|
|
}
|
|
if(!improvedThisPass)
|
|
break; // coordinate descent has converged - further passes cannot move anything
|
|
}
|
|
//--- SELECTION GATE. bestScore is a MAXIMUM over every candidate scored, so it carries the same
|
|
//--- defect the barrier-geometry winner test and the lag profile were fixed for: the maximum of
|
|
//--- N draws from a null sits well above any single draw, and installing on "it beat the
|
|
//--- incumbent" alone crowns noise.
|
|
bool install = (bestScore > startScore);
|
|
double pFamily = 1.0;
|
|
int distinct = (int)MathMax(evaluated + 1, 1); // candidates scored, plus the incumbent
|
|
if(install)
|
|
{
|
|
double wc[];
|
|
int wl[];
|
|
int wn = BuildMiSample(wc, wl);
|
|
if(wn >= MI_MIN_SAMPLES)
|
|
{
|
|
double obs = ScoreMiSample(wc, wl, wn, false);
|
|
int atLeast = 0, draws = 0;
|
|
for(int s = 0; s < MI_NOISE_PERMUTATIONS; s++)
|
|
{
|
|
//--- A truncated null is not a smaller null, it is a WRONG one - fewer draws shifts p toward
|
|
//--- significance. So a stop here abandons the test entirely (draws stays 0, pFamily stays
|
|
//--- 1.0, install becomes false) rather than installing on a partial null.
|
|
if(ShutdownRequested())
|
|
{
|
|
draws = 0;
|
|
break;
|
|
}
|
|
double d = ScoreMiSample(wc, wl, wn, true);
|
|
if(d < 0.0)
|
|
continue;
|
|
if(d >= obs)
|
|
atLeast++;
|
|
draws++;
|
|
}
|
|
if(draws > 0)
|
|
{
|
|
double pSingle = PermutationPValue(atLeast, draws);
|
|
pFamily = 1.0 - MathPow(1.0 - pSingle, (double)distinct);
|
|
}
|
|
}
|
|
install = (pFamily <= MI_TUNE_ALPHA);
|
|
}
|
|
if(!install)
|
|
{
|
|
ArrayCopy(best, configured);
|
|
bestScore = startScore;
|
|
}
|
|
//--- install the winner and leave the indicators/feature cache consistent with it
|
|
m_indicatorTuner.Unflatten(best);
|
|
ReInitTunableIndicators(m_indicatorsPtr);
|
|
RefreshData();
|
|
//--- A gated INSTALL is chart-level news, not just this model's: persist the winning periods so
|
|
//--- the classic votes, the signal-DB key and every later tuner seed adopt them on the next
|
|
//--- attach (restart-grained - see Variables\TunedPeriods.mqh for why not mid-run).
|
|
if(install)
|
|
SaveTunedPeriods(m_indicatorTuner.maPeriod, m_indicatorTuner.maType, m_indicatorTuner.rsiPeriod,
|
|
m_indicatorTuner.macdFast, m_indicatorTuner.macdSlow, m_indicatorTuner.macdSignal,
|
|
m_indicatorTuner.ichiTenkan, m_indicatorTuner.ichiKijun, m_indicatorTuner.ichiSenkou);
|
|
double candSpread = (evaluated > 0 && candMax >= candMin) ? (candMax - candMin) : 0.0;
|
|
Print(ID + StringFormat(": auto-tune complete - %d candidate settings scored in %.1fs, "
|
|
"feature/label mutual information %.5f -> %.5f nats%s | candidate scores span "
|
|
"%.5f (%.5f..%.5f)%s",
|
|
evaluated, (GetTickCount() - t0) / 1000.0, startScore, bestScore,
|
|
(bestScore <= startScore ? " (no improvement - keeping the configured settings)" : ""),
|
|
candSpread, (evaluated > 0 ? candMin : 0.0), (evaluated > 0 ? candMax : 0.0),
|
|
(evaluated > 0 && candSpread <= 0.0
|
|
? StringFormat(" <-- ZERO SPREAD: every candidate scored identically, so the "
|
|
"parameter change is STILL not reaching the scored features even "
|
|
"with the post-re-init RefreshData(). Least-ready tunable handle "
|
|
"had %d bars calculated - if that is 0 or far below the study "
|
|
"window, the handles are simply not done calculating yet and the "
|
|
"tuner needs to yield between candidates rather than score them "
|
|
"back to back.", (readyMin == INT_MAX ? -1 : readyMin))
|
|
: StringFormat(" | winner %s (selection p=%.4f after correcting for %d "
|
|
"candidates, need <=%.2f)",
|
|
(install ? "INSTALLED" : "REJECTED - keeping the configured "
|
|
"settings, since the best of N noise draws beats its incumbent "
|
|
"almost every time"),
|
|
pFamily, distinct, MI_TUNE_ALPHA))));
|
|
//--- An EXACTLY zero score is not a weak feature set, it is a broken measurement. Landing on
|
|
//--- 0.0000 means every column read back constant, which is what a feature-extraction fault
|
|
//--- looks like.
|
|
if(bestScore <= 0.0)
|
|
Print(ID + ": WARNING - every candidate scored 0.0000 nats. Finite-sample bias alone should put "
|
|
"noise above zero, so this indicates the feature values are not being read, not that the "
|
|
"features are uninformative. Indicator settings left at their configured values.");
|
|
ReportFeatureLabelInformation();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| The MI evidence screen (ReportFeatureLabelInformation and its |
|
|
//| lag-profile sub-report) moved to FeatureScreen.mqh on 2026-08-23 |
|
|
//| - SEARCH (this file) vs MEASUREMENT (that one) are two |
|
|
//| responsibilities. FeatureColumnMI/BuildMiSample/ScoreMiSample |
|
|
//| above stay here: both files call them, and a shared dependency |
|
|
//| used by two consumers is not itself a reason to split further. |
|
|
//+------------------------------------------------------------------+
|
|
void CExpertSignalAIBase::TuneIndicatorsAndTrain(datetime StartTrainBar = 0)
|
|
{
|
|
//--- FIRST STATEMENT IN THE WHOLE TRAINING ENTRY POINT, ahead of every latch below it (m_tuneFilterDone,
|
|
//--- g_ensembleChartTuneDone) so a stop cannot mark a sweep as "already run" without running it. The
|
|
//--- individual scans yield on ShutdownRequested() as well; this simply refuses to start the chain.
|
|
if(ShutdownRequested())
|
|
return;
|
|
//--- ONE-SHOT CAPACITY RE-DERIVE. A cold fleet sizes every model before any chart has published a
|
|
//--- pool file, so the first layer is budgeted as if training alone and then pinned. Rather than
|
|
//--- require the operator to run the fleet twice (once to fill the pool, once to size against it),
|
|
//--- notice that the pool has appeared and re-derive. ResetWeights() is the existing, tested path
|
|
//--- that re-measures all four sizes and rebuilds - this adds no second copy of that logic.
|
|
//---
|
|
//--- Bounded on every axis that could make it a loop: once per model (m_capacityResizeDone, set
|
|
//--- BEFORE the reset because ResetWeights zeroes m_eraCount and would otherwise re-qualify), only
|
|
//--- while the model is young enough that the discarded eras are worth nothing, only on real growth,
|
|
//--- and only if the recomputed width actually DIFFERS.
|
|
if(!m_capacityResizeDone && !m_trainingComplete && !m_inferenceOnly &&
|
|
m_eraCount <= CAPACITY_RESIZE_MAX_ERA && m_poolObsAtDerivation >= 0.0)
|
|
{
|
|
double poolNow = TopologyPooledIndependentBars();
|
|
bool grew = (poolNow > 0.0) &&
|
|
(m_poolObsAtDerivation <= 0.0 || poolNow >= m_poolObsAtDerivation * CAPACITY_RESIZE_MIN_GROWTH);
|
|
if(grew)
|
|
{
|
|
int widthNow = ComputeFirstLayerWidth();
|
|
if(widthNow != m_initialNeuronsCount)
|
|
{
|
|
m_capacityResizeDone = true;
|
|
Print(ID + StringFormat(": CAPACITY RE-DERIVE at era %d - the training pool was worth %.0f"
|
|
" independent observations when this model was sized and is worth"
|
|
" %.0f now, which moves the first layer %d -> %d. Rebuilding and"
|
|
" restarting from era 0; the discarded eras were trained against a"
|
|
" topology budgeted for a chart with no peers.",
|
|
(int)m_eraCount, m_poolObsAtDerivation, poolNow,
|
|
m_initialNeuronsCount, widthNow));
|
|
ResetWeights(); // re-measures all four sizes, rebuilds, rewrites the .cfg
|
|
return; // let the next poll start training on the new shape
|
|
}
|
|
//--- Grown but the width is unchanged: nothing to rebuild, and asking again every era would
|
|
//--- re-run the census forever. Settle it here.
|
|
m_capacityResizeDone = true;
|
|
PrintVerbose(ID + ": capacity re-derive not needed - the pool grew but the first layer stays " +
|
|
IntegerToString(m_initialNeuronsCount) + ".");
|
|
}
|
|
}
|
|
//--- Publish the caller's window anchor so StartLabelCachePrebuild() sizes its window with the SAME
|
|
//--- expression Train() uses.
|
|
m_tuneStartTrainBar = StartTrainBar;
|
|
bool anyTunable = m_useMA; //--- see ParamOwner's note: the MA feature is the last tunable one
|
|
//--- Tune once per fresh model, before any weight has been trained. Gated on m_labelCachePrebuilt
|
|
//--- because the score needs labels, and on era 0 because re-tuning a partly-trained network would
|
|
//--- change its inputs out from under weights already fitted to the old ones.
|
|
if(m_autoTuneIndicators && anyTunable && !m_tuneFilterDone && m_labelCachePrebuilt && m_eraCount == 0)
|
|
{
|
|
m_tuneFilterDone = true;
|
|
if(m_ensembleMember && g_ensembleChartTuneDone)
|
|
{
|
|
//--- Another member on this chart already ran the identical sweep - apply its outcome
|
|
//--- instead of recomputing it (see g_ensembleChartTuneDone at the top of this file). SAY
|
|
//--- IT ON THE PANEL, not only in the journal.
|
|
PublishStatus(ID + " : adopting the chart's tuned indicators...");
|
|
//--- THE PARAMETERS are adopted only when a winner was installed...
|
|
if(g_ensembleChartTuneInstalled)
|
|
m_indicatorTuner.Unflatten(g_ensembleChartTuneSettings);
|
|
//--- ...but the HANDLES must be rebuilt EITHER WAY, and that is not a tidiness point - it
|
|
//--- is the cause of the "silent block failure" that cost six sessions. That is the whole
|
|
//--- finding: it was never four handles, it was ONE.
|
|
ReInitTunableIndicators(m_indicatorsPtr);
|
|
RefreshData();
|
|
Print(ID + ": indicator auto-tune already ran on this chart - same indicators, same features, "
|
|
"same labels, same answer. " +
|
|
(g_ensembleChartTuneInstalled
|
|
? "Adopting the installed winner so every member trains on the same feature vector."
|
|
: "Keeping the configured settings (the sweep's winner was rejected by the selection gate).") +
|
|
" The first member's auto-tune report above is this model's too.");
|
|
}
|
|
else
|
|
{
|
|
//--- Names the SCOPE, because the scope is what the other rows' silence means.
|
|
PublishStatus(ID + (m_ensembleMember
|
|
? " : scoring indicator settings for the whole chart..."
|
|
: " : scoring indicator settings..."));
|
|
//--- Snapshot the configured settings first: "did the sweep install?" is answered by comparing
|
|
//--- against the final settings, since a rejected winner is restored to exactly these values.
|
|
double tuneCfgBefore[];
|
|
m_indicatorTuner.Flatten(tuneCfgBefore);
|
|
TuneIndicatorsByFilter();
|
|
if(m_ensembleMember)
|
|
{
|
|
m_indicatorTuner.Flatten(g_ensembleChartTuneSettings);
|
|
g_ensembleChartTuneInstalled = false;
|
|
for(int tp = 0; tp < ArraySize(tuneCfgBefore); tp++)
|
|
if(g_ensembleChartTuneSettings[tp] != tuneCfgBefore[tp])
|
|
{
|
|
g_ensembleChartTuneInstalled = true;
|
|
break;
|
|
}
|
|
g_ensembleChartTuneDone = true;
|
|
//--- The sweep ends in ReportFeatureLabelInformation(), so the chart-level MI report is
|
|
//--- done too - mark it, or every other member would rerun the ~200-draw nulls the MI
|
|
//--- gate below exists to save.
|
|
if(m_miReportDone)
|
|
g_ensembleChartMiReportDone = true;
|
|
}
|
|
}
|
|
//--- the winning parameters change the input vector, so the network must start from scratch on it
|
|
BuildFreshTopology();
|
|
}
|
|
//--- The DIAGNOSTIC half runs even when the sweep does not: on a resumed model, on one whose
|
|
//--- tuner is switched off, and on one with nothing tunable.
|
|
else if(!m_miReportDone && !m_labelCachePrebuilt && !m_labelPrebuildActive)
|
|
{
|
|
//--- Announce only on a start that actually took. StartLabelCachePrebuild() returns without arming
|
|
//--- if the buffers/history are not ready yet and is simply retried on the next call, so printing
|
|
//--- unconditionally would repeat the line once per bar event until it succeeds.
|
|
StartLabelCachePrebuild();
|
|
//--- Says WHICH case this is rather than asserting the resumed one. A diagnostic that
|
|
//--- misreports its own trigger is worse than one that says nothing, because it gets quoted
|
|
//--- back as evidence.
|
|
if(m_labelPrebuildActive)
|
|
Print(ID + (m_modelLoadedFromDisk
|
|
? ": MI diagnostics need a complete label cache and this model resumed from disk "
|
|
"(labels are filled lazily, so the cache covers only the bars training has "
|
|
"visited) - running the one-time pre-scan now, then the report. Training resumes "
|
|
"where it left off."
|
|
: ": MI diagnostics need a complete label cache and this model has not built one yet "
|
|
"- running the pre-scan now, then the report."));
|
|
}
|
|
else if(!m_miReportDone && m_labelCachePrebuilt)
|
|
{
|
|
//--- WAIT FOR THE CROSS-ASSET PANEL. It is part of the feature vector but it is built inside
|
|
//--- Train(), so on a fresh run this diagnostic would otherwise describe a NARROWER vector
|
|
//--- than the one training goes on to use.
|
|
if(m_ensembleMember && g_ensembleChartMiReportDone)
|
|
{
|
|
//--- see g_ensembleChartMiReportDone at the top of this file
|
|
m_miReportDone = true;
|
|
//--- Same reasoning as the tuner's adopt branch above: published, not just printed, so the row
|
|
//--- says why it is not repeating the measurement.
|
|
PublishStatus(ID + " : reusing the chart's information report...");
|
|
Print(ID + ": MI diagnostics already measured by another ensemble member on this chart - "
|
|
"same features, same labels, same answer. Skipped (saves the slowest part of the "
|
|
"ensemble's warm-up; the first member's report above is this model's too).");
|
|
}
|
|
else
|
|
if(m_crossAsset.IsReady() || m_miReportDeferrals >= MI_REPORT_MAX_DEFERRALS)
|
|
{
|
|
//--- THE LONGEST SINGLE STRETCH OF THE WARM-UP - the MI suite and the lag profile, each
|
|
//--- with its own few-hundred-draw permutation null - and until now it published NOTHING.
|
|
PublishStatus(ID + (m_ensembleMember
|
|
? " : measuring feature/label information for the whole chart..."
|
|
: " : measuring feature/label information..."));
|
|
ReportFeatureLabelInformation();
|
|
if(m_ensembleMember && m_miReportDone)
|
|
g_ensembleChartMiReportDone = true;
|
|
}
|
|
else
|
|
m_miReportDeferrals++;
|
|
}
|
|
Train(StartTrainBar);
|
|
}
|
|
#endif // WARRIOR_AIBASE_AUTOTUNE_MQH
|