//+------------------------------------------------------------------+ //| Warrior_EA | //| AnimateDread | //| | //| ZigZag pivot labelling and the async label-cache prebuild. | //| | //| PARTIAL IMPLEMENTATION FILE - not standalone. | //| This holds CExpertSignalAIBase method BODIES only. The class | //| declaration lives in Expert\ExpertSignalAIBase.mqh, which | //| #includes this file at the bottom, after the declaration. Do not | //| include it anywhere else and do not compile it on its own. | //| | //| Split out purely to make the 8216-line original navigable; the | //| code inside was moved verbatim, not rewritten. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AIBASE_LABELS_MQH #define WARRIOR_AIBASE_LABELS_MQH //+------------------------------------------------------------------+ //| (Re)sizes the label AND feature caches and clears them if `bars` | //| (or the now-relative index frame) has changed since the last | //| build - see the member declaration comments for why this is the | //| correct invalidation trigger. Returns true if a rebuild happened. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::EnsureBarCachesCapacity(int bars) { if(bars == m_labelCacheBars && m_Time.GetData(0) == m_labelCacheAnchorTime) return false; ArrayResize(m_labelCacheBuy, bars); ArrayResize(m_labelCacheSell, bars); ArrayResize(m_labelCacheHasValue, bars); ArrayInitialize(m_labelCacheHasValue, false); ArrayResize(m_featureCache, bars * m_neuronsCount); ArrayResize(m_featureCacheHasValue, bars); ArrayResize(m_featureCacheValid, bars); ArrayInitialize(m_featureCacheHasValue, false); m_labelCacheBars = bars; m_labelCacheAnchorTime = m_Time.GetData(0); return true; } //+------------------------------------------------------------------+ //| Lazy cache-miss fallback for a bar the eager prebuild pass (see | //| AdvanceZigZagLabelState()) didn't cover - e.g. a new candle that | //| closed after prebuild already completed. A genuine ZigZag pivot | //| can only be confirmed once price has deviated enough FROM it on a | //| LATER bar (see AdvanceZigZagLabelState()), which for a bar this | //| close to "now" may not have happened yet - exactly like a live | //| ZigZag indicator's most recent leg is always provisional/ | //| repainting until confirmed. Rather than guess, this always labels | //| Neutral; the sequential prebuild scan is what actually assigns | //| Buy/Sell once there's enough subsequent price action to know. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::ComputeLabelForBar(int i, int bars, bool &buy, bool &sell) { buy = false; sell = false; } //+------------------------------------------------------------------+ //| Confirms (or Neutral-labels) whichever candidate bar is exactly | //| m_swingConfirmationBars bars behind the one currently being | //| visited, by reading the real ADZigZag indicator's own verdict on | //| it directly - see m_swingConfirmationBars' declaration comment | //| for why that delay exists (a ZigZag's recent legs can repaint). | //| Buy = candidate bar is a confirmed bottom (ADZigZagBuffer == its | //| own Low), Sell = confirmed top (== its own High), else Neutral - | //| this is the "will the next candle be the ZigZag reversal" label | //| itself, at whatever now-relative index the feature window ends | //| at (unchanged from the old fractal-based labeling's index | //| convention - see BufferTempData()'s callers for how a training | //| example's feature window and its label index line up). | //| MUST be called in strict oldest-to-newest order (decreasing | //| now-relative index i) - AdvanceLabelCachePrebuild() (the only | //| caller) already guarantees this, though unlike the old fractal | //| scan this replaced, no state actually carries between calls here. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::AdvanceZigZagLabelState(int i, int bars) { int idx = i + MathMax(m_swingConfirmationBars, 1); if(idx >= bars || m_labelCacheHasValue[idx]) return; bool buy = false, sell = false; double zz = m_ADZigZag.GetData(0, idx); if(zz != 0.0) { double hi = m_High.GetData(idx); double lo = m_Low.GetData(idx); if(zz >= hi - _Point) sell = true; // confirmed top -> Sell else if(zz <= lo + _Point) buy = true; // confirmed bottom -> Buy } m_labelCacheBuy[idx] = buy; m_labelCacheSell[idx] = sell; m_labelCacheHasValue[idx] = true; } //+------------------------------------------------------------------+ //| Nearest confirmed ZigZag pivot at fromIdx or older (now-relative | //| index, so "older" means scanning with INCREASING p - see this | //| file's now-relative-index convention, same as AdvanceZigZagLabel- | //| State() above). Capped at SWING_SCAN_CAP_BARS so a long quiet | //| stretch with no qualifying pivot can't turn this into an unbounded | //| scan; returns false (no pivot found) rather than looping forever | //| if the cap is hit or history runs out first. | //| | //| Caller's responsibility, not this method's: applying the | //| m_swingConfirmationBars repainting embargo to fromIdx before | //| calling. This method itself just finds the nearest nonzero | //| ADZigZag buffer entry at/after whatever index it's given - it has | //| no opinion on whether that index is safe to read yet. The ONE | //| caller that needs the embargo (BufferTempDataCompute()'s | //| m_useSwingContext block, looking up "the pivot as of THIS bar") | //| applies it before the first call; the second call in that same | //| block (finding the PRIOR completed leg, starting from pivotIdx+1) | //| doesn't need to re-apply it - anything at or before an already- | //| confirmed pivot is necessarily even older, hence already confirmed | //| too. | //+------------------------------------------------------------------+ bool CExpertSignalAIBase::FindConfirmedZigZagPivot(int fromIdx, int &pivotIdx, double &pivotPrice, bool &pivotIsLow) { for(int p = MathMax(fromIdx, 0); p < fromIdx + SWING_SCAN_CAP_BARS; p++) { if(m_Open.GetData(p) == EMPTY_VALUE) return false; // ran off the end of available history double zz = m_ADZigZag.GetData(0, p); if(zz == 0.0) continue; pivotIdx = p; pivotPrice = zz; pivotIsLow = (zz <= m_Low.GetData(p) + _Point); return true; } return false; } //+------------------------------------------------------------------+ //| Kicks off the one-time eager label-cache pre-build for a fresh | //| start (see m_labelCachePrebuilt's declaration comment). Computes | //| the bar count/OOS split exactly as Train()'s era-start block | //| would, then arms AdvanceLabelCachePrebuild() to do the actual | //| chunked scan on this and subsequent Train() calls. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::StartLabelCachePrebuild(void) { int barsNow = (int)MathMin(Bars(m_symbol.Name(), PERIOD_CURRENT, dtStudied, TimeCurrent()) + m_historyBars, Bars(m_symbol.Name(), PERIOD_CURRENT)); if(!ResizeBuffers(barsNow) || !RefreshData()) return; // couldn't prep buffers yet - m_labelCachePrebuilt stays false, retried next call EnsureBarCachesCapacity(barsNow); int totalIter = (int)MathMax(barsNow - MathMax(m_historyBars, 0), 0); m_labelPrebuildBars = barsNow; m_labelPrebuildOosCutoff = (int)(MathMax(0, MathMin(100, m_oosSplitPct)) / 100.0 * totalIter); m_labelPrebuildIndex = (int)(barsNow - MathMax(m_historyBars, 0) - 1); m_labelPrebuildBuyCount = 0; m_labelPrebuildSellCount = 0; m_labelPrebuildNeutralCount = 0; m_labelPrebuildActive = true; } //+------------------------------------------------------------------+ //| Advances the eager label-cache pre-build by up to a time budget, | //| then yields (same chunking pattern as the era loop). Mirrors the | //| era loop's own labeling eligibility gate (minus the dPrevSignal | //| check, meaningless pre-first-feedForward). On completion, seeds | //| m_prevEraTrueBuyCount/Sell/Neutral from the upfront IS-only tally | //| so era 0 gets real class-balance oversampling instead of reps=1. | //+------------------------------------------------------------------+ void CExpertSignalAIBase::AdvanceLabelCachePrebuild(void) { const uint PREBUILD_TIME_BUDGET_MS = 80; uint chunkStartTick = GetTickCount(); int i; for(i = m_labelPrebuildIndex; i >= 2; i--) { if(GetTickCount() - chunkStartTick >= PREBUILD_TIME_BUDGET_MS) { m_labelPrebuildIndex = i; return; } if(!(i < (int)(m_labelPrebuildBars - MathMax(m_historyBars, 0) - 1) && m_Time.GetData(i) > dtStudied)) continue; // Unlike the old per-bar-independent ComputeLabelForBar(), a proper ZigZag pivot can only be // confirmed by a LATER (more recent, lower-index) bar than the candidate itself - see // AdvanceZigZagLabelState()'s declaration comment. So a bar's final Buy/Sell/Neutral label is // NOT necessarily known the moment we visit it here; it may get retroactively overwritten by a // later iteration of this same loop. The IS/OOS Buy/Sell/Neutral tally below is therefore done // in a separate final pass AFTER the whole scan completes, not incrementally per-bar like the // old cache-hit-or-compute code here used to. if(!m_labelCacheHasValue[i]) AdvanceZigZagLabelState(i, m_labelPrebuildBars); } //--- Sequential scan complete - every bar's own exact-pivot label is final. Widen each confirmed //--- pivot into its +-LABEL_WINDOW_BARS neighborhood (see that constant's declaration comment) //--- before the tally pass below, so both the logged "IS true-label distribution" and era 0's //--- oversampling ratio reflect what the network is actually trained/scored against. Spreads from a //--- frozen SNAPSHOT of the just-resolved genuine pivots, never from the live m_labelCache* arrays //--- being written below - reading live here would let a just-written window cell (i-1, i-2, ...) //--- get visited later in this same decreasing-i pass, look indistinguishable from a genuine pivot, //--- and re-spread from itself - a single real pivot cascading across the entire array instead of //--- staying bounded to +-LABEL_WINDOW_BARS (observed in practice: one training run's IS tally came //--- back Buy 5000 / Sell 4657 / Neutral 8 out of ~9665 bars). if(LABEL_WINDOW_BARS > 0) { bool origBuy[], origSell[]; ArrayCopy(origBuy, m_labelCacheBuy); ArrayCopy(origSell, m_labelCacheSell); int maxIdx = m_labelPrebuildBars - 1; for(i = maxIdx; i >= 2; i--) { if(!origBuy[i] && !origSell[i]) continue; // not a genuine confirmed pivot in the original scan - nothing to spread from bool isBuy = origBuy[i]; for(int w = 1; w <= LABEL_WINDOW_BARS; w++) { int lo = i - w, hi = i + w; // Only claim a neighbor that wasn't ITSELF a genuine pivot in the ORIGINAL snapshot - a // real pivot (of either class) always keeps its own exact label; overlapping window-bleed // from two nearby pivots may freely overwrite each other (harmless - neither is ground truth). if(lo >= 2 && !origBuy[lo] && !origSell[lo]) { m_labelCacheBuy[lo] = isBuy; m_labelCacheSell[lo] = !isBuy; m_labelCacheHasValue[lo] = true; } if(hi <= maxIdx && !origBuy[hi] && !origSell[hi]) { m_labelCacheBuy[hi] = isBuy; m_labelCacheSell[hi] = !isBuy; m_labelCacheHasValue[hi] = true; } } } } //--- Final tally pass (IS-only, matches isOOS = (i < oosCutoff) used by the era loop) to seed era //--- 0's oversampling ratio - now over the widened labels, not just the exact pivot bars. for(i = m_labelPrebuildBars - MathMax(m_historyBars, 0) - 1; i >= MathMax(2, m_labelPrebuildOosCutoff); i--) { if(!m_labelCacheHasValue[i]) continue; // e.g. bar was outside the dtStudied/window-edge eligibility gate above if(m_labelCacheBuy[i]) m_labelPrebuildBuyCount++; else if(m_labelCacheSell[i]) m_labelPrebuildSellCount++; else m_labelPrebuildNeutralCount++; } //--- Prebuild complete - seed era 0's oversampling ratio from the real upfront tally instead of //--- leaving it at the reps=1 fallback (see m_prevEraTrueBuyCount's declaration comment). Consumed //--- (and cleared) by Train()'s era-start block on era 0 specifically - see m_prebuildSeedPending. m_prevEraTrueBuyCount = m_labelPrebuildBuyCount; m_prevEraTrueSellCount = m_labelPrebuildSellCount; m_prevEraTrueNeutralCount = m_labelPrebuildNeutralCount; m_prebuildSeedPending = true; m_labelCachePrebuilt = true; m_labelPrebuildActive = false; //--- Measured-imbalance visibility: the oversampling rate is derived per era from tallies like //--- these (no hand-tuned rate anywhere - see MAX_OVERSAMPLE_REPLICAS's declaration comment), so //--- log what this window actually measured and the parity reps it will produce, on any //--- market/timeframe. int prebuildMinDir = (int)MathMin(m_labelPrebuildBuyCount, m_labelPrebuildSellCount); int prebuildMaxCls = (int)MathMax(m_labelPrebuildNeutralCount, MathMax(m_labelPrebuildBuyCount, m_labelPrebuildSellCount)); string prebuildRatioInfo = (prebuildMinDir > 0 && prebuildMaxCls > 0) ? " | measured imbalance ~" + DoubleToString((double)prebuildMaxCls / prebuildMinDir, 1) + ":1 -> reps up to " + IntegerToString((int)MathMin(MAX_OVERSAMPLE_REPLICAS, MathMax(1, MathRound((double)prebuildMaxCls / prebuildMinDir * m_oversampleParity)))) + "x (" + DoubleToString(m_oversampleParity * 100.0, 0) + "% parity)" : " | measured imbalance n/a (a directional class has no labeled bars in this window)"; Print(ID + ": label cache pre-built - IS true-label distribution -> Buy: " + IntegerToString(m_labelPrebuildBuyCount) + " | Sell: " + IntegerToString(m_labelPrebuildSellCount) + " | Neutral: " + IntegerToString(m_labelPrebuildNeutralCount) + prebuildRatioInfo + (m_eraCount == 0 ? " (seeding era 0's class-balance oversampling)" : " (mid-run rebuild after new-bar cache invalidation - era " + IntegerToString(m_eraCount) + " resumes on the relabeled window)")); //--- Cold-start fix: a freshly-initialized (random-weight) network's argmax is close to uniform //--- noise across the 3 classes, so on this typically heavily-skewed label distribution it fires //--- far more non-majority-class calls at the very start of era 0 than the true base rate warrants, //--- until enough backProp steps correct it. Now that the real prior is known, push the output //--- layer's bias toward whichever class actually dominates - only the bias term moves, the //--- per-input weights stay randomly initialized and still carry the real learning signal. Only //--- meaningful for the 3-output classification head, and only for a fresh net (this whole prebuild //--- path is skipped entirely when a trained net was loaded from disk - see m_labelCachePrebuilt). //--- m_eraCount==0 gate: the prebuild can also re-run MID-run now (new-bar cache invalidation - //--- see Train()'s era-start wipe check); stomping a partially-trained net's output biases with //--- +-3.0 cold-start values there would erase real learned calibration, so fresh runs only. if(m_outputNeuronsCount == 3 && m_eraCount == 0) { int dominant = 2; // Neutral int dominantCount = m_labelPrebuildNeutralCount; if(m_labelPrebuildBuyCount > dominantCount) { dominant = 0; dominantCount = m_labelPrebuildBuyCount; } if(m_labelPrebuildSellCount > dominantCount) { dominant = 1; dominantCount = m_labelPrebuildSellCount; } int totalLabeled = m_labelPrebuildBuyCount + m_labelPrebuildSellCount + m_labelPrebuildNeutralCount; if(totalLabeled > 0 && (double)dominantCount / totalLabeled > 0.40) { const double BIAS_MAGNITUDE = 3.0; // sigmoid(+-3) ~= 0.95/0.05 - comfortably outweighs a // fresh network's random per-input weighted-sum noise double biasValues[3] = { -BIAS_MAGNITUDE, -BIAS_MAGNITUDE, -BIAS_MAGNITUDE }; biasValues[dominant] = BIAS_MAGNITUDE; if(Net.SeedOutputLayerBias(biasValues)) PrintVerbose(ID + ": seeded output layer bias toward " + EnumToString((ENUM_SIGNAL)(dominant == 0 ? Buy : dominant == 1 ? Sell : Neutral)) + " (era 0 cold-start fix)"); } } } //+------------------------------------------------------------------+ //| Confirmed ZigZag label for an already-non-repainting bar - see | //| the header declaration and AdvanceZigZagLabelState() (identical | //| verdict). Caller must guarantee idx >= m_swingConfirmationBars so | //| the leg at idx can no longer repaint; this method does not re- | //| check that (it has no more information than the raw buffer). | //+------------------------------------------------------------------+ ENUM_SIGNAL CExpertSignalAIBase::ConfirmedZigZagLabel(int idx) { double zz = m_ADZigZag.GetData(0, idx); if(zz != 0.0) { double hi = m_High.GetData(idx); double lo = m_Low.GetData(idx); if(zz >= hi - _Point) return Sell; // confirmed top if(zz <= lo + _Point) return Buy; // confirmed bottom } return Neutral; } #endif // WARRIOR_AIBASE_LABELS_MQH