//+------------------------------------------------------------------+ //| SignalNeural.mqh | //| AnimateDread | //| | //| A NEURAL NETWORK AS AN ORDINARY SIGNAL MODULE. | //| | //| It derives from CWarriorSignal, implements LongCondition() and | //| ShortCondition() returning 0..100, and that is the whole of its | //| contract with the rest of the EA. The vote cannot tell it apart | //| from CSignalMA, and nothing in the parent knows a network exists. | //| | //| That is deliberate, and it is a return to how this repo started. | //| At 0a527b0 an NN module did exactly this. At 2f5adb4 it gained | //| SignedAIConfidence(), and from then on the parent had to special- | //| case it: confidence-scaled stops, confidence-scaled lots, tier | //| ladders, LiveVote(), VoteCapableWeight(), IsAIFilter(). None of | //| that made money and all of it made the net unrankable, because a | //| thing that votes in its own private currency cannot be compared | //| with the modules it votes alongside. | //| | //| IT BUILDS ITS OWN INPUTS. There is no separate feature-builder | //| file: a module that votes owns what it votes on. The names below | //| are the contract the model file is checked against, so a model | //| trained on a different column set is refused rather than fed. | //| | //| WHAT IT PREDICTS. Not direction - this project measured direction | //| dead at bar level (rho 0.00-0.02). It predicts whether the bar's | //| low survives the next two bars: the unknown half of a Bill | //| Williams fractal. Measured 2026-09-11 on 20 instrument/timeframe | //| series, out of sample, chronological split: AUC 0.772 (range | //| 0.756-0.792). Mean R by predicted-probability quintile ran -0.230 | //| / -0.089 / -0.026 / +0.002 / +0.017, monotone on 20 of 20. | //| | //| ⚠ SO ITS VALUE IS A REFUSAL, NOT A SELECTION. | //| Its value is in refusing the worst quintile, not in selecting the | //| best: the top quintile is breakeven before costs. A module that | //| returned a big number on a high score would be claiming an edge | //| the measurement does not support. | //| | //| The standard library has no way to refuse ONE side, and the note | //| above ShortCondition() works through what it does offer and what | //| this module therefore assumes. When a comment and the code | //| disagree, believe the journal: it counts what the module said. | //+------------------------------------------------------------------+ #ifndef WARRIOR_SIGNALNEURAL_MQH #define WARRIOR_SIGNALNEURAL_MQH #include "..\Expert\WarriorSignal.mqh" #include "..\System\WarriorNet.mqh" #include "Wyckoff\WyckoffFeed.mqh" #include "..\System\AltDataFeed.mqh" //--- 9 price shape + 9 Wyckoff + 3 volume + 4 calendar + 16 alt data. #define NEURAL_PRICE_F 9 #define NEURAL_WYK_F 9 #define NEURAL_VOL_F 3 #define NEURAL_TIME_F 4 #define NEURAL_FEATURES (NEURAL_PRICE_F + NEURAL_WYK_F + NEURAL_VOL_F + NEURAL_TIME_F + ALT_COLUMNS) class CSignalNeural : public CWarriorSignal { protected: CWarriorNet m_net; CiATR m_atr; int m_atrPeriod; int m_pattern_0; // the refusal, voted as a SHORT - see ShortCondition() int m_pattern_1; // the mild confirmation a high score earns double m_cut; // p <= 1-cut is the refusal band, p >= cut the confirmation bool m_trained; //--- THE SCORE, COMPUTED ONCE PER BAR. LongCondition() and ShortCondition() are both called on //--- the same evaluation by Direction() (ExpertSignal.mqh:431) and must read the SAME number - //--- scoring twice could answer two different things if anything underneath refreshed between //--- the calls, and the two bands would then no longer be mutually exclusive. datetime m_scoreBar; double m_score; // <0 when this bar has no usable score double Score(void); //--- Monotone, bounded, stateless: maps any real into (-1,1) while preserving order. Used where //--- a feature has no natural scale and a stored mean/variance would be one more thing to keep //--- in step with the model file. static double Squash(const double v) { return v / (1.0 + MathAbs(v)); } //--- DEFERRED TRAINING. In the tester Bars() at OnInit returns almost nothing - history accrues //--- AS THE RUN PROGRESSES - so training at init can never work in a backtest. Training partway //--- through instead is not a workaround, it is the correct shape: Bars() then returns //--- history-so-far, so the model can only ever have been fitted on the past and every bar it //--- votes on is genuinely out of sample. Retraining on an interval makes it walk-forward. int m_minBars; // do not train until this much history exists int m_retrainBars; // retrain every N bars (0 = train once) datetime m_lastTrainBar; int m_trainCount; int m_trainedAtBars; void TrainIfDue(void); //--- The column contract. Order is the contract; append only, never insert. static string FeatureName(const int i); bool BuildFeatures(double &x[], const int shift); //--- P(this bar's low survives the next two) - the fractal's unknown half. bool LabelAt(const int shift, double &label); bool TrainFromHistory(void); string ModelPath(void) const { return "Warrior_EA\\Nets\\" + m_symbol.Name() + "_" + IntegerToString(m_period) + "_fractal.net"; } public: CSignalNeural(void); ~CSignalNeural(void) {} void AtrPeriod(const int v) { m_atrPeriod = v; } void MinBars(const int v) { m_minBars = v; } void RetrainBars(const int v) { m_retrainBars = v; } void Confidence(const double v) { m_cut = v; } void Pattern_0(const int v) { m_pattern_0 = v; } void Pattern_1(const int v) { m_pattern_1 = v; } virtual void ApplyPatternWeight(int pattern, int weight) { if(pattern == 0) m_pattern_0 = weight; if(pattern == 1) m_pattern_1 = weight; } virtual bool InitIndicators(CIndicators *indicators) override; virtual int LongCondition(void) override; virtual int ShortCondition(void) override; }; //+------------------------------------------------------------------+ CSignalNeural::CSignalNeural(void) : m_atrPeriod(14), m_pattern_0(40), m_pattern_1(15), m_cut(0.50), m_trained(false), m_minBars(750), m_retrainBars(500), m_lastTrainBar(0), m_trainCount(0), m_trainedAtBars(0), m_scoreBar(0), m_score(-1.0) { m_id = "NEURAL"; m_pattern_count = 2; m_used_series = USE_SERIES_OPEN + USE_SERIES_HIGH + USE_SERIES_LOW + USE_SERIES_CLOSE + USE_SERIES_TICK_VOLUME; } //+------------------------------------------------------------------+ string CSignalNeural::FeatureName(const int i) { switch(i) { //--- Price shape, all scale-free. A raw price here would teach the net the era, not the market. case 0: return "close_in_bar"; // where the close sat in its own range case 1: return "range_atr"; // this bar's range, in ATR case 2: return "below_prior_low_atr"; // how far the low undercut the previous one case 3: return "ret1_atr"; // last return case 4: return "ret3_atr"; case 5: return "dist_sma50_atr"; // trend context case 6: return "prior_range_atr"; case 7: return "upper_wick_atr"; case 8: return "lower_wick_atr"; //--- Wyckoff structure, read through the shared feed. case 9: return "wyk_event_signed"; case 10: return "wyk_phase_signed"; case 11: return "wyk_spring_grade"; case 12: return "wyk_character"; case 13: return "wyk_range_open"; case 14: return "wyk_range_pos"; case 15: return "wyk_dist_creek_atr"; case 16: return "wyk_fail_value"; case 17: return "wyk_bar_quality"; //--- VOLUME, and never raw. Tick volume trends upward over a decade as the feed densifies, //--- so a raw count teaches the net which YEAR it is looking at - the same trap as a raw //--- price. All three are ratios against the recent past, which is scale-free and era-free. case 18: return "vol_rel_sma20"; case 19: return "vol_rel_sma5"; case 20: return "vol_chg1"; //--- TIME, CYCLICALLY ENCODED. The previous generation of nets fed CLOCK time as an ordinal, //--- which asserts that Friday is five times Monday and that December is twelve times //--- January, and puts a discontinuity between the last bar of one week and the first of the //--- next. sin/cos pairs make the wrap continuous and the ordering honest. On D1 there is no //--- intraday session to encode - the useful periodicities are the week and the year. case 21: return "dow_sin"; case 22: return "dow_cos"; case 23: return "month_sin"; case 24: return "month_cos"; } //--- THE ALT BLOCK, appended last so the existing columns keep their indices - the model file is //--- validated against these names, so an insert would invalidate every net ever trained. if(i >= 25 && i < 25 + ALT_COLUMNS) return "alt_" + CAltDataFeed::ColumnName(i - 25); return "?"; } //+------------------------------------------------------------------+ bool CSignalNeural::BuildFeatures(double &x[], const int shift) { ArrayResize(x, NEURAL_FEATURES); ArrayInitialize(x, 0.0); const double atr = m_atr.Main(shift); if(atr <= 0.0 || !MathIsValidNumber(atr)) return false; const double o = Open(shift), h = High(shift), l = Low(shift), c = Close(shift); const double rng = h - l; if(rng <= 0.0) return false; double sma = 0.0; for(int k = 0; k < 50; k++) sma += Close(shift + k); sma /= 50.0; int i = 0; x[i++] = (c - l) / rng; x[i++] = rng / atr; x[i++] = (Low(shift + 1) - l) / atr; x[i++] = (c - Close(shift + 1)) / atr; x[i++] = (c - Close(shift + 3)) / atr; x[i++] = (c - sma) / atr; x[i++] = (High(shift + 1) - Low(shift + 1)) / atr; x[i++] = (h - MathMax(o, c)) / atr; x[i++] = (MathMin(o, c) - l) / atr; //--- The Wyckoff half. The feed zeroes what it cannot say, and zero is a legitimate "no //--- structure here" for every one of these - so no have/haven't flag is needed. x[i++] = g_wyckoffFeed.Event(shift) / 9.0; x[i++] = g_wyckoffFeed.Phase(shift) / 5.0; x[i++] = g_wyckoffFeed.SpringGrade(shift) / 3.0; x[i++] = g_wyckoffFeed.Character(shift); const bool open = g_wyckoffFeed.RangeOpen(shift); x[i++] = open ? 1.0 : 0.0; const double top = g_wyckoffFeed.ZoneTop(shift), bot = g_wyckoffFeed.ZoneBottom(shift); //--- 0.5 when there is no range: a ratio against a zero-width range is not a small number, it is //--- a wrong one, and the midpoint is the honest "no information" value for a position feature. x[i++] = (open && (top - bot) > 0.0) ? ((c - bot) / (top - bot)) : 0.5; const double creek = g_wyckoffFeed.Creek(shift); x[i++] = (creek != 0.0) ? ((c - creek) / atr) : 0.0; x[i++] = g_wyckoffFeed.FailValue(shift) / 2.5; x[i++] = g_wyckoffFeed.BarQuality(shift) / 2.0; //--- VOLUME. A bar that moved on twice its usual participation is a different bar from one that //--- drifted on nothing, and the price-shape columns above cannot express that at all. double v20 = 0.0, v5 = 0.0; for(int k = 0; k < 20; k++) { const double v = (double)TickVolume(shift + k); v20 += v; if(k < 5) v5 += v; } v20 /= 20.0; v5 /= 5.0; const double vNow = (double)TickVolume(shift); const double vPrev = (double)TickVolume(shift + 1); //--- A dead bar would divide by zero; refuse the row rather than emit an infinity that //--- MathIsValidNumber would pass and the trainer would choke on. if(v20 <= 0.0 || v5 <= 0.0 || vPrev <= 0.0) return false; x[i++] = Squash(vNow / v20 - 1.0); x[i++] = Squash(vNow / v5 - 1.0); x[i++] = Squash(vNow / vPrev - 1.0); //--- CALENDAR POSITION, cyclic. See FeatureName() for why this is not an ordinal. MqlDateTime t; TimeToStruct(iTime(m_symbol.Name(), m_period, shift), t); const double dow = 2.0 * M_PI * t.day_of_week / 7.0; const double mon = 2.0 * M_PI * (t.mon - 1) / 12.0; x[i++] = MathSin(dow); x[i++] = MathCos(dow); x[i++] = MathSin(mon); x[i++] = MathCos(mon); //--- THE ALT BLOCK. Absent (pre-2010) means the ROW IS DROPPED, never zero-filled - see the //--- header of AltDataFeed.mqh. Squashed rather than z-scored: v/(1+|v|) is monotone, bounded, //--- and stateless, so it needs no training-set statistics to be stored in the model file and //--- cannot leak one fold's scale into another. double alt[]; if(!g_altData.Lookup(iTime(m_symbol.Name(), m_period, shift), alt)) return false; for(int c2 = 0; c2 < ALT_COLUMNS; c2++) x[i++] = Squash(alt[c2]); return (i == NEURAL_FEATURES); } //| THE LABEL. Known only at shift-2, so training must never use a | //| bar newer than that - which is what the offset in the loop below | //| enforces. Predicting it at `shift` is a genuine two-bar-ahead | //| question, not a restatement of the present. | //+------------------------------------------------------------------+ bool CSignalNeural::LabelAt(const int shift, double &label) { if(shift < 2) return false; const double l = Low(shift); label = (l < Low(shift - 1) && l < Low(shift - 2)) ? 1.0 : 0.0; return true; } //+------------------------------------------------------------------+ bool CSignalNeural::TrainFromHistory(void) { const int bars = Bars(m_symbol.Name(), m_period); //--- Leave the newest bars alone: the label needs two bars after it, and the features need the //--- 50 before it. Both ends are trimmed rather than clamped, so no row is built from a window //--- that does not exist. const int first = 60, last = bars - 3; if(last - first < 200) { Print("CSignalNeural: not enough history to train."); return false; } //--- REACH PAST THE STANDARD LIBRARY'S 1024-BAR BUFFER BEFORE WALKING HISTORY. //--- //--- Without this the sweep below reads 0.0 for every shift past 1023 and the rows simply stop //--- appearing - which is how this module trained on exactly 915 rows of an 11-year run, four //--- times, without ever printing a warning. Ask for what the run has actually accrued, never //--- more: in the tester Bars() is history-so-far, so this cannot reach past the bar being //--- decided and the fit stays walk-forward. //--- //--- THE TWO MUST GROW TOGETHER OR NOT AT ALL. The Wyckoff half of the feature vector reports //--- 0.0 for "no structure here", and a buffer that has run out reports the same 0.0. So deep //--- prices with a shallow feed would not lose rows - it would manufacture ~2,000 rows whose //--- nine structure features are all zero, and teach the net that old bars are featureless. //--- That is strictly worse than the shortfall it replaces, so a half-failure refuses. //--- The ATR is in the same group and for the same reason: BuildFeatures() divides nine of its //--- eighteen columns by it and refuses the row when it is not positive, so an ATR left at 1024 //--- would re-impose the exact ceiling the other two just lifted. const int want = MathMin(bars, WARRIOR_NET_HISTORY); const bool deepAtr = (want <= m_atr.BufferSize()) || m_atr.BufferResize(want); if(!DeepenPrices(want) || !deepAtr || !g_wyckoffFeed.Deepen(want)) { PrintFormat("CSignalNeural: could not deepen series to %d bar(s) - not trained rather than " "trained on a truncated or zero-filled window.", want); return false; } CMatrixDouble xy(last - first, NEURAL_FEATURES + 1); int rows = 0; double x[], label; for(int s = last; s >= first; s--) // oldest to newest: the split below is chronological { if(!BuildFeatures(x, s) || !LabelAt(s, label)) continue; bool ok = true; for(int f = 0; f < NEURAL_FEATURES; f++) if(!MathIsValidNumber(x[f])) { ok = false; break; } if(!ok) continue; for(int f = 0; f < NEURAL_FEATURES; f++) xy.Set(rows, f, x[f]); xy.Set(rows, NEURAL_FEATURES, label); rows++; } if(rows < 100) { Print("CSignalNeural: only ", rows, " usable row(s) - not trained."); return false; } //--- SAY WHAT WAS DROPPED. A sweep that silently yields a third of its candidates is exactly the //--- failure this module already shipped once, and it was invisible because the only number ever //--- printed was the one that survived. A row count means nothing without its denominator. const int candidates = last - first; if(rows < candidates) PrintFormat("CSignalNeural: %d of %d candidate bar(s) usable (%.0f%%) - %d dropped for an " "invalid ATR, a zero range or a non-finite feature.", rows, candidates, 100.0 * rows / candidates, candidates - rows); string names[]; ArrayResize(names, NEURAL_FEATURES); for(int f = 0; f < NEURAL_FEATURES; f++) names[f] = FeatureName(f); //--- Embargo of two rows: the label needs the two bars AFTER its own, so the last two training //--- rows share their answer with the first validation rows. Small here - a 2-bar window against //--- the management net's 60 - but the same leak, and a 0.5% cost to be sure of the tail. if(!m_net.Train(xy, rows, NEURAL_FEATURES, names, 2)) { Print("CSignalNeural: training refused - ", m_net.Why()); return false; } m_net.Save(ModelPath(), names, StringFormat("fractal-low survival, %s %s, %d rows", m_symbol.Name(), EnumToString((ENUM_TIMEFRAMES)m_period), rows)); return true; } //+------------------------------------------------------------------+ bool CSignalNeural::InitIndicators(CIndicators *indicators) { if(indicators == NULL || !CWarriorSignal::InitIndicators(indicators)) return false; if(!indicators.Add(GetPointer(m_atr)) || !m_atr.Create(m_symbol.Name(), m_period, m_atrPeriod)) { Print("CSignalNeural: could not create ATR"); return false; } if(!WyckoffFeedEnsure(indicators, m_symbol.Name(), m_period)) { Print("CSignalNeural: Wyckoff feed unavailable - ", g_wyckoffFeed.Why()); return false; } //--- THE ALT BLOCK IS MANDATORY, not best-effort. Sixteen of the forty-one inputs come from it, //--- and a net silently trained on sixteen zero columns is worse than no net: it would look //--- trained, pass every check, and have spent a third of its width on nothing. if(!g_altData.Load(m_symbol.Name())) { Print("CSignalNeural: alt data unavailable - ", g_altData.Why(), " - needs Common/Files/ADAltData/_D1.csv"); return false; } PrintFormat("CSignalNeural: alt data loaded, %d daily row(s), %d column(s).", g_altData.Rows(), ALT_COLUMNS); string names[]; ArrayResize(names, NEURAL_FEATURES); for(int f = 0; f < NEURAL_FEATURES; f++) names[f] = FeatureName(f); //--- LOAD ONLY. Training is deferred to TrainIfDue() - see m_minBars. A refused load (wrong //--- columns) is NOT retrained over silently: Load() has already said why, and quietly replacing //--- a model the operator may be mid-way through evaluating would destroy what is being measured. m_trained = m_net.Load(ModelPath(), names); if(m_cut <= 0.50) Print("CSignalNeural: confidence cut is 0.50 (no cut) - this module is INERT by design and " "will abstain on every bar. Raise it to vote."); if(m_trained) Print("CSignalNeural: model loaded; no training this run."); else PrintFormat("CSignalNeural: no model yet (%s). Will train once %d bars of history exist," " then every %d bars - each fit uses only bars older than itself.", m_net.Why(), m_minBars, m_retrainBars); return true; } //+------------------------------------------------------------------+ //| Train when enough history has accrued, then every m_retrainBars. | //| Called from the vote, so "now" is always the bar being decided and | //| Bars() is always history-so-far. | //+------------------------------------------------------------------+ void CSignalNeural::TrainIfDue(void) { const datetime bar = iTime(m_symbol.Name(), m_period, 0); if(bar == m_lastTrainBar) return; // at most one attempt per bar const int bars = Bars(m_symbol.Name(), m_period); if(bars < m_minBars) return; if(m_trainedAtBars > 0 && (m_retrainBars <= 0 || bars < m_trainedAtBars + m_retrainBars)) return; m_lastTrainBar = bar; //--- BACK OFF WHETHER IT SUCCEEDS OR FAILS. Without this a refusal retries on every single bar //--- and prints the same sentence hundreds of times - which buries the one line that matters and //--- costs a full history walk each time. m_trainedAtBars = bars; if(TrainFromHistory()) { m_trained = true; m_trainCount++; } } //+------------------------------------------------------------------+ //| WHAT A MODULE CAN ACTUALLY SAY, read out of the standard library | //| rather than assumed (Include\Expert\ExpertSignal.mqh): | //| | //| Direction() is m_weight*(LongCondition()-ShortCondition()), | //| line 431, and the root averages it over the number of filters | //| that answered, line 458. So a module has exactly THREE moves: | //| vote long, vote short, or return 0/0 - and 0/0 is not silence, | //| the parent still counts it in `number` and it dilutes everyone | //| else's vote. | //| | //| THERE IS NO PER-SIDE VETO. EMPTY_VALUE is not one. Returned by | //| a filter it makes the root return EMPTY_VALUE immediately | //| (line 449-450), and the top-level m_direction==EMPTY_VALUE | //| makes CheckOpenLong, CheckOpenShort, CheckCloseLong AND | //| CheckCloseShort all return false (lines 231, 254, 317, 340). | //| It is the NO-ACTION state - the constructor initialises | //| m_direction to EMPTY_VALUE as "nothing computed yet" (line | //| 103). So one filter returning it silences all twelve others for | //| that bar, and suppresses signal-driven exits too. | //| | //| I built the refusal that way and measured it: -98.49 over 46 | //| trades against +81.30 over 60 with the module off. It is the | //| bluntest instrument in the framework and the measurement does not | //| ask for it - so it is gone. | //| | //| WHAT IS LEFT IS THE HONEST TENSION. The quintile study measured | //| mean R BY QUINTILE TAKEN LONG: -0.230 / -0.089 / -0.026 / +0.002 | //| / +0.017. Its only claim is "the bottom quintile is a bad LONG". | //| Expressing that as a short vote assumes the payoff is | //| antisymmetric - that what a long loses a short earns - which the | //| study never tested and the spread contradicts. But the arithmetic | //| above offers no way to say "not long" without saying "short": | //| the same subtraction that pulls the average below the long | //| threshold pushes it toward the short one. | //| | //| So the short vote stays, and it is labelled for what it is: an | //| ASSUMPTION, not a measurement. Whether it pays is the question | //| the multi-symbol sweep exists to answer, because at 60 trades the | //| standard error on the mean is ~2.3 per trade and every result in | //| this file so far - +81.30, +30.44, -98.49 - sits inside it. | //+------------------------------------------------------------------+ double CSignalNeural::Score(void) { //--- CONF_50 MEANS OFF, and it has to mean off HERE or it means the opposite. The bands are //--- `p >= cut` and `p <= 1-cut`; at cut = 0.50 those are `p >= 0.5` and `p <= 0.5`, one of //--- which is true for EVERY p - so the module would speak on every bar instead of abstaining. //--- A dead band at 0.50 is not a tidy-up: without it, "no cut" is the most opinionated setting. if(!m_trained || m_cut <= 0.50) return -1.0; const datetime bar = iTime(m_symbol.Name(), m_period, 0); if(bar == m_scoreBar) return m_score; m_scoreBar = bar; m_score = -1.0; double x[]; if(!BuildFeatures(x, 1)) return -1.0; const double p = m_net.Score(x); m_score = (p < 0.0) ? -1.0 : p; return m_score; } //+------------------------------------------------------------------+ int CSignalNeural::LongCondition(void) { TrainIfDue(); const double p = Score(); //--- The confirmation, deliberately small: the top quintile measured +0.017 R, breakeven before //--- costs. A module returning a big number here would claim an edge the measurement lacks. if(p >= m_cut) { m_active_pattern = "Pattern_1"; m_active_direction = "Buy"; return m_pattern_1; } return 0; } //+------------------------------------------------------------------+ int CSignalNeural::ShortCondition(void) { //--- TrainIfDue() is NOT called here: LongCondition() runs first on every evaluation and has //--- already called it. Score() is cached per bar, so both sides read the same number. const double p = Score(); //--- THE ASSUMPTION, carrying the heavier weight because the bottom quintile is where the //--- measured separation is. See the note above: "bad long" -> "good short" is a step the //--- quintile study does not license, and this is the line that takes it. if(p >= 0.0 && p <= 1.0 - m_cut) { m_active_pattern = "Pattern_0"; m_active_direction = "Sell"; return m_pattern_0; } return 0; } #endif // WARRIOR_SIGNALNEURAL_MQH