forked from animatedread/Warrior_EA
36345540ca
- Adam: replace raw neuron gradient with per-weight gradient `g = gradient * outputVal` so that each input weight receives its own mt/vt, fixing a bug where all weights of a neuron were updated identically (uniform scaling). - `calcHiddenGradients`: compute `gradient = sumDOW * activationDerivative` directly instead of routing through `calcOutputGradients` with a clamped pseudo-target, which corrupted hidden-layer error signals (especially for PRELU activations).
162 lines
8.2 KiB
MQL5
162 lines
8.2 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronCPU.mqh |
|
|
//| AnimateDread |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
//| CNeuron - the plain-CPU (no DLL/OpenCL/DirectML) dense neuron, |
|
|
//| last-resort fallback tier. Needs CNeuronBase (AI\Network.mqh) |
|
|
//| and CLayer/CConnection already declared - included from |
|
|
//| Network.mqh at the exact point CNeuron used to sit, so ordering |
|
|
//| matches the original file. Extracted verbatim (SOLID cleanup) - |
|
|
//| no logic changes. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuron : public CNeuronBase
|
|
{
|
|
private:
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual bool updateInputWeights(CLayer *&prevLayer);
|
|
|
|
public:
|
|
CNeuron(void) {};
|
|
~CNeuron(void) { Connections.Shutdown(); }
|
|
//---
|
|
virtual bool calcOutputGradients(double targetVals);
|
|
virtual double sumDOW(CLayer *&nextLayer) ;
|
|
virtual int Type(void) const { return defNeuron; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuron::updateInputWeights(CLayer *&prevLayer)
|
|
{
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
int total = prevLayer.Total();
|
|
for(int n = 0; n < total && !IsStopped(); n++)
|
|
{
|
|
CNeuron *neuron = prevLayer.At(n);
|
|
CConnection *con = neuron.Connections.At(m_myIndex);
|
|
if(CheckPointer(con) == POINTER_INVALID)
|
|
continue;
|
|
if(optimization == SGD)
|
|
con.weight += con.deltaWeight = (gradient != 0 ? eta * neuron.getOutputVal() * gradient : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0);
|
|
else
|
|
{
|
|
// Per-WEIGHT gradient (neuron gradient x presynaptic output), matching the SGD branch above
|
|
// and every native backend's Adam kernel (`grad = g[i] * inp` in CPU_UpdateWeightsAdam).
|
|
// Previously fed the raw neuron gradient alone, giving every input weight of a neuron an
|
|
// IDENTICAL mt/vt/delta - the weight vector could only move uniformly across all inputs,
|
|
// so this tier couldn't learn per-feature structure at all.
|
|
double g = gradient * neuron.getOutputVal();
|
|
con.mt = b1 * con.mt + (1 - b1) * g;
|
|
con.vt = b2 * con.vt + (1 - b2) * g * g + 0.00000001;
|
|
con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
|
|
// No sign-agreement gate (removed 2026-07): gating each step on agreement with the CURRENT
|
|
// sample's gradient sign rectified the one-hot softmax-CCE stream - rare large true-class
|
|
// positives (1/3 of samples), frequent small wrong-class negatives (2/3) - into a permanent
|
|
// downward ratchet on every output neuron, sinking all three logits into sigmoid saturation
|
|
// together (the all-Neutral collapse; IS error frozen at sqrt(1/3)=0.58). The stale-step
|
|
// overshoot it guarded against is covered by the MAX_WEIGHT_DELTA clip, AdamW WEIGHT_DECAY
|
|
// and shuffle-interleaved oversampling. Removed from all four backends in sync (WarriorCPU
|
|
// .cpp / WarriorDML.cpp / Network.cl mirror this).
|
|
con.weight += con.deltaWeight;
|
|
}
|
|
// Mirrors AI\Network.cl's MAX_WEIGHT clamp (see that file's Conv/LSTM Adam kernels) - without
|
|
// it a gradient spike (e.g. from class-balance oversampling replaying the same rare-class bar
|
|
// several times in a row - see Train()'s reps loop) can drive a weight to +-Infinity; the next
|
|
// Adam step then divides Infinity by Infinity (mt/vt both Inf) producing NaN, which propagates
|
|
// through every FeedForward sum that touches it and never recovers, since Adam(NaN)=NaN forever
|
|
// after. That silently freezes the whole network's output at NaN - manifesting as every bar
|
|
// classifying to whatever the "can't decide" default is (e.g. all-Neutral, 0 Buy/Sell) with no
|
|
// error ever surfaced, since NaN comparisons are simply always false.
|
|
con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
|
|
}
|
|
if(optimization == ADAM)
|
|
t++;
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
double CNeuron::sumDOW(CLayer *&nextLayer)
|
|
{
|
|
double sum = 0.0;
|
|
int total = nextLayer.Total() - 1;
|
|
for(int n = 0; n < total; n++)
|
|
{
|
|
CConnection *con = Connections.At(n);
|
|
if(CheckPointer(con) == POINTER_INVALID)
|
|
continue;
|
|
double weight = con.weight;
|
|
if(weight != 0)
|
|
{
|
|
CNeuron *neuron = nextLayer.At(n);
|
|
sum += weight * neuron.gradient;
|
|
}
|
|
}
|
|
return sum;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuron::calcHiddenGradients(CLayer *&nextLayer)
|
|
{
|
|
// sumDOW * activation derivative, matching CNeuronConv::calcHiddenGradients (AI\Network.mqh) and
|
|
// the CalcHiddenGradient kernels in all three native backends. Previously routed through
|
|
// calcOutputGradients(sumDOW + outputVal), whose +-1 target clamp - added later for the OUTPUT
|
|
// layer's -1/0/1 targets - silently corrupted hidden gradients: for a hidden PRELU neuron with
|
|
// |outputVal| > 1 (routine, PRELU is unbounded above) the clamp discarded the backpropagated
|
|
// sumDOW entirely and replaced it with a spurious (+-1 - outputVal) magnitude penalty, and even
|
|
// inside [-1,1] it truncated any error signal pushing the pseudo-target past +-1.
|
|
gradient = sumDOW(nextLayer) * activationFunctionDerivative(outputVal);
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuron::calcOutputGradients(double targetVal)
|
|
{
|
|
// Deliberately NOT multiplied by activationFunctionDerivative(outputVal):
|
|
// for TANH that factor is (1-out^2), which vanishes as outputVal
|
|
// approaches +-1 - exactly where this neuron needs to converge for a +-1
|
|
// target (e.g. the buy/sell extremes of a single-neuron regression head),
|
|
// stalling training right when it matters most. See the matching fix in
|
|
// AI\Network.cl / DirectML\WarriorDML.cpp / DirectML\WarriorCPU.cpp.
|
|
double delta = (targetVal > 1 ? 1 : targetVal < -1 ? -1 : targetVal) - outputVal;
|
|
gradient = delta;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuron::feedForward(CLayer *prevLayer)
|
|
{
|
|
if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Type() != defLayer)
|
|
return false;
|
|
//---
|
|
prevVal = outputVal;
|
|
double sum = 0.0;
|
|
int total = prevLayer.Total();
|
|
for(int n = 0; n < total && !IsStopped(); n++)
|
|
{
|
|
CNeuron *temp = prevLayer.At(n);
|
|
double val = temp.getOutputVal();
|
|
if(val != 0)
|
|
{
|
|
CConnection *con = temp.Connections.At(m_myIndex);
|
|
if(CheckPointer(con) == POINTER_INVALID)
|
|
continue;
|
|
sum += val * con.weight;
|
|
}
|
|
}
|
|
outputVal = activationFunction(MathMin(MathMax(sum, -18), 18));
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|