Warrior_EA/AI/NeuronCPU.mqh

//+------------------------------------------------------------------+
//|                                                  NeuronCPU.mqh   |
//|                                                     AnimateDread |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
//| CNeuron - the plain-CPU (no DLL/OpenCL/DirectML) dense neuron,   |
//| last-resort fallback tier. Needs CNeuronBase (AI\Network.mqh)    |
//| and CLayer/CConnection already declared - included from          |
//| Network.mqh at the exact point CNeuron used to sit, so ordering  |
//| matches the original file. Extracted verbatim (SOLID cleanup) -  |
//| no logic changes.                                                 |
//+------------------------------------------------------------------+
class CNeuron  :  public CNeuronBase
  {
private:
   virtual bool      feedForward(CLayer *prevLayer);
   virtual bool      calcHiddenGradients(CLayer *&nextLayer);
   virtual bool      updateInputWeights(CLayer *&prevLayer);

public:
                     CNeuron(void)  {};
                    ~CNeuron(void) { Connections.Shutdown(); }
   //---
   virtual bool      calcOutputGradients(double targetVals);
   virtual double    sumDOW(CLayer *&nextLayer) ;
   virtual int       Type(void)   const   {  return defNeuron;   }
  };
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuron::updateInputWeights(CLayer *&prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID)
      return false;
//---
   double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
   double gradient2 = gradient * gradient;
   int total = prevLayer.Total();
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      CNeuron *neuron = prevLayer.At(n);
      CConnection *con = neuron.Connections.At(m_myIndex);
      if(CheckPointer(con) == POINTER_INVALID)
         continue;
      if(optimization == SGD)
         con.weight += con.deltaWeight = (gradient != 0 ? eta * neuron.getOutputVal() * gradient : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0);
      else
        {
         con.mt = b1 * con.mt + (1 - b1) * gradient;
         con.vt = b2 * con.vt + (1 - b2) * gradient2 + 0.00000001;
         con.deltaWeight = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, lt * con.mt / sqrt(con.vt) - lt * WEIGHT_DECAY * con.weight));
         // Sign-agreement gate (mirrors Dmitriy Gizlyk's reference NeuroNet.cl Adam kernel): only
         // apply the step if it agrees with the CURRENT raw gradient's direction. Momentum built
         // from several back-to-back near-identical oversampled gradients can keep pushing a weight
         // the same way for one extra step after a genuinely different bar flips the raw gradient's
         // sign - applying that stale-direction step is the overshoot mechanism implicated in the
         // OOS collapse cycles. mt/vt still update normally either way; this only ever suppresses
         // the contradicting steps.
         if(con.deltaWeight * gradient > 0)
            con.weight += con.deltaWeight;
        }
      // Mirrors AI\Network.cl's MAX_WEIGHT clamp (see that file's Conv/LSTM Adam kernels) - without
      // it a gradient spike (e.g. from class-balance oversampling replaying the same rare-class bar
      // several times in a row - see Train()'s reps loop) can drive a weight to +-Infinity; the next
      // Adam step then divides Infinity by Infinity (mt/vt both Inf) producing NaN, which propagates
      // through every FeedForward sum that touches it and never recovers, since Adam(NaN)=NaN forever
      // after. That silently freezes the whole network's output at NaN - manifesting as every bar
      // classifying to whatever the "can't decide" default is (e.g. all-Neutral, 0 Buy/Sell) with no
      // error ever surfaced, since NaN comparisons are simply always false.
      con.weight = MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, con.weight));
     }
   if(optimization == ADAM)
      t++;
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
double CNeuron::sumDOW(CLayer *&nextLayer)
  {
   double sum = 0.0;
   int total = nextLayer.Total() - 1;
   for(int n = 0; n < total; n++)
     {
      CConnection *con = Connections.At(n);
      if(CheckPointer(con) == POINTER_INVALID)
         continue;
      double weight = con.weight;
      if(weight != 0)
        {
         CNeuron *neuron = nextLayer.At(n);
         sum += weight * neuron.gradient;
        }
     }
   return sum;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuron::calcHiddenGradients(CLayer *&nextLayer)
  {
   double targetVal = sumDOW(nextLayer) + outputVal;
   return calcOutputGradients(targetVal);
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuron::calcOutputGradients(double targetVal)
  {
   // Deliberately NOT multiplied by activationFunctionDerivative(outputVal):
   // for TANH that factor is (1-out^2), which vanishes as outputVal
   // approaches +-1 - exactly where this neuron needs to converge for a +-1
   // target (e.g. the buy/sell extremes of a single-neuron regression head),
   // stalling training right when it matters most. See the matching fix in
   // AI\Network.cl / DirectML\WarriorDML.cpp / DirectML\WarriorCPU.cpp.
   double delta = (targetVal > 1 ? 1 : targetVal < -1 ? -1 : targetVal) - outputVal;
   gradient = delta;
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
bool CNeuron::feedForward(CLayer *prevLayer)
  {
   if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Type() != defLayer)
      return false;
//---
   prevVal = outputVal;
   double sum = 0.0;
   int total = prevLayer.Total();
   for(int n = 0; n < total && !IsStopped(); n++)
     {
      CNeuron *temp = prevLayer.At(n);
      double val = temp.getOutputVal();
      if(val != 0)
        {
         CConnection *con = temp.Connections.At(m_myIndex);
         if(CheckPointer(con) == POINTER_INVALID)
            continue;
         sum += val * con.weight;
        }
     }
   outputVal = activationFunction(MathMin(MathMax(sum, -18), 18));
//---
   return true;
  }
//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+