2026-08-01 11:27:28 -04:00
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| NetWeights.mqh |
|
|
|
|
|
//| |
|
|
|
|
|
//| CNet weight operations: EMA blend, in-memory snapshot/restore, |
|
|
|
|
|
//| per-layer learning report, topology contract checks. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Included from AI\Network.mqh AFTER every class declaration - |
|
|
|
|
|
//| bodies only, no declarations. Relocation is behaviour-neutral by |
|
|
|
|
|
//| construction: nothing here is reachable until Network.mqh ends. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
#ifndef WARRIOR_AI_IMPL_NETWEIGHTS_MQH
|
|
|
|
|
#define WARRIOR_AI_IMPL_NETWEIGHTS_MQH
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| See this method's declaration comment for the EMA shadow-weight |
|
|
|
|
|
//| deployment rationale. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::BlendWeightsFrom(CNet &live, double tau)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(live.layers) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
//--- Skip accounting. Every branch below blends only when BOTH sides hand back a weight block, and
|
|
|
|
|
//--- silently does nothing otherwise - deliberate, so a partial topology mismatch degrades instead of
|
|
|
|
|
//--- corrupting unrelated layers. But "silently" made a real defect invisible for 343 eras on SP500 H1
|
|
|
|
|
//--- (2026-07-31): the HYBRID shadow's LSTM layer had never run a forward pass, so its WeightsLSTM was
|
|
|
|
|
//--- still NULL, getWeightsLSTM() returned 0, and the blend skipped a 24,704-weight layer on EVERY era
|
|
|
|
|
//--- while reporting success. It showed up only as a shadow .nnw 791,120 bytes smaller than its live
|
|
|
|
|
//--- net - exactly the LSTM weight block and its Adam moments - which nothing was watching. A skip is
|
|
|
|
|
//--- never normal on a shadow cloned from live, so say so ONCE per net rather than never.
|
|
|
|
|
int skipped = 0;
|
|
|
|
|
int skippedLayer = -1, skippedType = 0;
|
|
|
|
|
int layerTotal = MathMin(layers.Total(), live.layers.Total());
|
|
|
|
|
for(int l = 0; l < layerTotal; l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *shadowLayer = layers.At(l);
|
|
|
|
|
CLayer *liveLayer = live.layers.At(l);
|
|
|
|
|
if(CheckPointer(shadowLayer) == POINTER_INVALID || CheckPointer(liveLayer) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
int neuronTotal = MathMin(shadowLayer.Total(), liveLayer.Total());
|
|
|
|
|
for(int n = 0; n < neuronTotal; n++)
|
|
|
|
|
{
|
|
|
|
|
//--- Type() first, through the true common base (CObject) - NOT through a CNeuronBaseOCL*-typed
|
|
|
|
|
//--- pointer. On the plain-CPU tier (no OpenCL/DirectML - the only option on Marketplace, which
|
|
|
|
|
//--- forbids DLLs) CNet::CNet() builds legacy CNeuronBase-hierarchy neurons (CNeuron/CNeuronConv/
|
|
|
|
|
//--- CNeuronPool/CNeuronLSTM), an UNRELATED class hierarchy from CNeuronBaseOCL/CNeuronConvOCL/
|
|
|
|
|
//--- CNeuronLSTMOCL. Assigning one of those objects to a CNeuronBaseOCL* and calling its virtual
|
|
|
|
|
//--- methods (as this used to do unconditionally) reads the wrong vtable/member layout - undefined
|
|
|
|
|
//--- behaviour, not just a silent no-op, every single online-learning step on that tier.
|
|
|
|
|
CObject *shadowObj = shadowLayer.At(n);
|
|
|
|
|
CObject *liveObj = liveLayer.At(n);
|
|
|
|
|
if(CheckPointer(shadowObj) == POINTER_INVALID || CheckPointer(liveObj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
if(shadowObj.Type() != liveObj.Type())
|
|
|
|
|
continue;
|
|
|
|
|
double shadowW[], liveW[];
|
|
|
|
|
switch(shadowObj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *shadowNeuron = shadowObj;
|
|
|
|
|
CNeuronBaseOCL *liveNeuron = liveObj;
|
|
|
|
|
int gotShadow = shadowNeuron.getWeights(shadowW);
|
|
|
|
|
int gotLive = liveNeuron.getWeights(liveW);
|
|
|
|
|
if(gotShadow > 0 && gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
|
|
|
|
|
for(int wi = 0; wi < wt; wi++)
|
|
|
|
|
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
|
|
|
|
|
shadowNeuron.setWeights(shadowW);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
//--- Only an ASYMMETRY is a defect: live has a weight block, the shadow does not.
|
|
|
|
|
//--- Both-empty is normal and common - a dense layer whose successor owns the weight
|
|
|
|
|
//--- matrix (see CNeuronBaseOCL::Init's numOutputs>0 guard) legitimately has none, which
|
|
|
|
|
//--- is also why LayerLearningReport prints NOWEIGHTS for those layers. Warning on that
|
|
|
|
|
//--- fired 5-6 times per net on every topology (2026-07-31) and was pure noise.
|
|
|
|
|
if(gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
skipped++;
|
|
|
|
|
skippedLayer = l;
|
|
|
|
|
skippedType = shadowObj.Type();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
{
|
|
|
|
|
//--- getWeightsBN packs the outgoing dense matrix and gamma/beta/statistics into one
|
|
|
|
|
//--- flat array, so the shared blend loop below needs no special case of its own.
|
|
|
|
|
CNeuronBatchNormOCL *shadowBN = shadowObj;
|
|
|
|
|
CNeuronBatchNormOCL *liveBN = liveObj;
|
|
|
|
|
int gotShadow = shadowBN.getWeightsBN(shadowW);
|
|
|
|
|
int gotLive = liveBN.getWeightsBN(liveW);
|
|
|
|
|
if(gotShadow > 0 && gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
|
|
|
|
|
for(int wi = 0; wi < wt; wi++)
|
|
|
|
|
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
|
|
|
|
|
shadowBN.setWeightsBN(shadowW);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
//--- Only an ASYMMETRY is a defect: live has a weight block, the shadow does not.
|
|
|
|
|
//--- Both-empty is normal and common - a dense layer whose successor owns the weight
|
|
|
|
|
//--- matrix (see CNeuronBaseOCL::Init's numOutputs>0 guard) legitimately has none, which
|
|
|
|
|
//--- is also why LayerLearningReport prints NOWEIGHTS for those layers. Warning on that
|
|
|
|
|
//--- fired 5-6 times per net on every topology (2026-07-31) and was pure noise.
|
|
|
|
|
if(gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
skipped++;
|
|
|
|
|
skippedLayer = l;
|
|
|
|
|
skippedType = shadowObj.Type();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronConvOCL *shadowConv = shadowObj;
|
|
|
|
|
CNeuronConvOCL *liveConv = liveObj;
|
|
|
|
|
int gotShadow = shadowConv.getWeightsConv(shadowW);
|
|
|
|
|
int gotLive = liveConv.getWeightsConv(liveW);
|
|
|
|
|
if(gotShadow > 0 && gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
|
|
|
|
|
for(int wi = 0; wi < wt; wi++)
|
|
|
|
|
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
|
|
|
|
|
shadowConv.setWeightsConv(shadowW);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
//--- Only an ASYMMETRY is a defect: live has a weight block, the shadow does not.
|
|
|
|
|
//--- Both-empty is normal and common - a dense layer whose successor owns the weight
|
|
|
|
|
//--- matrix (see CNeuronBaseOCL::Init's numOutputs>0 guard) legitimately has none, which
|
|
|
|
|
//--- is also why LayerLearningReport prints NOWEIGHTS for those layers. Warning on that
|
|
|
|
|
//--- fired 5-6 times per net on every topology (2026-07-31) and was pure noise.
|
|
|
|
|
if(gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
skipped++;
|
|
|
|
|
skippedLayer = l;
|
|
|
|
|
skippedType = shadowObj.Type();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronLSTMOCL *shadowLstm = shadowObj;
|
|
|
|
|
CNeuronLSTMOCL *liveLstm = liveObj;
|
|
|
|
|
int gotShadow = shadowLstm.getWeightsLSTM(shadowW);
|
|
|
|
|
int gotLive = liveLstm.getWeightsLSTM(liveW);
|
|
|
|
|
//--- SELF-HEAL, 2026-07-31. EnsureShadowNet() can bootstrap the clone from
|
|
|
|
|
//--- RefreshLatestSignal BEFORE the live net has run a single forward pass, and
|
|
|
|
|
//--- CNeuronLSTMOCL::Save omits every LSTM buffer for a layer in that state - so the shadow
|
|
|
|
|
//--- came back with no WeightsLSTM and, because only the LIVE net ever runs forward, never
|
|
|
|
|
//--- got one. The blend then skipped a 24,704-weight layer on EVERY era while returning
|
|
|
|
|
//--- true, for a whole 343-era run. It was visible only as a shadow .nnw 791,120 bytes
|
|
|
|
|
//--- short of its live net (exactly the LSTM block plus its Adam moments), and the shadow
|
|
|
|
|
//--- is what live inference and deployment read.
|
|
|
|
|
//--- An EMA whose accumulator does not exist yet must seed at the FIRST observation, not
|
|
|
|
|
//--- blend tau of it into freshly randomized weights - hence the outright copy.
|
|
|
|
|
if(gotShadow <= 0 && gotLive > 0 && shadowLstm.AdoptShapeFrom(liveLstm))
|
|
|
|
|
{
|
|
|
|
|
shadowLstm.setWeightsLSTM(liveW);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
if(gotShadow > 0 && gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
int wt = MathMin(ArraySize(shadowW), ArraySize(liveW));
|
|
|
|
|
for(int wi = 0; wi < wt; wi++)
|
|
|
|
|
shadowW[wi] = (1.0 - tau) * shadowW[wi] + tau * liveW[wi];
|
|
|
|
|
shadowLstm.setWeightsLSTM(shadowW);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
//--- Only an ASYMMETRY is a defect: live has a weight block, the shadow does not.
|
|
|
|
|
//--- Both-empty is normal and common - a dense layer whose successor owns the weight
|
|
|
|
|
//--- matrix (see CNeuronBaseOCL::Init's numOutputs>0 guard) legitimately has none, which
|
|
|
|
|
//--- is also why LayerLearningReport prints NOWEIGHTS for those layers. Warning on that
|
|
|
|
|
//--- fired 5-6 times per net on every topology (2026-07-31) and was pure noise.
|
|
|
|
|
if(gotLive > 0)
|
|
|
|
|
{
|
|
|
|
|
skipped++;
|
|
|
|
|
skippedLayer = l;
|
|
|
|
|
skippedType = shadowObj.Type();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
//--- Plain-CPU tier (no backend at all): dense/conv/pool/LSTM legacy neurons store weights
|
|
|
|
|
//--- per-connection (CConnection.weight) rather than in one contiguous buffer - blend those
|
|
|
|
|
//--- directly instead of silently dropping the shadow-EMA deployment on this tier.
|
|
|
|
|
case defNeuron:
|
|
|
|
|
case defNeuronConv:
|
|
|
|
|
case defNeuronPool:
|
|
|
|
|
case defNeuronLSTM:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBase *shadowBase = shadowObj;
|
|
|
|
|
CNeuronBase *liveBase = liveObj;
|
|
|
|
|
CArrayCon *shadowCon = shadowBase.getConnections();
|
|
|
|
|
CArrayCon *liveCon = liveBase.getConnections();
|
|
|
|
|
if(CheckPointer(shadowCon) == POINTER_INVALID || CheckPointer(liveCon) == POINTER_INVALID)
|
|
|
|
|
break;
|
|
|
|
|
int ct = MathMin(shadowCon.Total(), liveCon.Total());
|
|
|
|
|
for(int ci = 0; ci < ct; ci++)
|
|
|
|
|
{
|
|
|
|
|
CConnection *sc = shadowCon.At(ci);
|
|
|
|
|
CConnection *lc = liveCon.At(ci);
|
|
|
|
|
if(CheckPointer(sc) == POINTER_INVALID || CheckPointer(lc) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
sc.weight = (1.0 - tau) * sc.weight + tau * lc.weight;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if(skipped > 0 && !m_blendSkipLogged)
|
|
|
|
|
{
|
|
|
|
|
m_blendSkipLogged = true;
|
|
|
|
|
Print("CNet::BlendWeightsFrom: WARNING - ", skipped, " weight block(s) could not be blended into the EMA shadow ",
|
|
|
|
|
"(last: layer ", skippedLayer, ", neuron type ", skippedType, "). The shadow is what live inference and ",
|
|
|
|
|
"deployment read, so those layers are NOT tracking the trained model - they keep whatever they were ",
|
|
|
|
|
"initialized with. A shadow cloned from the live net should never skip; investigate rather than ignore.");
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Per-layer "is this layer actually learning?" report. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Returns one token per layer: <type><index>:<|W|>(<relative change |
|
|
|
|
|
//| since the previous call>). A layer whose relative change is ~0 era|
|
|
|
|
|
//| after era is NOT TRAINING, whatever the loss curve says. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Why this exists: two separate incidents in this engine presented |
|
|
|
|
|
//| identically - a flat metric with the model retreating to the |
|
|
|
|
|
//| majority class - and in both the real fault was that one stage |
|
|
|
|
|
//| received no usable gradient while every other stage trained |
|
|
|
|
|
//| normally. Neither the loss, the accuracy, nor the per-class recall|
|
|
|
|
|
//| can distinguish "this layer is frozen" from "this architecture |
|
|
|
|
|
//| does not suit the data", and guessing between those two costs a |
|
|
|
|
|
//| full retrain per guess. The weight norms distinguish them directly.|
|
|
|
|
|
//| See CNet::backProp's layer-1 LSTM special case for the shape the |
|
|
|
|
|
//| first such bug took. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
string CNet::LayerLearningReport(void)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return "";
|
|
|
|
|
int total = layers.Total();
|
|
|
|
|
if(ArraySize(m_prevLayerNorm) != total)
|
|
|
|
|
{
|
|
|
|
|
ArrayResize(m_prevLayerNorm, total);
|
|
|
|
|
ArrayInitialize(m_prevLayerNorm, -1.0);
|
|
|
|
|
}
|
|
|
|
|
//--- One slot per layer index, so At(l) lines up with the loop below without bookkeeping. Layers that
|
|
|
|
|
//--- own no weights just keep an empty array.
|
|
|
|
|
if(CheckPointer(m_prevLayerWeights) == POINTER_INVALID)
|
|
|
|
|
m_prevLayerWeights = new CArrayObj();
|
|
|
|
|
if(CheckPointer(m_prevLayerWeights) != POINTER_INVALID)
|
|
|
|
|
while(m_prevLayerWeights.Total() < total)
|
|
|
|
|
if(!m_prevLayerWeights.Add(new CArrayDouble()))
|
|
|
|
|
break;
|
|
|
|
|
string report = "";
|
|
|
|
|
for(int l = 0; l < total; l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID || layer.Total() <= 0)
|
|
|
|
|
continue;
|
|
|
|
|
CObject *obj = layer.At(0);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
double w[];
|
|
|
|
|
int got = 0;
|
|
|
|
|
string tag = "?";
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *n = (CNeuronBaseOCL*)obj;
|
|
|
|
|
got = n.getWeights(w);
|
|
|
|
|
tag = "dense";
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronConvOCL *c = (CNeuronConvOCL*)obj;
|
|
|
|
|
got = c.getWeightsConv(w);
|
|
|
|
|
tag = "conv";
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj;
|
|
|
|
|
got = ls.getWeightsLSTM(w);
|
|
|
|
|
tag = "lstm";
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
|
|
|
|
|
got = bn.getWeightsBN(w);
|
|
|
|
|
tag = "bn";
|
2026-08-02 08:12:47 -04:00
|
|
|
//--- getWeightsBN packs FOUR different kinds of number into one array: the outgoing dense
|
|
|
|
|
//--- matrix, the learned gamma/beta, the running mean/variance, and the Adam moment
|
|
|
|
|
//--- buffers. A single norm over all of them cannot say which one is moving - and on
|
|
|
|
|
//--- 2026-08-02 that ambiguity was the difference between "the weights are diverging"
|
|
|
|
|
//--- (an optimizer problem) and "the running variance is tracking activations that grew"
|
|
|
|
|
//--- (a scaling problem), which need opposite fixes. PAI reached bn3:465,684 with no way
|
|
|
|
|
//--- to tell them apart. So break the block down: the sub-norms cost one pass and turn a
|
|
|
|
|
//--- guess into a reading.
|
|
|
|
|
int oCount = bn.BatchOptionsTotal();
|
|
|
|
|
int wCount = got - oCount;
|
|
|
|
|
if(oCount > 0 && wCount >= 0)
|
|
|
|
|
{
|
|
|
|
|
double nW = 0.0, nGamma = 0.0, nBeta = 0.0, nMean = 0.0, nVar = 0.0, nAdam = 0.0;
|
|
|
|
|
for(int i = 0; i < wCount; i++)
|
|
|
|
|
nW += w[i] * w[i];
|
|
|
|
|
for(int o = 0; o < oCount; o++)
|
|
|
|
|
{
|
|
|
|
|
double v = w[wCount + o];
|
|
|
|
|
switch(o % BN_OPT_STRIDE)
|
|
|
|
|
{
|
|
|
|
|
case BN_OPT_MEAN: nMean += v * v; break;
|
|
|
|
|
case BN_OPT_VAR: nVar += v * v; break;
|
|
|
|
|
case BN_OPT_GAMMA: nGamma += v * v; break;
|
|
|
|
|
case BN_OPT_BETA: nBeta += v * v; break;
|
|
|
|
|
case BN_OPT_MG:
|
|
|
|
|
case BN_OPT_MB:
|
|
|
|
|
case BN_OPT_VG:
|
|
|
|
|
case BN_OPT_VB: nAdam += v * v; break;
|
|
|
|
|
default: break; // BN_OPT_NX is a forward-pass scratch value
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
report += StringFormat(" bn%d[W %.3g|g %.3g|b %.3g|mean %.3g|var %.3g|adam %.3g]",
|
|
|
|
|
l, MathSqrt(nW), MathSqrt(nGamma), MathSqrt(nBeta),
|
|
|
|
|
MathSqrt(nMean), MathSqrt(nVar), MathSqrt(nAdam));
|
|
|
|
|
}
|
2026-08-01 11:27:28 -04:00
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronPoolOCL:
|
|
|
|
|
//--- No parameters of its own. Named anyway so the report shows the real layer order.
|
|
|
|
|
report += " pool" + IntegerToString(l) + ":-";
|
|
|
|
|
continue;
|
|
|
|
|
default:
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if(got <= 0)
|
|
|
|
|
{
|
|
|
|
|
report += " " + tag + IntegerToString(l) + ":NOWEIGHTS";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
double sum = 0.0;
|
|
|
|
|
for(int i = 0; i < got; i++)
|
|
|
|
|
sum += w[i] * w[i];
|
|
|
|
|
double norm = MathSqrt(sum);
|
|
|
|
|
double prev = m_prevLayerNorm[l];
|
|
|
|
|
m_prevLayerNorm[l] = norm;
|
|
|
|
|
//--- |dW|: norm of the elementwise change since the last report. Only meaningful when the previous
|
|
|
|
|
//--- vector has the same length (a rebuilt topology invalidates it), hence the size check.
|
|
|
|
|
double stepRel = -1.0;
|
|
|
|
|
CArrayDouble *prevW = (CheckPointer(m_prevLayerWeights) != POINTER_INVALID && l < m_prevLayerWeights.Total()
|
|
|
|
|
? (CArrayDouble*)m_prevLayerWeights.At(l) : NULL);
|
|
|
|
|
if(CheckPointer(prevW) != POINTER_INVALID)
|
|
|
|
|
{
|
|
|
|
|
if(prevW.Total() == got && prev > 0.0)
|
|
|
|
|
{
|
|
|
|
|
double d2 = 0.0;
|
|
|
|
|
for(int i = 0; i < got; i++)
|
|
|
|
|
{
|
|
|
|
|
double d = w[i] - prevW.At(i);
|
|
|
|
|
d2 += d * d;
|
|
|
|
|
}
|
|
|
|
|
stepRel = MathSqrt(d2) / prev;
|
|
|
|
|
}
|
|
|
|
|
prevW.Clear();
|
|
|
|
|
for(int i = 0; i < got; i++)
|
|
|
|
|
prevW.Add(w[i]);
|
|
|
|
|
}
|
|
|
|
|
if(prev < 0.0)
|
|
|
|
|
{
|
|
|
|
|
report += " " + tag + IntegerToString(l) + ":" + DoubleToString(norm, 3) + "(init)";
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
//--- Relative, so a wide layer and a narrow one are comparable at a glance. Printed as
|
|
|
|
|
//--- norm(d|W| / |dW|): the FIRST is how much the length changed, the SECOND how far the vector
|
|
|
|
|
//--- actually moved. Decay-only shows the two roughly EQUAL with the norm falling; a learning
|
|
|
|
|
//--- layer shows the second clearly larger. See m_prevLayerWeights' declaration comment.
|
|
|
|
|
double rel = (prev > 0.0 ? MathAbs(norm - prev) / prev : 0.0);
|
|
|
|
|
report += " " + tag + IntegerToString(l) + ":" + DoubleToString(norm, 3) +
|
|
|
|
|
"(" + DoubleToString(100.0 * rel, 3) + "%/" +
|
|
|
|
|
(stepRel < 0.0 ? "n/a" : DoubleToString(100.0 * stepRel, 3) + "%") + ")";
|
|
|
|
|
}
|
|
|
|
|
return report;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Snapshot every neuron's weights into host memory (see the header |
|
|
|
|
|
//| declaration). Layer-major order; one CArrayDouble per neuron. Used|
|
|
|
|
|
//| as the mid-run best-era checkpoint - restored by RestoreWeights().|
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::CaptureWeights(void)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
if(CheckPointer(m_weightSnapshot) == POINTER_INVALID)
|
|
|
|
|
{
|
|
|
|
|
m_weightSnapshot = new CArrayObj();
|
|
|
|
|
if(CheckPointer(m_weightSnapshot) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
m_weightSnapshot.Clear(); // FreeMode deletes the previous snapshot's per-neuron arrays
|
|
|
|
|
m_haveWeightSnapshot = false;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
double w[];
|
|
|
|
|
int got = 0;
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
|
|
|
|
|
got = neuron.getWeights(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
{
|
|
|
|
|
//--- Dense matrix + gamma/beta/statistics as one array - see getWeightsBN. Without this
|
|
|
|
|
//--- the plateau ladder would restore the weights around this layer while leaving its
|
|
|
|
|
//--- own parameters at whatever the diverged era left behind.
|
|
|
|
|
CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
|
|
|
|
|
got = bn.getWeightsBN(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronConvOCL *c = (CNeuronConvOCL*)obj;
|
|
|
|
|
got = c.getWeightsConv(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj;
|
|
|
|
|
got = ls.getWeightsLSTM(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuron:
|
|
|
|
|
case defNeuronConv:
|
|
|
|
|
case defNeuronPool:
|
|
|
|
|
case defNeuronLSTM:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBase *neuron = (CNeuronBase*)obj;
|
|
|
|
|
CArrayCon *conns = neuron.getConnections();
|
|
|
|
|
if(CheckPointer(conns) != POINTER_INVALID)
|
|
|
|
|
{
|
|
|
|
|
got = conns.Total();
|
|
|
|
|
ArrayResize(w, got);
|
|
|
|
|
for(int i = 0; i < got; i++)
|
|
|
|
|
{
|
|
|
|
|
CConnection *con = conns.At(i);
|
|
|
|
|
w[i] = (CheckPointer(con) != POINTER_INVALID) ? con.weight : 0.0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
CArrayDouble *snap = new CArrayDouble();
|
|
|
|
|
if(CheckPointer(snap) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
if(got > 0)
|
|
|
|
|
snap.AssignArray(w);
|
|
|
|
|
if(!m_weightSnapshot.Add(snap))
|
|
|
|
|
{
|
|
|
|
|
delete snap;
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
m_haveWeightSnapshot = true;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Write the CaptureWeights() snapshot back into the live neurons IN |
|
|
|
|
|
//| PLACE (setWeights - no neuron re-creation, no new device tensors).|
|
|
|
|
|
//| Aborts (returning false, live weights untouched past that point) |
|
|
|
|
|
//| only on a neuron-count mismatch, which never happens within a run |
|
|
|
|
|
//| (fixed topology). See the header declaration for the full why. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::RestoreWeights(void)
|
|
|
|
|
{
|
|
|
|
|
if(!m_haveWeightSnapshot || CheckPointer(m_weightSnapshot) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
int idx = 0;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
if(idx >= m_weightSnapshot.Total())
|
|
|
|
|
return false; // topology/count mismatch - stop rather than mis-map weights
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
CArrayDouble *snap = m_weightSnapshot.At(idx);
|
|
|
|
|
idx++;
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID || CheckPointer(snap) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
int cnt = snap.Total();
|
|
|
|
|
if(cnt <= 0)
|
|
|
|
|
continue; // no weights captured for this neuron (e.g. output layer) - nothing to restore
|
|
|
|
|
double w[];
|
|
|
|
|
ArrayResize(w, cnt);
|
|
|
|
|
for(int i = 0; i < cnt; i++)
|
|
|
|
|
w[i] = snap.At(i);
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
|
|
|
|
|
neuron.setWeights(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
|
|
|
|
|
bn.setWeightsBN(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronConvOCL *c = (CNeuronConvOCL*)obj;
|
|
|
|
|
c.setWeightsConv(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronLSTMOCL *ls = (CNeuronLSTMOCL*)obj;
|
|
|
|
|
ls.setWeightsLSTM(w);
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuron:
|
|
|
|
|
case defNeuronConv:
|
|
|
|
|
case defNeuronPool:
|
|
|
|
|
case defNeuronLSTM:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBase *neuron = (CNeuronBase*)obj;
|
|
|
|
|
CArrayCon *conns = neuron.getConnections();
|
|
|
|
|
if(CheckPointer(conns) == POINTER_INVALID)
|
|
|
|
|
break;
|
|
|
|
|
int count = MathMin(cnt, conns.Total());
|
|
|
|
|
for(int i = 0; i < count; i++)
|
|
|
|
|
{
|
|
|
|
|
CConnection *con = conns.At(i);
|
|
|
|
|
if(CheckPointer(con) != POINTER_INVALID)
|
|
|
|
|
con.weight = w[i];
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return (idx == m_weightSnapshot.Total());
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
feat: mini-batch gradient accumulation (F4), front-end-aware capacity budget (F6), split Wyckoff categoricals (N1)
Completes the 2026-08-09 training audit. FORCES A RETRAIN of every
Wyckoff-enabled config (N1 re-keys the fingerprint), and BOTH DLLs must be
redeployed alongside the .ex5 - they carry new exports.
F4 - mini-batch accumulation, TRAIN_BATCH_SIZE=32. Training was pure online
SGD (one weight update per bar), which is the mechanical source of the
era-to-era whipsaw every downstream guard was built to cope with. The O(n^2)
outer product is native - AccumulateWeightGrad / AccumulateWeightGradConv /
AccumulateBufferInto in Network.cl, WarriorCPU and WarriorDML - while the
optimizer step is host-side MQL5 shared by all tiers (ApplyAccumToBlock), so
there is one Adam/SGD implementation instead of four that can drift.
- the LSTM needs no outer-product kernel (WeightsGradient already holds the
sample's full dW) but could NOT simply be left un-zeroed between samples:
CPU_LSTMSeqBackward/DML_LSTMSeqBackward memset it on entry. Hence a
separate accumulator plus an elementwise add.
- batch-norm gamma/beta accumulate in host arrays, not new BatchOptions
slots - BN_OPT_STRIDE is baked into every persisted .nnw.
- scoped to pass 2; online learning keeps immediate updates. Every save /
checkpoint / scoring boundary flushes, scaling by the real sample count.
- degrades to per-sample updates (one log line) on a tier that cannot
accumulate, so old devices and DLL-free builds are unaffected.
- verified offline: DirectML/batch_accum_check.cpp drives the real exports
against an independent reference; at B=1 the accumulator matches the
shipped unbatched kernel's own gradient to 1.1e-16. Math only - the
in-situ check remains the per-layer dW/W report on a real era.
F6 - ComputeFirstLayerWidth budgeted against the RAW input width even where a
conv/LSTM front end had already reduced it, so an LSTM's dense stack was
charged for 1,280 inputs when it receives 64. Confirmed from the deployed
.cfg files: CONV, LSTM and HYBRID were all pinned at the 16-unit floor. Now
budgeted against the front-end output and capped at it (never fan out), with
the derivation reordered so both stages settle first.
N1 - EventCode/EventPhase/StructuralPhase are signed categoricals packing
direction and Wyckoff stage into one scalar across a sign discontinuity. Split
into direction + [0,1] magnitude, the same convention the base OHLC block uses.
Information-preserving; 13 readings now occupy 16 inputs.
Compiled clean (0 errors, 0 warnings); both DLLs rebuilt.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-09 11:48:03 -04:00
|
|
|
//| Effective batch size - what training GETS, not what it asked for. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Accumulation needs a backend that can run the accumulate kernels: |
|
|
|
|
|
//| OpenCL only if they actually built on this device, DirectML/CPU |
|
|
|
|
|
//| DLL always (the exports are part of the DLL we ship). The legacy |
|
|
|
|
|
//| scalar tier - the no-backend path a Market build can land on - |
|
|
|
|
|
//| stores weights per CConnection with no buffers to accumulate into,|
|
|
|
|
|
//| so it keeps per-sample updates. Degrading to 1 rather than |
|
|
|
|
|
//| refusing to train is the point: an old device or a DLL-free build |
|
|
|
|
|
//| trains exactly as it did before this change. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
int CNet::BatchSize(void)
|
|
|
|
|
{
|
|
|
|
|
if(m_batchSizeRequested <= 1)
|
|
|
|
|
return 1;
|
|
|
|
|
bool canBatch = (CheckPointer(opencl) != POINTER_INVALID && m_batchKernelsOk) ||
|
|
|
|
|
CheckPointer(directml) != POINTER_INVALID;
|
|
|
|
|
if(!canBatch)
|
|
|
|
|
{
|
|
|
|
|
if(!m_batchWarned)
|
|
|
|
|
{
|
|
|
|
|
m_batchWarned = true;
|
|
|
|
|
Print("CNet::BatchSize: mini-batch training was requested (size ", m_batchSizeRequested,
|
|
|
|
|
") but this compute tier cannot accumulate gradients - falling back to one weight update ",
|
|
|
|
|
"per sample. Training is correct, just noisier; see the F4 note in ",
|
|
|
|
|
"research\\training_pipeline_audit_2026-08-09.md.");
|
|
|
|
|
}
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return m_batchSizeRequested;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Zero every accumulator, opening a fresh batch. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::BeginBatch(void)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
bool ok = true;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
case defNeuronPoolOCL:
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
|
|
|
|
|
ok = neuron.BeginGradAccum() && ok;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
m_batchCount = 0;
|
|
|
|
|
m_batchBegun = true;
|
|
|
|
|
return ok;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Apply whatever the current (possibly partial) batch accumulated, |
|
|
|
|
|
//| then close it. Safe and cheap to call when nothing is pending - |
|
|
|
|
|
//| every save/checkpoint/scoring boundary calls it unconditionally. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Scaling by the REAL sample count, not the requested batch size: |
|
|
|
|
|
//| a partial flush at an era boundary must be the mean of the samples|
|
|
|
|
|
//| it actually saw, or the last (short) batch of every era would take|
|
|
|
|
|
//| a systematically undersized step. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::FlushBatch(void)
|
|
|
|
|
{
|
|
|
|
|
if(m_batchCount <= 0 || CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
{
|
|
|
|
|
m_batchCount = 0;
|
|
|
|
|
m_batchBegun = false;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
double scale = 1.0 / (double)m_batchCount;
|
|
|
|
|
bool ok = true;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
case defNeuronPoolOCL:
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
|
|
|
|
|
ok = neuron.ApplyAccumulatedGradients(scale) && ok;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
m_batchCount = 0;
|
|
|
|
|
m_batchBegun = false;
|
|
|
|
|
return ok;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
fix: training-stability audit fixes F1/F2/F3/F5 - unbiased shuffle, real plateau escapes, fresh optimizer state on restore, pure OOS metric
Four of the six findings from research/training_pipeline_audit_2026-08-09.md
(F4 mini-batching and F6 feature re-encode deliberately deferred - see the
report's implementation-status section for why):
- F1: pass-2 Fisher-Yates (and AutoTune's MI block shuffle) used MathRand()%,
which is 15-bit - provably non-uniform on every full-history era over 32,768
queued samples. New 30-bit ShuffleRandomIndex().
- F2: plateau warm restarts were a no-op whenever eta already sat at its
ceiling (the normal state of a non-regressing plateau) - the ladder was just
a 24-era countdown. Restarts now overshoot to 5x the ceiling
(PLATEAU_RESTART_BOOST) and anneal geometrically back over the patience
window, SGDR-style; ETA_MIN widened 1e-4 -> 1e-5 so the decay schedule has
real range.
- F3: checkpoint restores put weights back but kept the rejected trajectory's
Adam moments, so the optimizer immediately pushed back toward the rolled-back
state (the restore->regress->restore oscillation). CNet::ResetOptimizerState()
zeroes moments/momentum/step counters (weights, BN statistics, gamma/beta
untouched) on every mid-run restore, every boosted restart, and the
deploy-time restore that online learning continues from.
- F5: batch-norm running statistics now freeze for the pass-3 OOS scoring walk,
so the selection metric the checkpoint ranking and deploy gate read is a pure
function of the checkpoint instead of partly measuring BN drift. Defensive
unfreeze in FinalizeTrainRun covers stop-mid-pass; live/online adaptation and
the OOS continual-learning simulation stay adaptive by design.
Compiled clean (0 errors, 0 warnings) via the staged-tree recipe.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-09 10:54:09 -04:00
|
|
|
//| Zero every neuron's optimizer state (Adam m/v, SGD momentum, step |
|
|
|
|
|
//| counters, batch-norm gamma/beta moments), weights untouched - see |
|
|
|
|
|
//| the declaration comment for the restore/warm-restart rationale. |
|
|
|
|
|
//| Traversal mirrors SetBatchNormFrozen; per-type specifics live in |
|
|
|
|
|
//| each neuron class's own ResetOptimizerState override. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::ResetOptimizerState(void)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
bool ok = true;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
//--- Type() first through CObject, then cast into the right hierarchy - the OCL and legacy
|
|
|
|
|
//--- scalar neuron families are unrelated classes, exactly as in BlendWeightsFrom above.
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
switch(obj.Type())
|
|
|
|
|
{
|
|
|
|
|
case defNeuronBaseOCL:
|
|
|
|
|
case defNeuronBatchNormOCL:
|
|
|
|
|
case defNeuronConvOCL:
|
|
|
|
|
case defNeuronPoolOCL:
|
|
|
|
|
case defNeuronLSTMOCL:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *neuron = (CNeuronBaseOCL*)obj;
|
|
|
|
|
ok = neuron.ResetOptimizerState() && ok;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
case defNeuron:
|
|
|
|
|
case defNeuronConv:
|
|
|
|
|
case defNeuronPool:
|
|
|
|
|
case defNeuronLSTM:
|
|
|
|
|
{
|
|
|
|
|
CNeuronBase *neuron = (CNeuronBase*)obj;
|
|
|
|
|
ok = neuron.ResetOptimizerState() && ok;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
default:
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return ok;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
2026-08-01 11:27:28 -04:00
|
|
|
//| See this method's declaration comment for the cold-start bias |
|
|
|
|
|
//| rationale. The weight block that produces the output layer's |
|
|
|
|
|
//| values is stored on the layer BEFORE it (see CNeuronBaseOCL:: |
|
|
|
|
|
//| feedForward(CNeuronBaseOCL*) - matrix_w comes from the SOURCE |
|
|
|
|
|
//| neuron, laid out as (sourceNeurons+1) values per destination |
|
|
|
|
|
//| neuron, the last of which is that neuron's bias term), so this |
|
|
|
|
|
//| reaches one layer back from the output layer to edit it. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::SeedOutputLayerBias(const double &biasValues[])
|
|
|
|
|
{
|
|
|
|
|
int outputs = ArraySize(biasValues);
|
|
|
|
|
if(outputs <= 0 || CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
|
|
|
|
|
return false;
|
|
|
|
|
CLayer *sourceLayer = layers.At(layers.Total() - 2);
|
|
|
|
|
if(CheckPointer(sourceLayer) == POINTER_INVALID || sourceLayer.Total() <= 0)
|
|
|
|
|
return false;
|
|
|
|
|
CObject *sourceObj = sourceLayer.At(0);
|
|
|
|
|
//--- Batch norm is accepted here as well as a plain dense layer. It is a CNeuronBaseOCL subclass
|
|
|
|
|
//--- with the identical (inputs+1)*outputs weight layout - it just happens to be the layer that
|
|
|
|
|
//--- carries the head's weight matrix once normalization is enabled (see AI\NeuronBatchNorm.mqh and
|
|
|
|
|
//--- BuildFreshTopology's batch-norm insertion). Without this the exact-type check silently failed
|
|
|
|
|
//--- and the cold-start output-bias seed stopped being applied for every batch-norm topology.
|
|
|
|
|
if(CheckPointer(sourceObj) == POINTER_INVALID ||
|
|
|
|
|
(sourceObj.Type() != defNeuronBaseOCL && sourceObj.Type() != defNeuronBatchNormOCL))
|
|
|
|
|
return false;
|
|
|
|
|
CNeuronBaseOCL *sourceNeuron = (CNeuronBaseOCL*)sourceObj;
|
|
|
|
|
if(CheckPointer(sourceNeuron) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
int inputs = sourceNeuron.Neurons();
|
|
|
|
|
double weights[];
|
|
|
|
|
int count = sourceNeuron.getWeights(weights);
|
|
|
|
|
if(count != (inputs + 1) * outputs)
|
|
|
|
|
return false; // layout doesn't match the assumed dense (source+1)*outputs block - don't guess
|
|
|
|
|
for(int i = 0; i < outputs; i++)
|
|
|
|
|
weights[(inputs + 1) * i + inputs] = biasValues[i];
|
|
|
|
|
return sourceNeuron.setWeights(weights);
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
void CNet::SetBatchNormFrozen(bool frozen)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return;
|
|
|
|
|
for(int l = 0; l < layers.Total(); l++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = (CLayer*)layers.At(l);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
for(int n = 0; n < layer.Total(); n++)
|
|
|
|
|
{
|
|
|
|
|
CObject *obj = layer.At(n);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID || obj.Type() != defNeuronBatchNormOCL)
|
|
|
|
|
continue;
|
|
|
|
|
CNeuronBatchNormOCL *bn = (CNeuronBatchNormOCL*)obj;
|
|
|
|
|
bn.SetStatsFrozen(frozen);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| Window of the FIRST convolutional layer in the loaded net, or 0 |
|
|
|
|
|
//| when there is none. |
|
|
|
|
|
//| |
|
|
|
|
|
//| Same problem as EnforceOutputActivation below, different field: a |
|
|
|
|
|
//| .nnw stores the window/step each conv layer was BUILT with, so a |
|
|
|
|
|
//| model saved before the receptive field changed keeps the old |
|
|
|
|
|
//| shape forever and goes on training under an architecture the code |
|
|
|
|
|
//| no longer specifies. Unlike the activation this CANNOT be |
|
|
|
|
|
//| repaired in place - the weight block is (window+1)*window_out and |
|
|
|
|
|
//| a different window is a different tensor - so the caller's only |
|
|
|
|
|
//| correct response is to retrain. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
uint CNet::FirstConvWindow(void)
|
|
|
|
|
{
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID)
|
|
|
|
|
return 0;
|
|
|
|
|
for(int i = 0; i < layers.Total(); i++)
|
|
|
|
|
{
|
|
|
|
|
CLayer *layer = layers.At(i);
|
|
|
|
|
if(CheckPointer(layer) == POINTER_INVALID || layer.Total() <= 0)
|
|
|
|
|
continue;
|
|
|
|
|
CObject *obj = layer.At(0);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
continue;
|
|
|
|
|
if(obj.Type() == defNeuronConvOCL)
|
|
|
|
|
{
|
|
|
|
|
CNeuronConvOCL *conv = (CNeuronConvOCL*)obj;
|
|
|
|
|
return conv.Window();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
//| See the declaration comment for why a loaded model's activation |
|
|
|
|
|
//| cannot be trusted and must be re-asserted from the topology spec. |
|
|
|
|
|
//+------------------------------------------------------------------+
|
|
|
|
|
bool CNet::EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous)
|
|
|
|
|
{
|
|
|
|
|
previous = intended;
|
|
|
|
|
if(CheckPointer(layers) == POINTER_INVALID || layers.Total() < 2)
|
|
|
|
|
return false;
|
|
|
|
|
CLayer *outputLayer = layers.At(layers.Total() - 1);
|
|
|
|
|
if(CheckPointer(outputLayer) == POINTER_INVALID || outputLayer.Total() <= 0)
|
|
|
|
|
return false;
|
|
|
|
|
CObject *obj = outputLayer.At(0);
|
|
|
|
|
if(CheckPointer(obj) == POINTER_INVALID)
|
|
|
|
|
return false;
|
|
|
|
|
//--- Both neuron models expose the same two accessors (CNeuronBase gained Activation() for exactly
|
|
|
|
|
//--- this), so one branch per family is enough - and anything else (conv/pool/LSTM can never be an
|
|
|
|
|
//--- output layer in this project's topologies) is left untouched rather than force-cast.
|
|
|
|
|
int t = obj.Type();
|
|
|
|
|
if(t == defNeuronBaseOCL || t == defNeuronConvOCL || t == defNeuronPoolOCL || t == defNeuronLSTMOCL ||
|
|
|
|
|
t == defNeuronBatchNormOCL)
|
|
|
|
|
{
|
|
|
|
|
CNeuronBaseOCL *n = (CNeuronBaseOCL*)obj;
|
|
|
|
|
previous = n.Activation();
|
|
|
|
|
if(previous == intended)
|
|
|
|
|
return false;
|
|
|
|
|
n.SetActivationFunction(intended);
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
if(t == defNeuron || t == defNeuronConv || t == defNeuronPool || t == defNeuronLSTM)
|
|
|
|
|
{
|
|
|
|
|
//--- The scalar CPU model stores activation per NEURON, not per layer, so every neuron in the
|
|
|
|
|
//--- output layer has to be corrected - not just index 0.
|
|
|
|
|
bool repaired = false;
|
|
|
|
|
for(int i = 0; i < outputLayer.Total(); i++)
|
|
|
|
|
{
|
|
|
|
|
CObject *cell = outputLayer.At(i);
|
|
|
|
|
if(CheckPointer(cell) == POINTER_INVALID || cell.Type() != defNeuron)
|
|
|
|
|
continue;
|
|
|
|
|
CNeuronBase *n = (CNeuronBase*)cell;
|
|
|
|
|
if(n.Activation() == intended)
|
|
|
|
|
continue;
|
|
|
|
|
if(!repaired)
|
|
|
|
|
previous = n.Activation();
|
|
|
|
|
n.SetActivationFunction(intended);
|
|
|
|
|
repaired = true;
|
|
|
|
|
}
|
|
|
|
|
return repaired;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
#endif
|