//+------------------------------------------------------------------+ //| NeuronBatchNorm.mqh | //| AnimateDread | //| https://www.mql5.com | //+------------------------------------------------------------------+ //| CNeuronBatchNormOCL - batch normalization (Ioffe & Szegedy 2015). | //| Needs CNeuronBaseOCL (AI\Network.mqh) already declared; included | //| from there, not standalone. | //| | //| WHY THIS EXISTS | //| Every hidden stage in this project is unbounded (PRELU dense, | //| PRELU conv) and the only bounded stage in the whole forward path | //| was the SIGMOID classification head. That is a network with no | //| internal scale control at all, and it showed: across four | //| topologies on SP500 H1 the models peaked early then decayed | //| monotonically into all-Neutral, and the decay ordered exactly by | //| DEPTH - the shallow perceptron held ~52% balanced accuracy while | //| the deepest (conv+pool+LSTM+dense taper) sat at the 33.3% | //| one-class floor. That depth ordering is the signature of internal | //| covariate shift, which is precisely what this layer addresses. | //| | //| Two further consequences worth spelling out, because they are the | //| actual mechanism by which this is expected to help here: | //| | //| 1. It decouples weight decay from the learned function. With a | //| normalized layer downstream, scaling the weights that feed it | //| leaves the output unchanged - so WEIGHT_DECAY can no longer | //| grind the discriminative signal away, it only rescales the | //| effective learning rate (van Laarhoven 2017). The observed | //| failure was exactly a slow monotonic shrink of the per-bar | //| logit spread (0.45 -> 0.38 over ~200 eras) until the evidence | //| tilt fell under the class-prior tilt and argmax degenerated | //| to constant-Neutral. See WEIGHT_DECAY's comment in | //| AI\Network.mqh, which describes that mechanism first-hand. | //| 2. It is what would make an UNBOUNDED logit head viable. The | //| 2026-07-27 attempt to run the head at NONE blew up (IS error | //| 5.6e15) specifically because nothing upstream constrained | //| scale. Do not retry that without a normalization layer | //| immediately before the head - and then also drop | //| CLASS_LOGIT_SCALE to 1.0 and BIAS_MAGNITUDE to ~0.5. | //| | //| IMPLEMENTATION SHAPE - deliberately host-side | //| The math is elementwise O(n), not O(n^2) like a dense layer, so | //| it runs in MQL5 against the host mirrors of the device buffers | //| rather than as a fourth copy of a kernel in Network.cl + | //| WarriorCPU.cpp + WarriorDML.cpp. That is a deliberate trade: | //| - it works identically on all four compute tiers with no DLL | //| rebuild and no risk of the backends drifting out of sync, | //| which has been a recurring source of bugs in this engine; | //| - it follows the precedent already set by the softmax+CCE | //| output gradient and the per-sample loss weighting, both of | //| which are computed in MQL5 for the same reason (see | //| CNet::backPropOCL); | //| - the cost is one buffer round-trip per BN layer per sample. | //| On the CPU-DLL tier those buffers are host memory anyway, so | //| it is a memcpy. On a real GPU it is a PCIe hop; if that ever | //| matters, port these three methods to kernels - the math here | //| is the specification. | //| | //| STATISTICS: exponential moving, not a stored mini-batch. | //| Training here is pure online SGD - one weight update per sample | //| (CNet::backProp per bar), never a batched pass - so there is no | //| mini-batch to average over. iBatchSize is therefore an EMA WINDOW | //| LENGTH, not a buffer size: mean and variance are updated per | //| sample toward the last ~iBatchSize samples. Same formulation as | //| the NeuroNet_DNG reference's BatchFeedForward kernel; the book | //| (ch. 6.1.1) explicitly endorses the exponential form to avoid | //| storing per-neuron history. | //| | //| The statistics keep adapting on EVERY forward pass, including | //| out-of-sample scoring and live inference - they are never frozen | //| the way classic batch norm freezes them at inference time. That | //| is deliberate and matches the rest of this system (see | //| OnlineLearnStep: the deployed model is designed to keep tracking | //| the market). Two consequences worth knowing: OOS scoring lets the | //| statistics see OOS activations - unsupervised, no label | //| information, but not a hermetic holdout - and a restored | //| checkpoint rewinds the statistics along with gamma/beta, since | //| getWeightsBN/setWeightsBN carry both. | //+------------------------------------------------------------------+ #ifndef WARRIOR_NEURON_BATCHNORM_MQH #define WARRIOR_NEURON_BATCHNORM_MQH //--- Per-neuron slot layout inside BatchOptions. A FIXED stride of 9 on every optimizer, unlike the //--- reference's 7-or-9 split: the two unused doubles per neuron are noise next to a weight matrix, //--- and a stride that changes with a persisted enum is an indexing bug waiting for the first time //--- somebody switches TrainingOptimizer on an existing model. #define BN_OPT_STRIDE 9 #define BN_OPT_MEAN 0 // running mean #define BN_OPT_VAR 1 // running variance #define BN_OPT_NX 2 // normalized input, cached from the forward pass for the backward pass #define BN_OPT_GAMMA 3 // learned scale, init 1 #define BN_OPT_BETA 4 // learned shift, init 0 #define BN_OPT_MG 5 // gamma: Adam first momentum, or SGD previous delta #define BN_OPT_MB 6 // beta: Adam first momentum, or SGD previous delta #define BN_OPT_VG 7 // gamma: Adam second momentum (stored already square-rooted, as the #define BN_OPT_VB 8 // beta: ... UpdateWeightsAdam kernels in this engine also do) //--- Variance floor. Serves the same purpose as MIN_ACTIVATION_DERIVATIVE: a neuron whose input //--- happens to be near-constant over the EMA window has a near-zero variance, and dividing by its //--- square root turns rounding noise into an arbitrarily large activation that then propagates. //--- Applied to the standard deviation (not the variance) so it reads as "no unit is amplified by //--- more than 1e4", which is the property that actually matters. #define BN_EPSILON 1.0e-10 #define BN_MIN_STD 1.0e-4 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBatchNormOCL : public CNeuronBaseOCL { protected: int iBatchSize; // EMA window length; <=1 disables normalization entirely //--- When true the running statistics are USED but not UPDATED, i.e. classic batch-norm inference //--- semantics. Off by default (see the note on adaptation in the class header). Exists because a //--- forward pass is otherwise not a pure function of its input: two evaluations of the same bar //--- return slightly different answers because the first one moved the statistics. Anything that //--- COMPARES two forward passes has to freeze them first or it is measuring its own side effect - //--- CExpertSignalAIBase::ValidateCpuInference() is exactly that, and would have silently failed //--- its DLL-free check on nothing but the one EMA step it took itself. bool bStatsFrozen; //--- Number of samples seen. Used only to ramp the effective window up from 1 to iBatchSize over //--- the first iBatchSize samples (standard EMA bias correction). Without it the running mean //--- starts at 0 and approaches the true mean over ~iBatchSize samples, during which the layer //--- emits near-zero for everything - a cold-start dead zone indistinguishable from the very //--- collapse this layer exists to prevent. int iSamplesSeen; //--- Host-only: no kernel ever reads it, so it deliberately gets no device buffer. Persisted //--- through Save/Load like any other parameter - gamma/beta are learned, and the running //--- statistics ARE the layer's inference behaviour, so a model that loses them is not the model //--- that was trained. CArrayDouble *BatchOptions; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); //--- shared by feedForward/feedForwardCPU: the whole forward transform for one already-read input //--- vector, writing straight into the host mirror of Output. bool NormalizeHost(const double &inputs[], int count); //--- ensures BatchOptions exists and is sized/seeded for `neurons` units bool InitOptions(int neurons); public: CNeuronBatchNormOCL(void); ~CNeuronBatchNormOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); //--- see bStatsFrozen. Deliberately NOT persisted: it is a transient evaluation mode, not model state. void SetStatsFrozen(bool v) { bStatsFrozen = v; } //--- Checkpoint/blend support. CNet::CaptureWeights/RestoreWeights/BlendWeightsFrom snapshot one //--- flat array per neuron object, so gamma/beta/mean/variance are appended AFTER the outgoing //--- dense weight matrix and split back out on restore. Without this the plateau ladder's //--- "restore best checkpoint" would put the dense weights back while leaving this layer's //--- parameters at whatever the diverged era left behind - a silently mismatched pair. virtual int getWeightsBN(double &values[]); virtual bool setWeightsBN(double &values[]); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronBatchNormOCL; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBatchNormOCL::CNeuronBatchNormOCL(void) : iBatchSize(1), bStatsFrozen(false), iSamplesSeen(0) { BatchOptions = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBatchNormOCL::~CNeuronBatchNormOCL(void) { if(CheckPointer(BatchOptions) != POINTER_INVALID) delete BatchOptions; } //+------------------------------------------------------------------+ //| gamma=1 / beta=0 / zeroed statistics for every unit. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::InitOptions(int neurons) { if(neurons <= 0) return false; if(CheckPointer(BatchOptions) == POINTER_INVALID) { BatchOptions = new CArrayDouble(); if(CheckPointer(BatchOptions) == POINTER_INVALID) return false; } BatchOptions.Clear(); if(!BatchOptions.Reserve(neurons * BN_OPT_STRIDE)) return false; for(int n = 0; n < neurons; n++) for(int s = 0; s < BN_OPT_STRIDE; s++) if(!BatchOptions.Add(s == BN_OPT_GAMMA ? 1.0 : 0.0)) return false; iSamplesSeen = 0; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type)) return false; //--- Identity forward transform. The activation belongs to the layer AFTER this one; normalizing //--- and then squashing in the same step would defeat the point (Ioffe & Szegedy place the //--- normalization immediately BEFORE the non-linearity, not around it). activation = NONE; iBatchSize = (int)batchSize; return InitOptions((int)numNeurons); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, direct_ml, numNeurons, optimization_type)) return false; activation = NONE; iBatchSize = (int)batchSize; return InitOptions((int)numNeurons); } //+------------------------------------------------------------------+ //| The forward transform, host-side. Mirrors the NeuroNet_DNG | //| BatchFeedForward kernel, plus the bias-corrected warm-up window. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::NormalizeHost(const double &inputs[], int count) { if(CheckPointer(Output) == POINTER_INVALID || CheckPointer(BatchOptions) == POINTER_INVALID) return false; if(count > Output.Total()) count = Output.Total(); if(BatchOptions.Total() < count * BN_OPT_STRIDE) return false; //--- A window of 1 makes mean==x and variance==0 for every sample, i.e. a constant-zero output. //--- Treat it as "normalization off" and pass the input through untouched rather than emit zeros. if(iBatchSize <= 1) { for(int i = 0; i < count; i++) if(!Output.Update(i, inputs[i])) return false; return true; } if(!bStatsFrozen && iSamplesSeen < iBatchSize) iSamplesSeen++; //--- Effective window: ramps 1,2,3... up to iBatchSize, so the first samples produce an honest //--- running mean instead of one biased toward the zero initializer. double w = (double)MathMax(1, iSamplesSeen); for(int i = 0; i < count; i++) { int shift = i * BN_OPT_STRIDE; double x = inputs[i]; double mean = BatchOptions.At(shift + BN_OPT_MEAN); double variance = BatchOptions.At(shift + BN_OPT_VAR); if(!bStatsFrozen) { mean = (mean * (w - 1.0) + x) / w; variance = (variance * (w - 1.0) + (x - mean) * (x - mean)) / w; } double delta = x - mean; double sd = MathMax(MathSqrt(variance + BN_EPSILON), BN_MIN_STD); double nx = delta / sd; double y = BatchOptions.At(shift + BN_OPT_GAMMA) * nx + BatchOptions.At(shift + BN_OPT_BETA); //--- nx is still cached when frozen: it costs nothing and keeps the buffer consistent with the //--- output just produced. There is no backward pass while frozen, so nothing reads it. if(!bStatsFrozen && (!BatchOptions.Update(shift + BN_OPT_MEAN, mean) || !BatchOptions.Update(shift + BN_OPT_VAR, variance))) return false; if(!BatchOptions.Update(shift + BN_OPT_NX, nx)) return false; if(!Output.Update(i, y)) return false; } return true; } //+------------------------------------------------------------------+ //| Device path: pull the previous layer's output to the host, run | //| the transform, push our output back. See the file header for why | //| this is not a kernel. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; double inputs[]; int count = NeuronOCL.getOutputVal(inputs); if(count <= 0) return false; if(!NormalizeHost(inputs, count)) return false; //--- Unlike the dense/conv kernels this layer's Output does NOT stay device-resident by itself - //--- it was just written host-side, so it has to be pushed before the next layer's kernel reads //--- it through getOutputIndex(). return Output.BufferWrite(); } //+------------------------------------------------------------------+ //| Pure-MQL5 inference path (no backend at all): the previous | //| layer's values live only in its host mirror, so read them there. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; int count = NeuronOCL.Neurons(); if(count <= 0) return false; double inputs[]; if(ArrayResize(inputs, count) != count) return false; for(int i = 0; i < count; i++) inputs[i] = NeuronOCL.OutputHost(i); //--- No BufferWrite: there is no device buffer on this path, and Output's host mirror is what //--- the next layer's feedForwardCPU (and GetOutputsCPU) read. return NormalizeHost(inputs, count); } //+------------------------------------------------------------------+ //| Backward. Called by the layer BELOW this one (inverted-call | //| convention, same as Conv/Pool/LSTM): consumes this layer's own | //| Gradient (dL/dy, already filled by calcHiddenGradients against | //| the layer above) and writes dL/dx into NeuronOCL's Gradient. | //| | //| dL/dx = gamma/sd * dL/dy. | //| | //| The mean/variance are treated as CONSTANTS with respect to the | //| current sample. That is exact for the running-statistics form | //| this layer implements, and it is also what batch norm does at | //| inference time. The reference's kernel additionally carries two | //| correction terms for the current sample's own contribution to the | //| statistics; both are O(1/batch) and vanish at the window lengths | //| used here, and its variance term (mean*x / 2*var^1.5) does not | //| follow from the derivative in the paper - so they are deliberately| //| NOT copied. Numerical robustness matters far more in this engine | //| than a correction that is already below the noise floor. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(BatchOptions) == POINTER_INVALID) return false; double grad[]; int count = getGradient(grad); //--- Gradient is allocated with one slot more than Neurons() (see CNeuronBaseOCL::Init); only the //--- real units carry a value. int units = Neurons(); if(count < units || units <= 0) return false; int inputsCount = NeuronOCL.Neurons(); if(inputsCount != units) return false; // batch norm is elementwise - a size mismatch means the topology is wrong double prevOut[]; if(NeuronOCL.getOutputVal(prevOut) < inputsCount) return false; double ig[]; if(ArrayResize(ig, inputsCount) != inputsCount) return false; for(int i = 0; i < inputsCount; i++) { int shift = i * BN_OPT_STRIDE; double g = grad[i]; if(iBatchSize > 1) { double sd = MathMax(MathSqrt(BatchOptions.At(shift + BN_OPT_VAR) + BN_EPSILON), BN_MIN_STD); g = g * BatchOptions.At(shift + BN_OPT_GAMMA) / sd; } //--- Then the previous layer's own activation derivative, byte-for-byte the same treatment //--- Network.cl's CaclHiddenGradient applies - including the clamp-to-range "implied target" //--- reformulation - so that from the previous layer's point of view a batch-norm layer is //--- indistinguishable from any other. NONE falls through unscaled. double out = prevOut[i]; switch(NeuronOCL.Activation()) { case TANH: g = MathMax(-1.0, MathMin(1.0, g + out)) - out; g = g * MathMax(MIN_ACTIVATION_DERIVATIVE, 1.0 - out * out); break; case SIGMOID: g = MathMax(0.0, MathMin(1.0, g + out)) - out; g = g * MathMax(MIN_ACTIVATION_DERIVATIVE, out * (1.0 - out)); break; case PRELU: g = g * (out >= 0 ? 1.0 : 0.01); break; default: break; } ig[i] = g; } return NeuronOCL.setGradient(ig); } //+------------------------------------------------------------------+ //| gamma/beta update. NeuronOCL is the previous layer and is | //| deliberately unused: batch norm consumes its output elementwise, | //| so there is no incoming weight matrix to update (the CNet | //| constructor gives the previous layer 0 outgoing weights when this | //| layer follows it). | //| | //| NO WEIGHT DECAY on gamma/beta - unlike every other parameter in | //| this engine, and on purpose. Decaying gamma toward zero shrinks | //| the layer's output toward the constant beta, which is the exact | //| pathology this layer was added to stop; excluding normalization | //| parameters from weight decay is standard practice for the same | //| reason. | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(BatchOptions) == POINTER_INVALID) return false; if(iBatchSize <= 1) return true; // normalization off - gamma/beta are not in the graph double grad[]; int count = getGradient(grad); int units = Neurons(); if(count < units || units <= 0) return false; //--- Same bias-corrected step size the dense Adam path computes, so gamma/beta move on the same //--- schedule as the weights around them. double lt = (optimization == SGD) ? 0.0 : eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); for(int i = 0; i < units; i++) { int shift = i * BN_OPT_STRIDE; double g = grad[i]; //--- dL/dgamma = dL/dy * nx | dL/dbeta = dL/dy double gGamma = g * BatchOptions.At(shift + BN_OPT_NX); double gBeta = g; double gamma = BatchOptions.At(shift + BN_OPT_GAMMA); double beta = BatchOptions.At(shift + BN_OPT_BETA); double dGamma = 0.0, dBeta = 0.0; if(optimization == SGD) { dGamma = eta * gGamma + alpha * BatchOptions.At(shift + BN_OPT_MG); dBeta = eta * gBeta + alpha * BatchOptions.At(shift + BN_OPT_MB); if(!BatchOptions.Update(shift + BN_OPT_MG, dGamma) || !BatchOptions.Update(shift + BN_OPT_MB, dBeta)) return false; } else { //--- Second momentum is stored ALREADY square-rooted so it can be used as the denominator //--- directly, the same convention this engine's UpdateWeightsAdam kernels use //--- (matrix_v[wi] = sqrt(...)). //--- DELIBERATE DIFFERENCE from those kernels: they feed the stored value straight back in //--- as `v`, i.e. sqrt(b2*sqrt(v_prev) + (1-b2)*g^2), which mixes a standard deviation with a //--- variance and is not the Adam recursion. Squaring it back first (below) is. The two are //--- not reconciled here because changing the weight kernels would alter the behaviour of //--- every existing model on all four backends at once; new code gets it right. double mg = b1 * BatchOptions.At(shift + BN_OPT_MG) + (1 - b1) * gGamma; double mb = b1 * BatchOptions.At(shift + BN_OPT_MB) + (1 - b1) * gBeta; double vg = sqrt(b2 * pow(BatchOptions.At(shift + BN_OPT_VG), 2) + (1 - b2) * gGamma * gGamma); double vb = sqrt(b2 * pow(BatchOptions.At(shift + BN_OPT_VB), 2) + (1 - b2) * gBeta * gBeta); dGamma = lt * mg / (vg > 0 ? vg : lt * 10); dBeta = lt * mb / (vb > 0 ? vb : lt * 10); if(!BatchOptions.Update(shift + BN_OPT_MG, mg) || !BatchOptions.Update(shift + BN_OPT_MB, mb) || !BatchOptions.Update(shift + BN_OPT_VG, vg) || !BatchOptions.Update(shift + BN_OPT_VB, vb)) return false; } dGamma = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, dGamma)); dBeta = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, dBeta)); if(!MathIsValidNumber(dGamma) || !MathIsValidNumber(dBeta)) continue; if(!BatchOptions.Update(shift + BN_OPT_GAMMA, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, gamma + dGamma))) || !BatchOptions.Update(shift + BN_OPT_BETA, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, beta + dBeta)))) return false; } if(optimization != SGD && t < INT_MAX) t++; return true; } //+------------------------------------------------------------------+ //| Outgoing dense weight matrix followed by the whole BatchOptions | //| block, as one flat array - see the declaration comment. | //+------------------------------------------------------------------+ int CNeuronBatchNormOCL::getWeightsBN(double &values[]) { double w[]; int wCount = getWeights(w); if(wCount < 0) wCount = 0; int oCount = (CheckPointer(BatchOptions) == POINTER_INVALID) ? 0 : BatchOptions.Total(); if(ArrayResize(values, wCount + oCount) != wCount + oCount) return 0; for(int i = 0; i < wCount; i++) values[i] = w[i]; for(int i = 0; i < oCount; i++) values[wCount + i] = BatchOptions.At(i); return wCount + oCount; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::setWeightsBN(double &values[]) { int total = ArraySize(values); int oCount = (CheckPointer(BatchOptions) == POINTER_INVALID) ? 0 : BatchOptions.Total(); int wCount = total - oCount; if(wCount < 0) return false; // snapshot predates this layer's parameters - refuse rather than half-restore if(wCount > 0) { double w[]; if(ArrayResize(w, wCount) != wCount) return false; for(int i = 0; i < wCount; i++) w[i] = values[i]; if(!setWeights(w)) return false; } for(int i = 0; i < oCount; i++) if(!BatchOptions.Update(i, values[wCount + i])) return false; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) return false; if(FileWriteInteger(file_handle, iBatchSize, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, iSamplesSeen, INT_VALUE) < INT_VALUE) return false; if(CheckPointer(BatchOptions) == POINTER_INVALID) return false; return BatchOptions.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) return false; iBatchSize = FileReadInteger(file_handle, INT_VALUE); iSamplesSeen = FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(BatchOptions) == POINTER_INVALID) { BatchOptions = new CArrayDouble(); if(CheckPointer(BatchOptions) == POINTER_INVALID) return false; } if(!BatchOptions.Load(file_handle)) return false; //--- A model whose statistics block does not match its own width is not usable for inference - //--- fail loudly here rather than index past the end on the first forward pass. if(BatchOptions.Total() != Neurons() * BN_OPT_STRIDE) { Print(__FUNCTION__ + ": batch-norm parameter block is " + IntegerToString(BatchOptions.Total()) + " values but this layer has " + IntegerToString(Neurons()) + " units (expected " + IntegerToString(Neurons() * BN_OPT_STRIDE) + ") - file does not match the topology"); return false; } return true; } #endif // WARRIOR_NEURON_BATCHNORM_MQH //+------------------------------------------------------------------+