forked from animatedread/Warrior_EA
ea9d86b3ee
CaclHiddenGradient computed this layer's gradient as matrix_w[(outputs+1)*i + k] against a buffer whose actual layout (one row per NEXT-layer neuron, stride inputs+1) makes the correct read matrix_w[k*(inputs+1) + i]: the transpose for square layers, and for the non-square boundaries this EA actually builds (tapered stacks, the 3-neuron head) a mis-strided walk that ran past the buffer end - garbage on OpenCL, zeroed reads on the CPU DLL, so the tiers did not even agree with each other. Every gradient crossing a dense boundary on its way down - the entire learning signal reaching the BN/conv/LSTM front ends - passed through a fixed wrong matrix: feedback-alignment dynamics, not backprop, which is why nets still "learned something" and this survived. The book reference (NeuroNet_DNG) fixed this in a later article version; our kernel descended from the earlier one. Confounds every model-based negative verdict to date. Also in this commit, same root cause family: - per-sample UpdateWeightsAdam (OpenCL): input for slot group j was read at matrix_i[j] instead of matrix_i[j*4] (corrupted outer product past group 0), and dispatch dim 1 sized on ceil(inputs/4) left the bias column unreachable whenever inputs%4==0 - dense biases never trained on OpenCL. Rewritten as a lane-guarded scalar loop keeping our Adam conventions (sqrt-stored v, decoupled decay, both clamps, no sign gate). The batched accum path never had either bug; this kernel is what SetBatchSize(1) runs - including online continual learning on client machines, where OpenCL is the only tier. - conv backward passed raw (int)Activation() where the kernels expect NativeActivationCode(): NONE took the tanh branch (clamping a BN layer's unbounded z-scores), TANH took sigmoid, PRELU took none. Dormant only because the conv sits at layer 1 today. - hidden-gradient dispatch over Neurons()+1 dropped to Neurons(): biases get no backprop gradient and the extra work-item only ever read past matrix_o. All three backends (Network.cl, WarriorCPU.cpp, WarriorDML.cpp HLSL) changed in lockstep; DML gained an `inputs` constant to derive the row stride. New dense_backprop_check.cpp proves the CPU kernel is central-finite-difference consistent with the real forward kernel on 8x8, 64x3, 33x64, 5x3 (max diff 3e-9) and that all three activation branches match transcription. All 16 checks pass. Offline math check only - the in-situ proof remains the per-layer dW/W report on a real era. FORCES FULL RETRAIN. Both DLLs rebuilt and redeployed to MQL5\Libraries. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1129 lines
48 KiB
MQL5
1129 lines
48 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronOCLBase.mqh |
|
|
//| |
|
|
//| CNeuronBaseOCL - the accelerated dense neuron (OpenCL / DirectML |
|
|
//| / CPU-DLL tiers). |
|
|
//| |
|
|
//| Included from AI\Network.mqh AFTER every class declaration - |
|
|
//| bodies only, no declarations. Relocation is behaviour-neutral by |
|
|
//| construction: nothing here is reachable until Network.mqh ends. |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
|
|
#define WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum),
|
|
activation(TANH),
|
|
optimization(SGD),
|
|
t(1)
|
|
{
|
|
OpenCL = NULL;
|
|
DirectML = NULL;
|
|
Output = new CBufferDouble();
|
|
PrevOutput = new CBufferDouble();
|
|
Weights = new CBufferDouble();
|
|
DeltaWeights = new CBufferDouble();
|
|
Gradient = new CBufferDouble();
|
|
FirstMomentum = new CBufferDouble();
|
|
SecondMomentum = new CBufferDouble();
|
|
//--- Allocated lazily by EnsureGradAccum() only if a batched update ever runs - see its declaration.
|
|
GradAccum = NULL;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBaseOCL::~CNeuronBaseOCL(void)
|
|
{
|
|
if(CheckPointer(Output) != POINTER_INVALID)
|
|
delete Output;
|
|
if(CheckPointer(PrevOutput) != POINTER_INVALID)
|
|
delete PrevOutput;
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
if(CheckPointer(Gradient) != POINTER_INVALID)
|
|
delete Gradient;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
delete FirstMomentum;
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
delete SecondMomentum;
|
|
if(CheckPointer(GradAccum) != POINTER_INVALID)
|
|
delete GradAccum;
|
|
OpenCL = NULL;
|
|
DirectML = NULL;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
|
|
return false;
|
|
OpenCL = open_cl;
|
|
optimization = optimization_type;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Output.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!Output.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!PrevOutput.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!PrevOutput.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
|
|
return false;
|
|
if(!Gradient.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(numOutputs > 0)
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int count = (int)((numNeurons + 1) * numOutputs);
|
|
if(!Weights.Reserve(count))
|
|
return false;
|
|
// He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
|
|
// et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
|
|
// LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
|
|
// 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
|
|
// ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
|
|
// for the fact that they zero out roughly half their input distribution, whereas the
|
|
// previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
|
|
// was ~2x too conservative here. Applied to every layer through this one shared Init()
|
|
// (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
|
|
// the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
|
|
// differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
|
|
// extremes on every backend, so the imprecision there is not worth the much larger, riskier
|
|
// change of threading activation awareness through every neuron subtype's Init() overload.
|
|
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!Weights.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!Weights.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeights.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeights.BufferCreate(OpenCL))
|
|
return false;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
{
|
|
delete FirstMomentum;
|
|
FirstMomentum = NULL;
|
|
}
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
{
|
|
delete SecondMomentum;
|
|
SecondMomentum = NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
{
|
|
delete DeltaWeights;
|
|
DeltaWeights = NULL;
|
|
}
|
|
//---
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentum.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentum.BufferCreate(OpenCL))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same |
|
|
//| buffer layout, buffers just get created on the DML backend. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0)
|
|
return false;
|
|
DirectML = direct_ml;
|
|
optimization = optimization_type;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Output.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!Output.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!PrevOutput.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!PrevOutput.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
|
|
return false;
|
|
if(!Gradient.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(numOutputs > 0)
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int count = (int)((numNeurons + 1) * numOutputs);
|
|
if(!Weights.Reserve(count))
|
|
return false;
|
|
// He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
|
|
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!Weights.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!Weights.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeights.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeights.BufferCreate(DirectML))
|
|
return false;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
{
|
|
delete FirstMomentum;
|
|
FirstMomentum = NULL;
|
|
}
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
{
|
|
delete SecondMomentum;
|
|
SecondMomentum = NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
{
|
|
delete DeltaWeights;
|
|
DeltaWeights = NULL;
|
|
}
|
|
//---
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentum.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentum.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
|
|
{
|
|
if(CheckPointer(SourceObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
switch(SourceObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
case defNeuronConvOCL:
|
|
case defNeuronLSTMOCL:
|
|
case defNeuronPoolOCL:
|
|
case defNeuronBatchNormOCL:
|
|
temp = SourceObject;
|
|
return feedForward(temp);
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
|
|
NeuronOCL.Neurons(), NativeActivationCode(activation)))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " FeedForward failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Output.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = Output.Total();
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
|
|
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
|
|
if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel FeedForward: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
|
|
//--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
|
|
//--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
|
|
//--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
|
|
//--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Pure-MQL5 dense forward pass - exact double-precision mirror of |
|
|
//| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, |
|
|
//| which (Gizlyk convention) owns both the inputs (its Output) and |
|
|
//| the weights connecting them to THIS layer (its Weights), laid out |
|
|
//| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host |
|
|
//| buffers only; used when no compute backend exists (CPU inference).|
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
int inputs = NeuronOCL.Neurons();
|
|
int outCount = Output.Total();
|
|
int wTotal = NeuronOCL.WeightsCount();
|
|
if(wTotal < (inputs + 1) * outCount)
|
|
return false; // weight buffer smaller than the dense layout requires - refuse rather than misread
|
|
for(int i = 0; i < outCount; i++)
|
|
{
|
|
int shift = (inputs + 1) * i;
|
|
double sum = 0.0;
|
|
for(int k = 0; k < inputs; k++)
|
|
sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k);
|
|
sum += NeuronOCL.WeightHost(shift + inputs); // bias
|
|
switch(activation)
|
|
{
|
|
case TANH:
|
|
sum = tanh(sum);
|
|
break;
|
|
case SIGMOID:
|
|
sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum))));
|
|
break;
|
|
case PRELU:
|
|
if(sum < 0.0)
|
|
sum *= 0.01;
|
|
break;
|
|
// NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case.
|
|
}
|
|
if(!Output.Update(i, sum))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Host-side input write for the CPU inference path's layer 0 (there |
|
|
//| is no device buffer to write into, unlike CNet::feedForward's |
|
|
//| BufferWrite branch). Copies inputVals into the Output host array. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals)
|
|
{
|
|
if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
int total = MathMin(Output.Total(), inputVals.Total());
|
|
for(int i = 0; i < total; i++)
|
|
if(!Output.Update(i, inputVals.At(i)))
|
|
return false;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Host-side output read for getResults() on the CPU inference path |
|
|
//| (no device BufferRead available). Returns the count copied. |
|
|
//+------------------------------------------------------------------+
|
|
int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values)
|
|
{
|
|
if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return 0;
|
|
values.Clear();
|
|
int n = Output.Total();
|
|
for(int i = 0; i < n; i++)
|
|
if(!values.Add(Output.At(i)))
|
|
return i;
|
|
return n;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
//--- count is exactly Neurons() - biases receive no backprop gradient, and the DLL needs the
|
|
//--- true neuron count to derive the weight-row stride (inputs + 1). The old Neurons() + 1
|
|
//--- "bias row" only ever produced out-of-bounds reads (see the 2026-08-11 transpose fix in
|
|
//--- the kernels: Network.cl CaclHiddenGradient, WarriorCPU/WarriorDML CalcHiddenGradient).
|
|
if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
|
|
NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons()))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcHiddenGradient failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Gradient.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
//--- Exactly Neurons(): the kernel derives the weight-row stride from get_global_size(0), and
|
|
//--- biases receive no backprop gradient (the old +1 work-item only ever read past matrix_o).
|
|
global_work_size[0] = Neurons();
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
|
|
if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
|
|
//--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
|
|
{
|
|
if(CheckPointer(Target) == POINTER_INVALID)
|
|
return false;
|
|
int count = Target.Total();
|
|
for(int i = 0; i < count; i++)
|
|
if(!Gradient.Update(i, Target.At(i)))
|
|
return false;
|
|
Gradient.BufferWrite();
|
|
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcOutputGradient failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Gradient.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = count;
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
|
|
//--- BufferRead()s itself - see the note in feedForward() above.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
int inputs = NeuronOCL.Neurons();
|
|
int neurons = Neurons();
|
|
if(optimization == SGD)
|
|
{
|
|
if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
|
|
NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons,
|
|
0))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsMomentum failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
|
|
NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
|
|
inputs, lt, b1, b2, neurons))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsAdam failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
//--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex()
|
|
//--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand.
|
|
return true;
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2];
|
|
global_work_size[0] = Neurons();
|
|
global_work_size[1] = NeuronOCL.Neurons();
|
|
if(optimization == SGD)
|
|
{
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
|
|
return false;
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2))
|
|
return false;
|
|
//--- Dim 1 covers the full (inputs + 1)-wide weight row in groups of 4, so the bias column is
|
|
//--- dispatched too. Sizing this on Neurons() (without the +1) left the bias group unreachable
|
|
//--- whenever inputs % 4 == 0 - see the kernel's 2026-08-11 comment.
|
|
global_work_size[1] = NeuronOCL.Neurons() + 1;
|
|
uint rest = global_work_size[1] % 4;
|
|
global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
//--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
|
|
//--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| MINI-BATCH ACCUMULATE (dense) - see the declaration comment. |
|
|
//| Same dispatch shape as updateInputWeights above, but it only ADDS |
|
|
//| this sample's outer product into GradAccum; no weight is touched |
|
|
//| and no optimizer state advances until ApplyAccumulatedGradients. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
//--- The weight block being accumulated for lives on the SOURCE neuron, exactly as
|
|
//--- updateInputWeights' matrix_w does - so the accumulator is sized and owned there too.
|
|
if(!NeuronOCL.EnsureGradAccum(NeuronOCL.Weights))
|
|
return false;
|
|
int inputs = NeuronOCL.Neurons();
|
|
int neurons = Neurons();
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.AccumulateWeightGrad(NeuronOCL.GradAccum.GetIndex(), getGradientIndex(),
|
|
NeuronOCL.getOutputIndex(), inputs, neurons))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " AccumulateWeightGrad failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2];
|
|
global_work_size[0] = neurons;
|
|
global_work_size[1] = inputs + 1; // the last slot is the bias, exactly as the Adam kernel treats it
|
|
if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_acc, NeuronOCL.GradAccum.GetIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_g, getGradientIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_AccumulateWeightGrad, def_k_awg_matrix_i, NeuronOCL.getOutputIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_AccumulateWeightGrad, def_k_awg_inputs, inputs))
|
|
return false;
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_AccumulateWeightGrad, 2, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel AccumulateWeightGrad: %d", GetLastError());
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Type-dispatching wrapper, mirroring updateInputWeights(CObject*). |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::accumulateInputWeightGrads(CObject *SourceObject)
|
|
{
|
|
if(CheckPointer(SourceObject) == POINTER_INVALID)
|
|
return false;
|
|
//--- Every *OCL neuron type derives from CNeuronBaseOCL and stores its incoming dense matrix in
|
|
//--- Weights, so one branch covers them all. Conv and LSTM own EXTRA weight blocks and override
|
|
//--- this method; the legacy scalar hierarchy never reaches here (CNet refuses to batch without a
|
|
//--- backend - see CNet::BatchSize).
|
|
switch(SourceObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
case defNeuronBatchNormOCL:
|
|
case defNeuronConvOCL:
|
|
case defNeuronPoolOCL:
|
|
case defNeuronLSTMOCL:
|
|
{
|
|
CNeuronBaseOCL *temp = SourceObject;
|
|
return accumulateInputWeightGrads(temp);
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Zero this neuron's accumulator - see the declaration comment. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::BeginGradAccum(void)
|
|
{
|
|
if(CheckPointer(GradAccum) == POINTER_INVALID || GradAccum.Total() <= 0)
|
|
return true; // nothing accumulated here yet; the first accumulate allocates it zeroed
|
|
return ZeroOptimizerBuffer(GradAccum);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| DEVICE-SIDE mini-batch apply (OpenCL only). One dispatch, nothing |
|
|
//| crosses the bus. |
|
|
//| |
|
|
//| This exists because the host-side version below moved EIGHT full |
|
|
//| weight-matrix transfers per batch per weight block - read w, acc, |
|
|
//| m, v; write m, v, w, acc - each a blocking sync. At batch size 8 |
|
|
//| that is roughly one whole weight matrix over the bus PER SAMPLE, |
|
|
//| and it is why an RX 580 lost to a CPU thread pool on this |
|
|
//| workload. Market builds cannot import a DLL, so OpenCL is the |
|
|
//| tier paying clients run: this path is the product's speed. |
|
|
//| |
|
|
//| Returns false (rather than reporting) if anything is missing, so |
|
|
//| the caller can fall back to the host implementation - which stays |
|
|
//| the reference for the DLL and pure-MQL5 tiers and must be kept |
|
|
//| line-for-line identical to the kernels. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale, int total)
|
|
{
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID || total <= 0)
|
|
return false;
|
|
uint offset[1] = {0};
|
|
uint size[1];
|
|
size[0] = (uint)total;
|
|
if(optimization == ADAM)
|
|
{
|
|
if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID ||
|
|
m.Total() < total || v.Total() < total || m.GetIndex() < 0 || v.GetIndex() < 0)
|
|
return false;
|
|
//--- Same bias-corrected step the host path and the unbatched kernel compute. t is NOT advanced
|
|
//--- here - ApplyAccumulatedGradients owns that, once per batch across every block it holds.
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_w, w.GetIndex()) ||
|
|
!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_acc, acc.GetIndex()) ||
|
|
!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_m, m.GetIndex()) ||
|
|
!OpenCL.SetArgumentBuffer(def_k_ApplyAccumAdam, def_k_aaa_matrix_v, v.GetIndex()) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_scale, (float)scale) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_l, (float)lt) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b1, (float)b1) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumAdam, def_k_aaa_b2, (float)b2))
|
|
return false;
|
|
ResetLastError();
|
|
return OpenCL.Execute(def_k_ApplyAccumAdam, 1, offset, size);
|
|
}
|
|
if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total || dw.GetIndex() < 0)
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_w, w.GetIndex()) ||
|
|
!OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_acc, acc.GetIndex()) ||
|
|
!OpenCL.SetArgumentBuffer(def_k_ApplyAccumMomentum, def_k_aam_matrix_dw, dw.GetIndex()) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_scale, (float)scale) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_lr, (float)eta) ||
|
|
!OpenCL.SetArgument(def_k_ApplyAccumMomentum, def_k_aam_momentum, (float)alpha))
|
|
return false;
|
|
ResetLastError();
|
|
return OpenCL.Execute(def_k_ApplyAccumMomentum, 1, offset, size);
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| MINI-BATCH APPLY (dense) - ONE optimizer step on the batch MEAN. |
|
|
//| |
|
|
//| OpenCL takes the device path above. The host implementation below |
|
|
//| remains the one shared by DirectML, the CPU DLL and the pure-MQL5 |
|
|
//| tier, and is the REFERENCE the kernels are transcribed from: it |
|
|
//| is a line-for-line copy of Network.cl's UpdateWeightsAdam / |
|
|
//| UpdateWeightsMomentum, including both clamps and the decoupled |
|
|
//| weight decay, so that batch size 1 reproduces them exactly. Edit |
|
|
//| the two together or the tiers silently disagree. |
|
|
//| |
|
|
//| One deliberate difference remains between the two: this runs in |
|
|
//| fp64 while the kernel runs in fp32. That is not a regression, it |
|
|
//| is the OpenCL tier becoming SELF-consistent - its device buffers |
|
|
//| are already fp32 (see CBufferDouble's m_data_f) and its unbatched |
|
|
//| optimizer already ran in fp32, so the batched path was the odd |
|
|
//| one out. The DLL tiers keep fp64 end to end. |
|
|
//| |
|
|
//| `scale` is 1/batchCount - the MEAN, not the sum. Note Adam is |
|
|
//| very nearly invariant to a global gradient rescale (mt/sqrt(vt) |
|
|
//| cancels it), so this matters mainly for SGD and for keeping the |
|
|
//| decoupled decay term correctly proportioned. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale)
|
|
{
|
|
if(CheckPointer(acc) == POINTER_INVALID || acc.Total() <= 0 ||
|
|
CheckPointer(w) == POINTER_INVALID || w.Total() <= 0)
|
|
return true; // no weight block here (e.g. an output-layer neuron) - nothing to apply
|
|
int total = MathMin(acc.Total(), w.Total());
|
|
//--- DEVICE FAST PATH. Latched off on first failure rather than retried every batch: a kernel that
|
|
//--- did not build will not build later, and the host path below is a correct answer, so the right
|
|
//--- behaviour is one warning and full speed on the fallback - not a failed dispatch per batch.
|
|
if(g_applyAccumKernelUsable && CheckPointer(OpenCL) != POINTER_INVALID && w.GetIndex() >= 0 &&
|
|
acc.GetIndex() >= 0)
|
|
{
|
|
if(ApplyAccumOnDevice(w, acc, m, v, dw, scale, total))
|
|
return true;
|
|
g_applyAccumKernelUsable = false;
|
|
PrintFormat("%s: device-side mini-batch apply failed (error %d) - falling back to the host step "
|
|
"for the rest of this run. Training stays correct; each batch now costs a full "
|
|
"weight-matrix round trip, so expect it to be substantially slower.",
|
|
__FUNCTION__, GetLastError());
|
|
}
|
|
//--- Pull the device-side copies into the host mirrors. On a host-only net (no backend) there is no
|
|
//--- device to read FROM and the host mirror is already the truth, so the round-trip is skipped
|
|
//--- rather than treated as a failure.
|
|
bool haveDevice = (w.GetIndex() >= 0);
|
|
if(haveDevice && (!w.BufferRead() || !acc.BufferRead()))
|
|
return false;
|
|
if(optimization == ADAM)
|
|
{
|
|
if(CheckPointer(m) == POINTER_INVALID || CheckPointer(v) == POINTER_INVALID ||
|
|
m.Total() < total || v.Total() < total)
|
|
return false;
|
|
if(haveDevice && (!m.BufferRead() || !v.BufferRead()))
|
|
return false;
|
|
//--- Bias correction reads t but does NOT advance it here - the caller advances once per batch
|
|
//--- after every block it owns has been stepped, because t counts optimizer STEPS and a batch is
|
|
//--- one step no matter how many weight blocks the neuron carries.
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
double grad = acc.At(i) * scale;
|
|
double wv = w.At(i);
|
|
double mt = b1 * m.At(i) + (1 - b1) * grad;
|
|
//--- v is STORED square-rooted, so square it back before the recursion. See Network.cl's
|
|
//--- UpdateWeightsAdam for why feeding the stored sqrt straight in is not Adam at all.
|
|
double vt = sqrt(b2 * v.At(i) * v.At(i) + (1 - b2) * grad * grad);
|
|
double delta = lt * mt / (vt > 0 ? vt : lt * 10) - lt * WEIGHT_DECAY * wv;
|
|
delta = MathMax(-MAX_WEIGHT_DELTA, MathMin(MAX_WEIGHT_DELTA, delta));
|
|
if(!w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, wv + delta))) ||
|
|
!m.Update(i, mt) || !v.Update(i, vt))
|
|
return false;
|
|
}
|
|
if(haveDevice && (!m.BufferWrite() || !v.BufferWrite()))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(dw) == POINTER_INVALID || dw.Total() < total)
|
|
return false;
|
|
if(haveDevice && !dw.BufferRead())
|
|
return false;
|
|
for(int i = 0; i < total; i++)
|
|
{
|
|
double grad = acc.At(i) * scale;
|
|
double delta = eta * grad + alpha * dw.At(i);
|
|
if(!dw.Update(i, delta) ||
|
|
!w.Update(i, MathMax(-MAX_WEIGHT, MathMin(MAX_WEIGHT, w.At(i) + delta))))
|
|
return false;
|
|
}
|
|
if(haveDevice && !dw.BufferWrite())
|
|
return false;
|
|
}
|
|
//--- Clear the accumulator for the next batch and push the new weights back to the device.
|
|
for(int i = 0; i < acc.Total(); i++)
|
|
if(!acc.Update(i, 0.0))
|
|
return false;
|
|
if(haveDevice && (!w.BufferWrite() || !acc.BufferWrite()))
|
|
return false;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::ApplyAccumulatedGradients(double scale)
|
|
{
|
|
if(!ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale))
|
|
return false;
|
|
if(optimization == ADAM)
|
|
t++;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
|
|
{
|
|
if(CheckPointer(TargetObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
CNeuronConvOCL *tempConv = NULL;
|
|
CNeuronLSTMOCL *tempLstm = NULL;
|
|
CNeuronPoolOCL *tempPool = NULL;
|
|
CNeuronBatchNormOCL *tempBN = NULL;
|
|
switch(TargetObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
temp = TargetObject;
|
|
return calcHiddenGradients(temp);
|
|
break;
|
|
case defNeuronBatchNormOCL:
|
|
//--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step
|
|
//--- and writes into this->Gradient. Routing it through the dense branch instead would run
|
|
//--- CaclHiddenGradient against a weight matrix batch norm does not have.
|
|
tempBN = TargetObject;
|
|
return tempBN.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronConvOCL:
|
|
//--- Conv owns the backward step (calcInputGradients), called on itself with
|
|
//--- "this" (the earlier layer) passed in so it writes into this->Gradient.
|
|
tempConv = TargetObject;
|
|
return tempConv.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronLSTMOCL:
|
|
tempLstm = TargetObject;
|
|
return tempLstm.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronPoolOCL:
|
|
tempPool = TargetObject;
|
|
return tempPool.calcInputGradients(GetPointer(this));
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
|
|
{
|
|
if(CheckPointer(SourceObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
switch(SourceObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
case defNeuronConvOCL:
|
|
case defNeuronLSTMOCL:
|
|
case defNeuronPoolOCL:
|
|
case defNeuronBatchNormOCL:
|
|
temp = SourceObject;
|
|
return updateInputWeights(temp);
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Save(const int file_handle)
|
|
{
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
|
|
return false;
|
|
//---
|
|
if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
FileWriteInteger(file_handle, 0);
|
|
return true;
|
|
}
|
|
else
|
|
FileWriteInteger(file_handle, 1);
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
|
|
return false;
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Load(const int file_handle)
|
|
{
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
//---
|
|
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
|
|
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
t = FileReadInteger(file_handle, INT_VALUE);
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Output.GetIndex() >= 0)
|
|
Output.BufferFree();
|
|
if(!Output.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Output))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(PrevOutput.GetIndex() >= 0)
|
|
PrevOutput.BufferFree();
|
|
if(!PrevOutput.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(PrevOutput))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Gradient.GetIndex() >= 0)
|
|
Gradient.BufferFree();
|
|
if(!Gradient.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Gradient))
|
|
return false;
|
|
//---
|
|
if(FileReadInteger(file_handle) == 0)
|
|
return true;
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Weights.GetIndex() >= 0)
|
|
Weights.BufferFree();
|
|
if(!Weights.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Weights))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(DeltaWeights.GetIndex() >= 0)
|
|
DeltaWeights.BufferFree();
|
|
if(!DeltaWeights.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(DeltaWeights))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(FirstMomentum.GetIndex() >= 0)
|
|
FirstMomentum.BufferFree();
|
|
if(!FirstMomentum.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(FirstMomentum))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(SecondMomentum.GetIndex() >= 0)
|
|
SecondMomentum.BufferFree();
|
|
if(!SecondMomentum.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(SecondMomentum))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
#endif
|