5f647ba5db
- BufferDouble: replace hardcoded "DirectML/CPU-DLL" with dynamic backend name and add buffer index/element count to all error prints for easier debugging. - NetPersistence: distinguish missing file from transient lock by probing FileIsExist before logging, eliminating false "sharing violation" warnings when no saved model exists on first run.
864 lines
34 KiB
MQL5
864 lines
34 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| NeuronOCLBase.mqh |
|
|
//| |
|
|
//| CNeuronBaseOCL - the accelerated dense neuron (OpenCL / DirectML |
|
|
//| / CPU-DLL tiers). |
|
|
//| |
|
|
//| Included from AI\Network.mqh AFTER every class declaration - |
|
|
//| bodies only, no declarations. Relocation is behaviour-neutral by |
|
|
//| construction: nothing here is reachable until Network.mqh ends. |
|
|
//+------------------------------------------------------------------+
|
|
#ifndef WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
|
|
#define WARRIOR_AI_IMPL_NEURONOCLBASE_MQH
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum),
|
|
activation(TANH),
|
|
optimization(SGD),
|
|
t(1)
|
|
{
|
|
OpenCL = NULL;
|
|
DirectML = NULL;
|
|
Output = new CBufferDouble();
|
|
PrevOutput = new CBufferDouble();
|
|
Weights = new CBufferDouble();
|
|
DeltaWeights = new CBufferDouble();
|
|
Gradient = new CBufferDouble();
|
|
FirstMomentum = new CBufferDouble();
|
|
SecondMomentum = new CBufferDouble();
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
CNeuronBaseOCL::~CNeuronBaseOCL(void)
|
|
{
|
|
if(CheckPointer(Output) != POINTER_INVALID)
|
|
delete Output;
|
|
if(CheckPointer(PrevOutput) != POINTER_INVALID)
|
|
delete PrevOutput;
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
if(CheckPointer(Gradient) != POINTER_INVALID)
|
|
delete Gradient;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
delete FirstMomentum;
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
delete SecondMomentum;
|
|
OpenCL = NULL;
|
|
DirectML = NULL;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0)
|
|
return false;
|
|
OpenCL = open_cl;
|
|
optimization = optimization_type;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Output.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!Output.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!PrevOutput.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!PrevOutput.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
|
|
return false;
|
|
if(!Gradient.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(numOutputs > 0)
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int count = (int)((numNeurons + 1) * numOutputs);
|
|
if(!Weights.Reserve(count))
|
|
return false;
|
|
// He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He
|
|
// et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the
|
|
// LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a
|
|
// 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky
|
|
// ReLU family) - He is the variant actually derived for ReLU-family activations, accounting
|
|
// for the fact that they zero out roughly half their input distribution, whereas the
|
|
// previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and
|
|
// was ~2x too conservative here. Applied to every layer through this one shared Init()
|
|
// (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through -
|
|
// the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely
|
|
// differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting
|
|
// extremes on every backend, so the imprecision there is not worth the much larger, riskier
|
|
// change of threading activation awareness through every neuron subtype's Init() overload.
|
|
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!Weights.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!Weights.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeights.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeights.BufferCreate(OpenCL))
|
|
return false;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
{
|
|
delete FirstMomentum;
|
|
FirstMomentum = NULL;
|
|
}
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
{
|
|
delete SecondMomentum;
|
|
SecondMomentum = NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
{
|
|
delete DeltaWeights;
|
|
DeltaWeights = NULL;
|
|
}
|
|
//---
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentum.BufferCreate(OpenCL))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentum.BufferCreate(OpenCL))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same |
|
|
//| buffer layout, buffers just get created on the DML backend. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type)
|
|
{
|
|
if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0)
|
|
return false;
|
|
DirectML = direct_ml;
|
|
optimization = optimization_type;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Output.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!Output.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!PrevOutput.BufferInit(numNeurons, 1.0))
|
|
return false;
|
|
if(!PrevOutput.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!Gradient.BufferInit(numNeurons + 1, 0.0))
|
|
return false;
|
|
if(!Gradient.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(numOutputs > 0)
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
int count = (int)((numNeurons + 1) * numOutputs);
|
|
if(!Weights.Reserve(count))
|
|
return false;
|
|
// He-scaled init - see the matching OpenCL Init() overload above for the full rationale.
|
|
double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0));
|
|
for(int i = 0; i < count; i++)
|
|
{
|
|
double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale;
|
|
if(weigh == 0)
|
|
weigh = 0.001;
|
|
if(!Weights.Add(weigh))
|
|
return false;
|
|
}
|
|
if(!Weights.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!DeltaWeights.BufferInit(count, 0))
|
|
return false;
|
|
if(!DeltaWeights.BufferCreate(DirectML))
|
|
return false;
|
|
if(CheckPointer(FirstMomentum) != POINTER_INVALID)
|
|
{
|
|
delete FirstMomentum;
|
|
FirstMomentum = NULL;
|
|
}
|
|
if(CheckPointer(SecondMomentum) != POINTER_INVALID)
|
|
{
|
|
delete SecondMomentum;
|
|
SecondMomentum = NULL;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
{
|
|
delete DeltaWeights;
|
|
DeltaWeights = NULL;
|
|
}
|
|
//---
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!FirstMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!FirstMomentum.BufferCreate(DirectML))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(!SecondMomentum.BufferInit(count, 0))
|
|
return false;
|
|
if(!SecondMomentum.BufferCreate(DirectML))
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(Weights) != POINTER_INVALID)
|
|
delete Weights;
|
|
if(CheckPointer(DeltaWeights) != POINTER_INVALID)
|
|
delete DeltaWeights;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForward(CObject *SourceObject)
|
|
{
|
|
if(CheckPointer(SourceObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
switch(SourceObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
case defNeuronConvOCL:
|
|
case defNeuronLSTMOCL:
|
|
case defNeuronPoolOCL:
|
|
case defNeuronBatchNormOCL:
|
|
temp = SourceObject;
|
|
return feedForward(temp);
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(),
|
|
NeuronOCL.Neurons(), NativeActivationCode(activation)))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " FeedForward failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Output.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = Output.Total();
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex());
|
|
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation));
|
|
if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel FeedForward: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex()
|
|
//--- (a device buffer handle), never through this CPU mirror. Any caller that does need the
|
|
//--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through
|
|
//--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData().
|
|
//--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Pure-MQL5 dense forward pass - exact double-precision mirror of |
|
|
//| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, |
|
|
//| which (Gizlyk convention) owns both the inputs (its Output) and |
|
|
//| the weights connecting them to THIS layer (its Weights), laid out |
|
|
//| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host |
|
|
//| buffers only; used when no compute backend exists (CPU inference).|
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
int inputs = NeuronOCL.Neurons();
|
|
int outCount = Output.Total();
|
|
int wTotal = NeuronOCL.WeightsCount();
|
|
if(wTotal < (inputs + 1) * outCount)
|
|
return false; // weight buffer smaller than the dense layout requires - refuse rather than misread
|
|
for(int i = 0; i < outCount; i++)
|
|
{
|
|
int shift = (inputs + 1) * i;
|
|
double sum = 0.0;
|
|
for(int k = 0; k < inputs; k++)
|
|
sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k);
|
|
sum += NeuronOCL.WeightHost(shift + inputs); // bias
|
|
switch(activation)
|
|
{
|
|
case TANH:
|
|
sum = tanh(sum);
|
|
break;
|
|
case SIGMOID:
|
|
sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum))));
|
|
break;
|
|
case PRELU:
|
|
if(sum < 0.0)
|
|
sum *= 0.01;
|
|
break;
|
|
// NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case.
|
|
}
|
|
if(!Output.Update(i, sum))
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Host-side input write for the CPU inference path's layer 0 (there |
|
|
//| is no device buffer to write into, unlike CNet::feedForward's |
|
|
//| BufferWrite branch). Copies inputVals into the Output host array. |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals)
|
|
{
|
|
if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
int total = MathMin(Output.Total(), inputVals.Total());
|
|
for(int i = 0; i < total; i++)
|
|
if(!Output.Update(i, inputVals.At(i)))
|
|
return false;
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Host-side output read for getResults() on the CPU inference path |
|
|
//| (no device BufferRead available). Returns the count copied. |
|
|
//+------------------------------------------------------------------+
|
|
int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values)
|
|
{
|
|
if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID)
|
|
return 0;
|
|
values.Clear();
|
|
int n = Output.Total();
|
|
for(int i = 0; i < n; i++)
|
|
if(!values.Add(Output.At(i)))
|
|
return i;
|
|
return n;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(),
|
|
NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcHiddenGradient failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Gradient.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = Neurons() + 1;
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation));
|
|
if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see
|
|
//--- the note in feedForward() above - self-syncing GetData() covers any real host consumer.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target)
|
|
{
|
|
if(CheckPointer(Target) == POINTER_INVALID)
|
|
return false;
|
|
int count = Target.Total();
|
|
for(int i = 0; i < count; i++)
|
|
if(!Gradient.Update(i, Target.At(i)))
|
|
return false;
|
|
Gradient.BufferWrite();
|
|
//--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcOutputGradient failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
return Gradient.BufferRead();
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[1] = {0};
|
|
uint global_work_size[1];
|
|
global_work_size[0] = count;
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex());
|
|
OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation));
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel CaclOutputGradient: %d", GetLastError());
|
|
return false;
|
|
}
|
|
//--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which
|
|
//--- BufferRead()s itself - see the note in feedForward() above.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL)
|
|
{
|
|
if(CheckPointer(NeuronOCL) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
{
|
|
int inputs = NeuronOCL.Neurons();
|
|
int neurons = Neurons();
|
|
if(optimization == SGD)
|
|
{
|
|
if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
|
|
NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons,
|
|
0))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsMomentum failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(),
|
|
NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(),
|
|
inputs, lt, b1, b2, neurons))
|
|
{
|
|
Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsAdam failed, error " + IntegerToString(DirectML.LastError()));
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
//--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex()
|
|
//--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand.
|
|
return true;
|
|
}
|
|
if(CheckPointer(OpenCL) == POINTER_INVALID)
|
|
return false;
|
|
uint global_work_offset[2] = {0, 0};
|
|
uint global_work_size[2];
|
|
global_work_size[0] = Neurons();
|
|
global_work_size[1] = NeuronOCL.Neurons();
|
|
if(optimization == SGD)
|
|
{
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex());
|
|
OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons());
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha);
|
|
OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError());
|
|
return false;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()))
|
|
return false;
|
|
if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()))
|
|
return false;
|
|
double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t));
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1))
|
|
return false;
|
|
if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2))
|
|
return false;
|
|
uint rest = global_work_size[1] % 4;
|
|
global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0);
|
|
ResetLastError();
|
|
if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size))
|
|
{
|
|
printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError());
|
|
return false;
|
|
}
|
|
t++;
|
|
}
|
|
//--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save()
|
|
//--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself.
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject)
|
|
{
|
|
if(CheckPointer(TargetObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
CNeuronConvOCL *tempConv = NULL;
|
|
CNeuronLSTMOCL *tempLstm = NULL;
|
|
CNeuronPoolOCL *tempPool = NULL;
|
|
CNeuronBatchNormOCL *tempBN = NULL;
|
|
switch(TargetObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
temp = TargetObject;
|
|
return calcHiddenGradients(temp);
|
|
break;
|
|
case defNeuronBatchNormOCL:
|
|
//--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step
|
|
//--- and writes into this->Gradient. Routing it through the dense branch instead would run
|
|
//--- CaclHiddenGradient against a weight matrix batch norm does not have.
|
|
tempBN = TargetObject;
|
|
return tempBN.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronConvOCL:
|
|
//--- Conv owns the backward step (calcInputGradients), called on itself with
|
|
//--- "this" (the earlier layer) passed in so it writes into this->Gradient.
|
|
tempConv = TargetObject;
|
|
return tempConv.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronLSTMOCL:
|
|
tempLstm = TargetObject;
|
|
return tempLstm.calcInputGradients(GetPointer(this));
|
|
break;
|
|
case defNeuronPoolOCL:
|
|
tempPool = TargetObject;
|
|
return tempPool.calcInputGradients(GetPointer(this));
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject)
|
|
{
|
|
if(CheckPointer(SourceObject) == POINTER_INVALID)
|
|
return false;
|
|
//---
|
|
CNeuronBaseOCL *temp = NULL;
|
|
switch(SourceObject.Type())
|
|
{
|
|
case defNeuronBaseOCL:
|
|
case defNeuronConvOCL:
|
|
case defNeuronLSTMOCL:
|
|
case defNeuronPoolOCL:
|
|
case defNeuronBatchNormOCL:
|
|
temp = SourceObject;
|
|
return updateInputWeights(temp);
|
|
break;
|
|
}
|
|
//---
|
|
return false;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Save(const int file_handle)
|
|
{
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, Type()) < INT_VALUE)
|
|
return false;
|
|
//---
|
|
if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE)
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
FileWriteInteger(file_handle, 0);
|
|
return true;
|
|
}
|
|
else
|
|
FileWriteInteger(file_handle, 1);
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle))
|
|
return false;
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle))
|
|
return false;
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
bool CNeuronBaseOCL::Load(const int file_handle)
|
|
{
|
|
if(file_handle == INVALID_HANDLE)
|
|
return false;
|
|
//---
|
|
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
|
|
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
t = FileReadInteger(file_handle, INT_VALUE);
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
{
|
|
Output = new CBufferDouble();
|
|
if(CheckPointer(Output) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Output.GetIndex() >= 0)
|
|
Output.BufferFree();
|
|
if(!Output.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Output))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
{
|
|
PrevOutput = new CBufferDouble();
|
|
if(CheckPointer(PrevOutput) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(PrevOutput.GetIndex() >= 0)
|
|
PrevOutput.BufferFree();
|
|
if(!PrevOutput.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(PrevOutput))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
{
|
|
Gradient = new CBufferDouble();
|
|
if(CheckPointer(Gradient) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Gradient.GetIndex() >= 0)
|
|
Gradient.BufferFree();
|
|
if(!Gradient.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Gradient))
|
|
return false;
|
|
//---
|
|
if(FileReadInteger(file_handle) == 0)
|
|
return true;
|
|
//---
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
{
|
|
Weights = new CBufferDouble();
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(Weights.GetIndex() >= 0)
|
|
Weights.BufferFree();
|
|
if(!Weights.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(Weights))
|
|
return false;
|
|
//---
|
|
if(optimization == SGD)
|
|
{
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
{
|
|
DeltaWeights = new CBufferDouble();
|
|
if(CheckPointer(DeltaWeights) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(DeltaWeights.GetIndex() >= 0)
|
|
DeltaWeights.BufferFree();
|
|
if(!DeltaWeights.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(DeltaWeights))
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
{
|
|
FirstMomentum = new CBufferDouble();
|
|
if(CheckPointer(FirstMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(FirstMomentum.GetIndex() >= 0)
|
|
FirstMomentum.BufferFree();
|
|
if(!FirstMomentum.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(FirstMomentum))
|
|
return false;
|
|
//---
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
{
|
|
SecondMomentum = new CBufferDouble();
|
|
if(CheckPointer(SecondMomentum) == POINTER_INVALID)
|
|
return false;
|
|
}
|
|
if(SecondMomentum.GetIndex() >= 0)
|
|
SecondMomentum.BufferFree();
|
|
if(!SecondMomentum.Load(file_handle))
|
|
return false;
|
|
if(!BackendBufferCreate(SecondMomentum))
|
|
return false;
|
|
}
|
|
//---
|
|
return true;
|
|
}
|
|
#endif
|