//+------------------------------------------------------------------+ //| NeuronOCLBase.mqh | //| | //| CNeuronBaseOCL - the accelerated dense neuron (OpenCL / DirectML | //| / CPU-DLL tiers). | //| | //| Included from AI\Network.mqh AFTER every class declaration - | //| bodies only, no declarations. Relocation is behaviour-neutral by | //| construction: nothing here is reachable until Network.mqh ends. | //+------------------------------------------------------------------+ #ifndef WARRIOR_AI_IMPL_NEURONOCLBASE_MQH #define WARRIOR_AI_IMPL_NEURONOCLBASE_MQH //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum), activation(TANH), optimization(SGD), t(1) { OpenCL = NULL; DirectML = NULL; Output = new CBufferDouble(); PrevOutput = new CBufferDouble(); Weights = new CBufferDouble(); DeltaWeights = new CBufferDouble(); Gradient = new CBufferDouble(); FirstMomentum = new CBufferDouble(); SecondMomentum = new CBufferDouble(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::~CNeuronBaseOCL(void) { if(CheckPointer(Output) != POINTER_INVALID) delete Output; if(CheckPointer(PrevOutput) != POINTER_INVALID) delete PrevOutput; if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; if(CheckPointer(Gradient) != POINTER_INVALID) delete Gradient; if(CheckPointer(FirstMomentum) != POINTER_INVALID) delete FirstMomentum; if(CheckPointer(SecondMomentum) != POINTER_INVALID) delete SecondMomentum; OpenCL = NULL; DirectML = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) return false; OpenCL = open_cl; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(OpenCL)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init: k=sqrt(2/fan_in), weight drawn uniform in [-k,k] (variance-matched to He // et al.'s normal-distribution formulation, just uniform instead of Gaussian - same as the // LeCun-uniform scheme this replaced, which used the same uniform-draw convention with a // 1/sqrt(fan_in+1) scale). BuildFreshTopology() puts every hidden layer on PRELU (leaky // ReLU family) - He is the variant actually derived for ReLU-family activations, accounting // for the fact that they zero out roughly half their input distribution, whereas the // previous LeCun-uniform scale was tuned for tanh/sigmoid-style saturating activations and // was ~2x too conservative here. Applied to every layer through this one shared Init() // (input/output included, not just hidden) rather than threading ENUM_ACTIVATION through - // the output layer is only m_outputNeuronsCount (3) neurons wide, where fan-in barely // differs from the old scale's, and MAX_WEIGHT/MAX_WEIGHT_DELTA already clip any resulting // extremes on every backend, so the imprecision there is not worth the much larger, riskier // change of threading activation awareness through every neuron subtype's Init() overload. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(OpenCL)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(OpenCL)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(OpenCL)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(OpenCL)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| DirectML/D3D12 tier equivalent of Init(COpenCLMy*) above - same | //| buffer layout, buffers just get created on the DML backend. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(direct_ml) == POINTER_INVALID || numNeurons <= 0) return false; DirectML = direct_ml; optimization = optimization_type; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(!Output.BufferInit(numNeurons, 1.0)) return false; if(!Output.BufferCreate(DirectML)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) return false; if(!PrevOutput.BufferCreate(DirectML)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(!Gradient.BufferInit(numNeurons + 1, 0.0)) return false; if(!Gradient.BufferCreate(DirectML)) return false; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) return false; // He-scaled init - see the matching OpenCL Init() overload above for the full rationale. double weighScale = MathSqrt(2.0 / ((double)numNeurons + 1.0)); for(int i = 0; i < count; i++) { double weigh = ((MathRand() + 1) / 32768.0 - 0.5) * 2.0 * weighScale; if(weigh == 0) weigh = 0.001; if(!Weights.Add(weigh)) return false; } if(!Weights.BufferCreate(DirectML)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(!DeltaWeights.BufferInit(count, 0)) return false; if(!DeltaWeights.BufferCreate(DirectML)) return false; if(CheckPointer(FirstMomentum) != POINTER_INVALID) { delete FirstMomentum; FirstMomentum = NULL; } if(CheckPointer(SecondMomentum) != POINTER_INVALID) { delete SecondMomentum; SecondMomentum = NULL; } } else { if(CheckPointer(DeltaWeights) != POINTER_INVALID) { delete DeltaWeights; DeltaWeights = NULL; } //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(!FirstMomentum.BufferInit(count, 0)) return false; if(!FirstMomentum.BufferCreate(DirectML)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(!SecondMomentum.BufferInit(count, 0)) return false; if(!SecondMomentum.BufferCreate(DirectML)) return false; } } else { if(CheckPointer(Weights) != POINTER_INVALID) delete Weights; if(CheckPointer(DeltaWeights) != POINTER_INVALID) delete DeltaWeights; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return feedForward(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.FeedForward(NeuronOCL.getWeightsIndex(), NeuronOCL.getOutputIndex(), Output.GetIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation))) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " FeedForward failed, error " + IntegerToString(DirectML.LastError())); return false; } return Output.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_FeedForward, def_k_ff_matrix_o, Output.GetIndex()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_FeedForward, def_k_ff_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_FeedForward, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel FeedForward: %d", GetLastError()); return false; } //--- Output stays GPU-resident; the next layer's feedForward reads it via getOutputIndex() //--- (a device buffer handle), never through this CPU mirror. Any caller that does need the //--- host-side array (getResults(), backPropOCL()'s error-metric read) goes through //--- getOutputVal()/GetData(), which calls BufferRead() itself - see CBufferDouble::GetData(). //--- Eagerly reading here on every layer, every sample was pure host<->device sync overhead. return true; } //+------------------------------------------------------------------+ //| Pure-MQL5 dense forward pass - exact double-precision mirror of | //| Network.cl's FeedForward kernel. NeuronOCL is the PREVIOUS layer, | //| which (Gizlyk convention) owns both the inputs (its Output) and | //| the weights connecting them to THIS layer (its Weights), laid out | //| [thisNeuron][prevNeurons+1] with the +1 bias last. Reads host | //| buffers only; used when no compute backend exists (CPU inference).| //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForwardCPU(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int inputs = NeuronOCL.Neurons(); int outCount = Output.Total(); int wTotal = NeuronOCL.WeightsCount(); if(wTotal < (inputs + 1) * outCount) return false; // weight buffer smaller than the dense layout requires - refuse rather than misread for(int i = 0; i < outCount; i++) { int shift = (inputs + 1) * i; double sum = 0.0; for(int k = 0; k < inputs; k++) sum += NeuronOCL.OutputHost(k) * NeuronOCL.WeightHost(shift + k); sum += NeuronOCL.WeightHost(shift + inputs); // bias switch(activation) { case TANH: sum = tanh(sum); break; case SIGMOID: sum = 1.0 / (1.0 + exp(-MathMax(-50.0, MathMin(50.0, sum)))); break; case PRELU: if(sum < 0.0) sum *= 0.01; break; // NONE (raw logits): identity - matches NativeActivationCode()'s -1/default kernel case. } if(!Output.Update(i, sum)) return false; } return true; } //+------------------------------------------------------------------+ //| Host-side input write for the CPU inference path's layer 0 (there | //| is no device buffer to write into, unlike CNet::feedForward's | //| BufferWrite branch). Copies inputVals into the Output host array. | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SetInputsCPU(CArrayDouble *inputVals) { if(CheckPointer(inputVals) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return false; int total = MathMin(Output.Total(), inputVals.Total()); for(int i = 0; i < total; i++) if(!Output.Update(i, inputVals.At(i))) return false; return true; } //+------------------------------------------------------------------+ //| Host-side output read for getResults() on the CPU inference path | //| (no device BufferRead available). Returns the count copied. | //+------------------------------------------------------------------+ int CNeuronBaseOCL::GetOutputsCPU(CArrayDouble *values) { if(CheckPointer(values) == POINTER_INVALID || CheckPointer(Output) == POINTER_INVALID) return 0; values.Clear(); int n = Output.Total(); for(int i = 0; i < n; i++) if(!values.Add(Output.At(i))) return i; return n; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcHiddenGradient(getWeightsIndex(), NeuronOCL.getGradientIndex(), getOutputIndex(), getGradientIndex(), NeuronOCL.Neurons(), NativeActivationCode(activation), Neurons() + 1)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcHiddenGradient failed, error " + IntegerToString(DirectML.LastError())); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons() + 1; OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_w, getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclHiddenGradient, def_k_chg_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_CaclHiddenGradient, def_k_chg_activation, NativeActivationCode(activation)); if(!OpenCL.Execute(def_k_CaclHiddenGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclHiddenGradient: %d", GetLastError()); return false; } //--- Gradient stays GPU-resident (consumed by the previous layer via getGradientIndex()); see //--- the note in feedForward() above - self-syncing GetData() covers any real host consumer. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcOutputGradients(CArrayDouble *Target) { if(CheckPointer(Target) == POINTER_INVALID) return false; int count = Target.Total(); for(int i = 0; i < count; i++) if(!Gradient.Update(i, Target.At(i))) return false; Gradient.BufferWrite(); //--- note: Gradient is reused here as matrix_t (target) below, exactly as the OpenCL path does if(CheckPointer(DirectML) != POINTER_INVALID) { if(!DirectML.CalcOutputGradient(getGradientIndex(), getOutputIndex(), getGradientIndex(), NativeActivationCode(activation), count)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " CalcOutputGradient failed, error " + IntegerToString(DirectML.LastError())); return false; } return Gradient.BufferRead(); } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_t, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_o, getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_CaclOutputGradient, def_k_cog_matrix_ig, getGradientIndex()); OpenCL.SetArgument(def_k_CaclOutputGradient, def_k_cog_activation, NativeActivationCode(activation)); ResetLastError(); if(!OpenCL.Execute(def_k_CaclOutputGradient, 1, global_work_offset, global_work_size)) { printf("Error of execution kernel CaclOutputGradient: %d", GetLastError()); return false; } //--- backPropOCL()'s sampleWeight scaling reads this via getGradient()/GetData(), which //--- BufferRead()s itself - see the note in feedForward() above. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) return false; if(CheckPointer(DirectML) != POINTER_INVALID) { int inputs = NeuronOCL.Neurons(); int neurons = Neurons(); if(optimization == SGD) { if(!DirectML.UpdateWeightsMomentum(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getDeltaWeightsIndex(), inputs, eta, alpha, neurons, 0)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsMomentum failed, error " + IntegerToString(DirectML.LastError())); return false; } } else { double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!DirectML.UpdateWeightsAdam(NeuronOCL.getWeightsIndex(), getGradientIndex(), NeuronOCL.getOutputIndex(), NeuronOCL.getFirstMomentumIndex(), NeuronOCL.getSecondMomentumIndex(), inputs, lt, b1, b2, neurons)) { Print(__FUNCTION__ + ": " + DirectML.BackendName() + " UpdateWeightsAdam failed, error " + IntegerToString(DirectML.LastError())); return false; } t++; } //--- Weights stay DLL-resident; the next feedForward/backProp reads them via getWeightsIndex() //--- (same as the OpenCL branch below). Save()/BlendWeightsFrom() BufferRead() on demand. return true; } if(CheckPointer(OpenCL) == POINTER_INVALID) return false; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons(); if(optimization == SGD) { OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex()); OpenCL.SetArgumentBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons()); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, (float)eta); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, (float)alpha); OpenCL.SetArgument(def_k_UpdateWeightsMomentum, def_k_uwm_optimizer, 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsMomentum, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsMomentum: %d", GetLastError()); return false; } } else { if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex())) return false; if(!OpenCL.SetArgumentBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex())) return false; double lt = eta * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t)); if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons())) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, (float)lt)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, (float)b1)) return false; if(!OpenCL.SetArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, (float)b2)) return false; uint rest = global_work_size[1] % 4; global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0); ResetLastError(); if(!OpenCL.Execute(def_k_UpdateWeightsAdam, 2, global_work_offset, global_work_size)) { printf("Error of execution kernel UpdateWeightsAdam: %d", GetLastError()); return false; } t++; } //--- Weights stays GPU-resident; the next feedForward reads it via getWeightsIndex(). Save() //--- and BlendWeightsFrom()'s getWeights() both call GetData(), which BufferRead()s itself. return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CObject *TargetObject) { if(CheckPointer(TargetObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; CNeuronConvOCL *tempConv = NULL; CNeuronLSTMOCL *tempLstm = NULL; CNeuronPoolOCL *tempPool = NULL; CNeuronBatchNormOCL *tempBN = NULL; switch(TargetObject.Type()) { case defNeuronBaseOCL: temp = TargetObject; return calcHiddenGradients(temp); break; case defNeuronBatchNormOCL: //--- Same inverted-call convention as conv/pool/LSTM: batch norm owns its own backward step //--- and writes into this->Gradient. Routing it through the dense branch instead would run //--- CaclHiddenGradient against a weight matrix batch norm does not have. tempBN = TargetObject; return tempBN.calcInputGradients(GetPointer(this)); break; case defNeuronConvOCL: //--- Conv owns the backward step (calcInputGradients), called on itself with //--- "this" (the earlier layer) passed in so it writes into this->Gradient. tempConv = TargetObject; return tempConv.calcInputGradients(GetPointer(this)); break; case defNeuronLSTMOCL: tempLstm = TargetObject; return tempLstm.calcInputGradients(GetPointer(this)); break; case defNeuronPoolOCL: tempPool = TargetObject; return tempPool.calcInputGradients(GetPointer(this)); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CObject *SourceObject) { if(CheckPointer(SourceObject) == POINTER_INVALID) return false; //--- CNeuronBaseOCL *temp = NULL; switch(SourceObject.Type()) { case defNeuronBaseOCL: case defNeuronConvOCL: case defNeuronLSTMOCL: case defNeuronPoolOCL: case defNeuronBatchNormOCL: temp = SourceObject; return updateInputWeights(temp); break; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) return false; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) return false; if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE) return false; //--- if(CheckPointer(Output) == POINTER_INVALID || !Output.BufferRead() || !Output.Save(file_handle)) return false; if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.BufferRead() || !PrevOutput.Save(file_handle)) return false; if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.BufferRead() || !Gradient.Save(file_handle)) return false; //--- if(CheckPointer(Weights) == POINTER_INVALID) { FileWriteInteger(file_handle, 0); return true; } else FileWriteInteger(file_handle, 1); //--- if(CheckPointer(Weights) == POINTER_INVALID || !Weights.BufferRead() || !Weights.Save(file_handle)) return false; if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.BufferRead() || !DeltaWeights.Save(file_handle)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.BufferRead() || !FirstMomentum.Save(file_handle)) return false; if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.BufferRead() || !SecondMomentum.Save(file_handle)) return false; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) return false; //--- activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferDouble(); if(CheckPointer(Output) == POINTER_INVALID) return false; } if(Output.GetIndex() >= 0) Output.BufferFree(); if(!Output.Load(file_handle)) return false; if(!BackendBufferCreate(Output)) return false; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferDouble(); if(CheckPointer(PrevOutput) == POINTER_INVALID) return false; } if(PrevOutput.GetIndex() >= 0) PrevOutput.BufferFree(); if(!PrevOutput.Load(file_handle)) return false; if(!BackendBufferCreate(PrevOutput)) return false; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferDouble(); if(CheckPointer(Gradient) == POINTER_INVALID) return false; } if(Gradient.GetIndex() >= 0) Gradient.BufferFree(); if(!Gradient.Load(file_handle)) return false; if(!BackendBufferCreate(Gradient)) return false; //--- if(FileReadInteger(file_handle) == 0) return true; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferDouble(); if(CheckPointer(Weights) == POINTER_INVALID) return false; } if(Weights.GetIndex() >= 0) Weights.BufferFree(); if(!Weights.Load(file_handle)) return false; if(!BackendBufferCreate(Weights)) return false; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferDouble(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) return false; } if(DeltaWeights.GetIndex() >= 0) DeltaWeights.BufferFree(); if(!DeltaWeights.Load(file_handle)) return false; if(!BackendBufferCreate(DeltaWeights)) return false; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferDouble(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) return false; } if(FirstMomentum.GetIndex() >= 0) FirstMomentum.BufferFree(); if(!FirstMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(FirstMomentum)) return false; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferDouble(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) return false; } if(SecondMomentum.GetIndex() >= 0) SecondMomentum.BufferFree(); if(!SecondMomentum.Load(file_handle)) return false; if(!BackendBufferCreate(SecondMomentum)) return false; } //--- return true; } #endif