//+------------------------------------------------------------------+ //| NeuroNet_Misc.mqh | //| Copyright 2026, DNG | //| https://www.mql5.com/ru/users/dng | //+------------------------------------------------------------------+ #ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE //+------------------------------------------------------------------+ //| Logical method implementations generated from NeuroNet.mqh: | //| Misc. | //+------------------------------------------------------------------+ //| Release the runtime-only posterior samplers. | //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiModel layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Init(uint numInputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models ) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) ReturnFalse; OpenCL = open_cl; optimization = ADAM; iBatch = 1; iModels = models; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferFloat(); if(CheckPointer(Output) == POINTER_INVALID) ReturnFalse; } if(!Output.BufferInit(numNeurons * models, 0.0)) ReturnFalse; if(!Output.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferFloat(); if(CheckPointer(PrevOutput) == POINTER_INVALID) ReturnFalse; } if(!PrevOutput.BufferInit(numNeurons * models, 1.0)) ReturnFalse; if(!PrevOutput.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferFloat(); if(CheckPointer(Gradient) == POINTER_INVALID) ReturnFalse; } if(!Gradient.BufferInit((numNeurons + 1)*models, 0.0)) ReturnFalse; if(!Gradient.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) ReturnFalse; } int count = (int)((numInputs + 1) * numNeurons * models); if(!Weights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(numInputs + 1)); for(int i = 0; i < count; i++) { if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!Weights.BufferCreate(OpenCL)) ReturnFalse; //--- DeleteObj(DeltaWeights); //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!FirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentum.BufferInit(count, 0)) ReturnFalse; if(!SecondMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Assemble the posterior parameter tensor. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Output.Total() / iModels; global_work_size[1] = iModels; setBuffer(def_k_FFMultiModels, def_k_ff_matrix_w, getWeightsIndex()) setBuffer(def_k_FFMultiModels, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_FFMultiModels, def_k_ff_matrix_o, Output.GetIndex()) setArgument(def_k_FFMultiModels, def_k_ff_inputs, NeuronOCL.Neurons() / iModels) setArgument(def_k_FFMultiModels, def_k_ff_activation, (int)activation) kernelExecute(def_k_FFMultiModels, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMultiModel calc hidden gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::calcHiddenGradients(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iModels; global_work_size[1] = iModels; setBuffer(def_k_HGMultiModels, def_k_chg_matrix_w, getWeightsIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_g, getGradientIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(def_k_HGMultiModels, def_k_chg_outputs, Neurons() / iModels) setArgument(def_k_HGMultiModels, def_k_chg_activation, NeuronOCL.Activation()) iUpdateModel = (int)MathRound(MathRand() / 32767.0 * (iModels - 1)); setArgument(def_k_HGMultiModels, def_k_chg_model, iUpdateModel) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_HGMultiModels, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiModel trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons() / iModels; global_work_size[1] = NeuronOCL.Neurons() / iModels + 1; uint rest = 0; float lt = lr; setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_w, getWeightsIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_g, getGradientIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_m, getFirstMomentumIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_v, getSecondMomentumIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_UWMultiModels, def_k_uwa_inputs, NeuronOCL.Neurons() / iModels) setArgument(def_k_UWMultiModels, def_k_uwa_l, lt) setArgument(def_k_UWMultiModels, def_k_uwa_b1, b1) setArgument(def_k_UWMultiModels, def_k_uwa_b2, b2) setArgument(def_k_UWMultiModels, def_k_uwa_model, iUpdateModel) global_work_size[1] = (global_work_size[1] + 3) / 4; ResetLastError(); kernelExecute(def_k_UWMultiModels, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights.BufferRead()) ReturnFalse; #endif t++; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiModel state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iModels) <= 0) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiModel state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; iModels = FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Constructor: initializes the CNeuronConcatenate layer defaults. | //+------------------------------------------------------------------+ CNeuronConcatenate::CNeuronConcatenate(void) : i_SecondInputs(0) { ConcWeights = new CBufferFloat(); ConcDeltaWeights = new CBufferFloat(); ConcFirstMomentum = new CBufferFloat(); ConcSecondMomentum = new CBufferFloat; } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronConcatenate GPU resources. | //+------------------------------------------------------------------+ CNeuronConcatenate::~CNeuronConcatenate() { DeleteObj(ConcWeights); DeleteObj(ConcDeltaWeights); DeleteObj(ConcFirstMomentum); DeleteObj(ConcSecondMomentum); } //+------------------------------------------------------------------+ //| Initializes the CNeuronConcatenate layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint numInputs1, uint numInputs2, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; //--- i_SecondInputs = (int)numInputs2; if(!ConcWeights) { ConcWeights = new CBufferFloat(); if(!ConcWeights) ReturnFalse; } int count = (int)((numInputs1 + numInputs2 + 1) * numNeurons); if(!ConcWeights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(numNeurons + 1)); for(int i = 0; i < count; i++) { if(!ConcWeights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!ConcWeights.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(!ConcDeltaWeights) { ConcDeltaWeights = new CBufferFloat(); if(!ConcDeltaWeights) ReturnFalse; } if(!ConcDeltaWeights.BufferInit(count, 0)) ReturnFalse; if(!ConcDeltaWeights.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(ConcFirstMomentum); DeleteObj(ConcSecondMomentum); } else { DeleteObj(ConcDeltaWeights); //--- if(!ConcFirstMomentum) { ConcFirstMomentum = new CBufferFloat(); if(CheckPointer(ConcFirstMomentum) == POINTER_INVALID) ReturnFalse; } if(!ConcFirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!ConcFirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(!ConcSecondMomentum) { ConcSecondMomentum = new CBufferFloat(); if(!ConcSecondMomentum) ReturnFalse; } if(!ConcSecondMomentum.BufferInit(count, 0)) ReturnFalse; if(!ConcSecondMomentum.BufferCreate(OpenCL)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronConcatenate feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_o, Output.GetIndex()) setArgument(def_k_ConcatFeedForward, def_k_cff_inputs1, (int)NeuronOCL.Neurons()) setArgument(def_k_ConcatFeedForward, def_k_cff_inputs2, (int)i_SecondInputs) setArgument(def_k_ConcatFeedForward, def_k_cff_activation, (int)activation) //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); kernelExecute(def_k_ConcatFeedForward, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronConcatenate layer. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!OpenCL || !NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(SecondInput.Total() < i_SecondInputs || SecondGradient.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; if(SecondGradient.GetIndex() < 0 && !SecondGradient.BufferCreate(OpenCL)) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = NeuronOCL.Neurons() + i_SecondInputs; setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_g, Gradient.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig1, NeuronOCL.getGradientIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig2, SecondGradient.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o2, SecondInput.GetIndex()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs2, i_SecondInputs) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_outputs, Neurons()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation1, NeuronOCL.Activation()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation2, (int)SecondActivation) kernelExecute(def_k_ConcatCalcHiddenGradient, global_work_offset, global_work_size) #ifdef _DEBUG if(!ConcWeights.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronConcatenate trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons() + i_SecondInputs + 1; float lt = lr; ResetLastError(); switch(NeuronOCL.Optimization()) { case SGD: setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_g, getGradientIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_dw, ConcDeltaWeights.GetIndex()) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, i_SecondInputs) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_learning_rates, lr) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_momentum, alpha) ResetLastError(); kernelExecute(def_k_ConcatUpdWeightsMomentum, global_work_offset, global_work_size) break; case ADAM: case ADAM_MINI: setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_g, getGradientIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_m, ConcFirstMomentum.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_v, ConcSecondMomentum.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs2, i_SecondInputs) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_l, lt) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b1, b1) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b2, b2) kernelExecute(def_k_ConcatUpdWeightsAdam, global_work_offset, global_work_size) t++; break; default: ReturnFalse; break; } #ifdef _DEBUG if(!ConcWeights.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronConcatenate state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, i_SecondInputs) < INT_VALUE) ReturnFalse; if(!ConcWeights.Save(file_handle)) ReturnFalse; if(optimization == SGD) { if(!ConcDeltaWeights.Save(file_handle)) ReturnFalse; } else { if(!ConcFirstMomentum.Save(file_handle)) ReturnFalse; if(!ConcSecondMomentum.Save(file_handle)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronConcatenate state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- i_SecondInputs = FileReadInteger(file_handle); if(!ConcWeights.Load(file_handle)) ReturnFalse; ConcWeights.BufferCreate(OpenCL); if(optimization == SGD) { if(!ConcDeltaWeights.Load(file_handle)) ReturnFalse; ConcDeltaWeights.BufferCreate(OpenCL); } else { if(!ConcFirstMomentum.Load(file_handle)) ReturnFalse; if(!ConcSecondMomentum.Load(file_handle)) ReturnFalse; ConcFirstMomentum.BufferCreate(OpenCL); ConcSecondMomentum.BufferCreate(OpenCL); } return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronConcatenate GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronConcatenate::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!ConcWeights) return; ConcWeights.BufferCreate(OpenCL); if(optimization == SGD) { if(!ConcDeltaWeights) return; ConcDeltaWeights.BufferCreate(OpenCL); } else { if(!ConcFirstMomentum || !ConcSecondMomentum) return; ConcFirstMomentum.BufferCreate(OpenCL); ConcSecondMomentum.BufferCreate(OpenCL); } } //+------------------------------------------------------------------+ //| Averages the CNeuronConcatenate weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronConcatenate::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronConcatenate *temp = source; if(ConcWeights.Total() != temp.ConcWeights.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {ConcWeights.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, ConcFirstMomentum.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, ConcSecondMomentum.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!ConcFirstMomentum.BufferRead()) ReturnFalse; #endif } else { setBuffer(def_k_SoftUpdate, def_k_su_target, ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, temp.ConcWeights.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) #ifdef _DEBUG if(!ConcWeights.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCSCMOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch ) { const uint layers = windows.Size(); if(layers <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs_count * variables, optimization_type, batch)) ReturnFalse; //--- ia_Windows.Clear(); ia_Windows.Reserve(layers); for(uint i = 0; i < layers; i++) if(!ia_Windows.Add((int)windows[i])) ReturnFalse; i_Variables = variables; i_Count = inputs_count / ia_Windows[0]; b_NeedTranspose = need_transpose; //--- if(b_NeedTranspose) { CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, 0, OpenCL, inputs_count, i_Variables, optimization, iBatch)) { DeleteObj(transp); ReturnFalse; } if(!caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, 1, OpenCL, i_Variables, inputs_count, optimization, iBatch)) { DeleteObj(transp); ReturnFalse; } if(!caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } if(!SetOutput(transp.getOutput()) || !SetGradient(transp.getGradient()) ) ReturnFalse; } //--- uint total = ia_Windows[0] * i_Count; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, 0, OpenCL, inputs_count, inputs_count, total, 1, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SIGMOID); if(!caConvolutions.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- total = 0; for(uint i = 0; i < layers; i++) { conv = new CNeuronConvOCL(); if(!conv.Init(0, i + 1, OpenCL, ia_Windows[i], ia_Windows[i], (i < (layers - 1) ? ia_Windows[i + 1] : 1), i_Count, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SIGMOID); if(!caConvolutions.Add(conv)) { DeleteObj(conv); ReturnFalse; } if(!caConvOutputs.Add(conv.getOutput()) || !caConvGradients.Add(conv.getGradient()) ) ReturnFalse; total += conv.Neurons(); } //--- CNeuronBaseOCL *comul = new CNeuronBaseOCL(); if(!comul.Init(0, 0, OpenCL, total, optimization, iBatch)) { DeleteObj(comul); ReturnFalse; } if(!caMLP.Add(comul)) { DeleteObj(comul); ReturnFalse; } if(layers == 1) { comul.SetOutput(conv.getOutput()); comul.SetGradient(conv.getGradient()); } conv = new CNeuronConvOCL(); if(!conv.Init(0, 0, OpenCL, total / i_Variables, total / i_Variables, inputs_count, 1, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } if(!caMLP.Add(conv)) { DeleteObj(conv); ReturnFalse; } if(!b_NeedTranspose) { if(!SetOutput(conv.getOutput()) || !SetGradient(conv.getGradient()) ) ReturnFalse; } //--- CBufferFloat *buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } //--- caConvOutputs.FreeMode(false); caConvGradients.FreeMode(false); caTranspose.FreeMode(true); caConvolutions.FreeMode(true); caMLP.FreeMode(true); caTemp.FreeMode(true); SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCSCMOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- if(b_NeedTranspose) { current = caTranspose.At(0); if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- int layers = caConvolutions.Total() - 1; for(int l = 0; l <= layers; l++) { current = caConvolutions.At(l); if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- current = caMLP.At(0); if(!current) ReturnFalse; switch(layers) { case 0: ReturnFalse; case 1: break; case 2: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), current.getOutput(), ia_Windows[1], 1, i_Variables * i_Count)) ReturnFalse; break; case 3: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), current.getOutput(), ia_Windows[1], ia_Windows[2], 1, i_Variables * i_Count)) ReturnFalse; break; case 4: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), caConvOutputs.At(3), current.getOutput(), ia_Windows[1], ia_Windows[2], ia_Windows[3], 1, i_Variables * i_Count)) ReturnFalse; break; default: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), caConvOutputs.At(3), caTemp.At(0), ia_Windows[1], ia_Windows[2], ia_Windows[3], ia_Windows[4], i_Variables * i_Count)) ReturnFalse; break; } uint last_buf = 0; for(int i = 4; i < layers; i += 3) { uint buf_size = 0; for(int j = 1; j <= i; j++) buf_size += ia_Windows[j]; switch(layers - i) { case 1: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), current.getOutput(), buf_size, 1, i_Variables * i_Count)) ReturnFalse; break; case 2: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), current.getOutput(), buf_size, ia_Windows[i + 1], 1, i_Variables * i_Count)) ReturnFalse; break; case 3: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), caConvOutputs.At(i + 2), current.getOutput(), buf_size, ia_Windows[i + 1], ia_Windows[i + 2], 1, i_Variables * i_Count)) ReturnFalse; break; default: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), caConvOutputs.At(i + 2), caTemp.At((last_buf + 1) % 2), buf_size, ia_Windows[i + 1], ia_Windows[i + 2], ia_Windows[i + 3], i_Variables * i_Count)) ReturnFalse; break; } last_buf = (last_buf + 1) % 2; } //--- inp = current; current = caMLP.At(1); if(!current || !current.FeedForward(inp)) ReturnFalse; //--- if(b_NeedTranspose) { inp = current; current = caTranspose.At(1); if(!current || !current.FeedForward(inp)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronCSCMOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *current = caMLP.At(0); CNeuronBaseOCL *next = caMLP.At(1); if(b_NeedTranspose) { if(!next.CalcHiddenGradients(caTranspose.At(1))) ReturnFalse; } if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; //--- int layers = caConvGradients.Total(); if(layers == 1) { next = caConvolutions.At(1); if(next.Activation() != None) { if(!DeActivation(next.getOutput(), next.getGradient(), next.getGradient(), next.Activation())) ReturnFalse; } } else { int prev_window = 0; for(int i = 1; i < layers; i++) prev_window += int(ia_Windows[i]); if(!DeConcat(caTemp.At(0), caConvGradients.At(layers - 1), next.getGradient(), prev_window, 1, i_Variables * i_Count)) ReturnFalse; next = caConvolutions.At(layers); int current_buf = 0; for(int l = layers; l > 1; l--) { current = caConvolutions.At(l - 1); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; int window = int(ia_Windows[l - 1]); prev_window -= window; if(!DeConcat(caTemp.At((current_buf + 1) % 2), caTemp.At(2), caTemp.At(current_buf), prev_window, window, i_Variables * i_Count)) ReturnFalse; if(current.Activation() != None) { if(!DeActivation(current.getOutput(), caTemp.At(2), caTemp.At(2), current.Activation())) ReturnFalse; } if(!SumAndNormalize(current.getGradient(), caTemp.At(2), current.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; next = current; current_buf = (current_buf + 1) % 2; } } current = caConvolutions.At(0); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; //--- if(b_NeedTranspose) { current = caTranspose.At(0); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; } //--- if(!prevLayer.CalcHiddenGradients(next.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCSCMOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CObject *prev = (b_NeedTranspose ? caTranspose.At(0) : NeuronOCL); CNeuronBaseOCL *current = NULL; for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(1); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current; } current = caMLP.At(1); if(!current || !current.UpdateInputWeights(caMLP.At(0)) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCSCMOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(i_Variables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(i_Count)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(b_NeedTranspose)) < INT_VALUE) ReturnFalse; //--- Objects if(!ia_Windows.Save(file_handle)) ReturnFalse; if(b_NeedTranspose) { for(int i = 0; i < 2; i++) if(!caTranspose.At(i).Save(file_handle)) ReturnFalse; } int layers = caConvolutions.Total(); if(FileWriteInteger(file_handle, layers) < INT_VALUE) ReturnFalse; for(int i = 0; i < layers; i++) { if(!caConvolutions.At(i).Save(file_handle)) ReturnFalse; } for(int i = 0; i < 2; i++) if(!caMLP.At(i).Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) if(!caTemp.At(i).Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCSCMOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Load(const int file_handle) { CBufferFloat *temp = new CBufferFloat(); if(!temp) ReturnFalse; if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) || !SetOutput(temp, false)) { DeleteObj(temp); ReturnFalse; } temp = new CBufferFloat(); if(!temp) ReturnFalse; if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) || !SetGradient(temp, false)) { DeleteObj(temp); ReturnFalse; } //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; i_Variables = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; i_Count = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; b_NeedTranspose = bool(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; //--- Objects if(!ia_Windows.Load(file_handle)) ReturnFalse; if(b_NeedTranspose) { caTranspose.Clear(); CNeuronTransposeOCL *transp = NULL; for(int i = 0; i < 2; i++) { transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, i, OpenCL, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, transp) || !caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } } } int layers = FileReadInteger(file_handle); caConvolutions.Clear(); caConvOutputs.Clear(); caConvGradients.Clear(); CNeuronConvOCL *conv = NULL; for(int i = 0; i < layers; i++) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, i, OpenCL, 1, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caConvolutions.Add(conv) ) { DeleteObj(conv); ReturnFalse; } if(i > 0) { if(!caConvOutputs.Add(conv.getOutput()) || !caConvGradients.Add(conv.getGradient()) ) ReturnFalse; } } for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, caMLP.At(i))) ReturnFalse; for(int i = 0; i < 3; i++) if(!caTemp.At(i).Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = (b_NeedTranspose ? caTranspose.At(1) : caMLP.At(1)); if(!SetOutput(neuron.getOutput()) || !SetGradient(neuron.getGradient()) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCSCMOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronCSCMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); CNeuronBaseOCL *current = NULL; if(b_NeedTranspose) for(int i = 0; i < 2; i++) { current = caTranspose.At(i); current.SetOpenCL(OpenCL); } //--- for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(i); current.SetOpenCL(OpenCL); } //--- for(int i = 0; i < 2; i++) { current = caMLP.At(i); current.SetOpenCL(OpenCL); } for(int i = 0; i < 3; i++) { CBufferFloat *buf = caTemp.At(i); buf.BufferCreate(OpenCL); } } //+------------------------------------------------------------------+ //| Averages the CNeuronCSCMOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronCSCMOCL *Source = source; if(caConvolutions.Total() != Source.caConvolutions.Total()) ReturnFalse; CNeuronBaseOCL *current = NULL; for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(i); if(!current || !current.WeightsUpdate(Source.caConvolutions.At(i), tau) ) ReturnFalse; } current = caMLP.At(1); if(!current || !current.WeightsUpdate(Source.caMLP.At(1), tau) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDiffusion layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAddNoise.Init(0, 0, OpenCL, window * units_count, iBatch, optimization)) ReturnFalse; cAddNoise.SetActivationFunction(None); if(!cAttention[0].Init(0, 1, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 2, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronDiffusion *temp = new CNeuronDiffusion(); if(!temp) ReturnFalse; if(!temp.Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 3, OpenCL, window, window, window, (units_count + 1) / 2, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 4, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 5, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)cMergeSplit[1].Activation()); //--- if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject())) ReturnFalse; //--- if(!SetOutput(cRevIn.getOutput(), true)) ReturnFalse; //--- cRevIn.SetActivationFunction(None); SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDiffusion feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cAddNoise.FeedForward(NeuronOCL)) ReturnFalse; if(!cAttention[0].FeedForward(cAddNoise.AsObject())) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(cAddNoise.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cRevIn.FeedForward(cResidual.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronDiffusion | //| layer. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- float error = 1; if(!cRevIn.calcOutputGradients(prevLayer.getOutput(), error) || !SumAndNormalize(cRevIn.getGradient(), Gradient, cRevIn.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cResidual.CalcHiddenGradients(cRevIn.AsObject())) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAddNoise.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cAddNoise.getGradient(), cResidual.getGradient(), cAddNoise.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAddNoise.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDiffusion trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cAddNoise.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cAttention[0].UpdateInputWeights(cAddNoise.AsObject())) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDiffusion GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronDiffusion::SetOpenCL(COpenCLMy * obj) { CNeuronUShapeAttention::SetOpenCL(obj); cAddNoise.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cRevIn.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronDiffusion state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Save(const int file_handle) { if(!CNeuronUShapeAttention::Save(file_handle)) ReturnFalse; if(!cAddNoise.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDiffusion state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronUShapeAttention: cNeck = new CNeuronUShapeAttention(); if(!cNeck) ReturnFalse; break; case defNeuronDiffusion: cNeck = new CNeuronDiffusion(); if(!cNeck || !((CNeuronDiffusion*)cNeck).Init(0, 0, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) ReturnFalse; break; case defNeuronConvOCL: cNeck = new CNeuronConvOCL(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(!cAddNoise.Init(0, 0, OpenCL, Neurons(), iBatch, optimization) || !LoadInsideLayer(file_handle, cAddNoise.AsObject())) ReturnFalse; //--- if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject())) ReturnFalse; //--- if(!SetOutput(cRevIn.getOutput(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHyperProjection layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + 1)*units_count, optimization_type, batch)) ReturnFalse; iWindow = window; iUnits = units_count; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHyperProjection feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; uint local_work_size[2] = {1, iWindow}; int kernel = def_k_HyperProjection; ResetLastError(); setBuffer(kernel, def_k_lp_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_lp_outputs, getOutputIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronHyperProjection layer. | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_HyperProjectionGrad; ResetLastError(); setBuffer(kernel, def_k_lpg_inputs, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_lpg_inputs_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_lpg_outputs_gr, getGradientIndex()) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronHyperProjection state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHyperProjection state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronHyperboloids layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * centroids, optimization_type, batch)) ReturnFalse; //--- iWindows = window; iUnits = units_count; iCentroids = centroids; //--- cHyperCentroids.Clear(); cHyperCurvatures.Clear(); cHyperCentroids.SetOpenCL(OpenCL); cHyperCurvatures.SetOpenCL(OpenCL); //--- CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 0, OpenCL, iUnits, iWindows, optimization, iBatch) || !cHyperCentroids.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction(None); //--- CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iUnits, iUnits, iCentroids, iWindows, 1, optimization, iBatch) || !cHyperCentroids.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 2, OpenCL, iCentroids, iCentroids, iCentroids, 1, iWindows, optimization, iBatch) || !cHyperCentroids.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 3, OpenCL, iWindows, iCentroids, optimization, iBatch) || !cHyperCentroids.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 4, OpenCL, iWindows, iWindows, iWindows, iCentroids, 1, optimization, iBatch) || !cHyperCurvatures.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 5, OpenCL, iWindows, iWindows, 1, 1, iCentroids, optimization, iBatch) || !cHyperCurvatures.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- uint size = iCentroids * iUnits * sizeof(float); iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iProducts < 0) ReturnFalse; iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iDistances < 0) ReturnFalse; iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iNormes < 0) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronHyperboloids log map operation. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::LogMap(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs ) { if(!featers || !centroids || !curvatures || !outputs) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iCentroids, iWindows}; uint local_work_size[3] = {1, 1, iWindows}; int kernel = def_k_LogMap; ResetLastError(); setBuffer(kernel, def_k_logmap_centroids, centroids.getOutputIndex()) setBuffer(kernel, def_k_logmap_curvatures, curvatures.getOutputIndex()) setBuffer(kernel, def_k_logmap_features, featers.getOutputIndex()) setBuffer(kernel, def_k_logmap_outputs, outputs.getOutputIndex()) setBuffer(kernel, def_k_logmap_distance, iDistances) setBuffer(kernel, def_k_logmap_product, iProducts) setBuffer(kernel, def_k_logmap_norma, iNormes) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!outputs.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronHyperboloids log map grad operation. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::LogMapGrad(CNeuronBaseOCL* featers, CNeuronBaseOCL* centroids, CNeuronBaseOCL* curvatures, CNeuronBaseOCL* outputs ) { if(!featers || !centroids || !curvatures || !outputs) ReturnFalse; //--- //--- CBufferFloat *temp = featers.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; temp = centroids.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; temp = curvatures.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iCentroids, iWindows}; uint local_work_size[3] = {1, 1, iWindows}; int kernel = def_k_LogMapGrad; ResetLastError(); setBuffer(kernel, def_k_logmapgr_centroids, centroids.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_centroids_gr, centroids.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_curvatures, curvatures.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_curvatures_gr, curvatures.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_features, featers.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_features_gr, featers.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_outputs, outputs.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_outputs_gr, outputs.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_distance, iDistances) setBuffer(kernel, def_k_logmapgr_product, iProducts) setBuffer(kernel, def_k_logmapgr_norma, iNormes) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!featers.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronHyperboloids feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *centroids = NULL; CNeuronBaseOCL *curvatures = NULL; //--- Centroids for(int i = 0; i < cHyperCentroids.Total(); i++) { centroids = cHyperCentroids[i]; if(!centroids || !centroids.FeedForward(prev)) ReturnFalse; prev = centroids; } //--- Curvatures for(int i = 0; i < cHyperCurvatures.Total(); i++) { curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.FeedForward(prev)) ReturnFalse; prev = curvatures; } if(!LogMap(NeuronOCL, centroids, curvatures, AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronHyperboloids layer. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- CObject *next = NULL; CNeuronBaseOCL *centroids = cHyperCentroids[-1]; CNeuronBaseOCL *curvatures = cHyperCurvatures[-1]; //--- if(!LogMapGrad(prevLayer, centroids, curvatures, AsObject())) ReturnFalse; //--- Curvatures for(int i = cHyperCurvatures.Total() - 2; i >= 0; i--) { next = curvatures; curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.CalcHiddenGradients(next)) ReturnFalse; } CBufferFloat *temp = centroids.getGradient(); if(centroids.Activation() != None) if(!DeActivation(centroids.getOutput(), temp, temp, centroids.Activation())) ReturnFalse; if(!centroids.SetGradient(centroids.getPrevOutput(), false) || !centroids.CalcHiddenGradients(curvatures.AsObject()) || !SumAndNormalize(temp, centroids.getGradient(), temp, iWindows, false, 0, 0, 0, 1) || !centroids.SetGradient(temp, false) ) ReturnFalse; //--- Centroids for(int i = cHyperCentroids.Total() - 2; i >= 0; i--) { next = centroids; centroids = cHyperCentroids[i]; if(!centroids || !centroids.CalcHiddenGradients(next)) ReturnFalse; } //--- temp = prevLayer.getGradient(); if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), temp, temp, prevLayer.Activation())) ReturnFalse; if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) || !prevLayer.CalcHiddenGradients(centroids.AsObject()) || !SumAndNormalize(temp, prevLayer.getGradient(), temp, iWindows, false, 0, 0, 0, 1) || !prevLayer.SetGradient(temp, false) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronHyperboloids trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *centroids = NULL; CNeuronBaseOCL *curvatures = NULL; //--- Centroids for(int i = 0; i < cHyperCentroids.Total(); i++) { centroids = cHyperCentroids[i]; if(!centroids || !centroids.UpdateInputWeights(prev)) ReturnFalse; prev = centroids; } //--- Curvatures for(int i = 0; i < cHyperCurvatures.Total(); i++) { curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.UpdateInputWeights(prev)) ReturnFalse; prev = curvatures; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronHyperboloids GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronHyperboloids::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { OpenCL.BufferFree(iProducts); OpenCL.BufferFree(iDistances); OpenCL.BufferFree(iNormes); } //--- CNeuronBaseOCL::SetOpenCL(obj); cHyperCentroids.SetOpenCL(OpenCL); cHyperCurvatures.SetOpenCL(OpenCL); //--- uint size = iCentroids * iUnits * sizeof(float); iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Saves the CNeuronHyperboloids state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cHyperCentroids.Save(file_handle)) ReturnFalse; if(!cHyperCurvatures.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindows)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCentroids)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronHyperboloids state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cHyperCentroids.Load(file_handle)) ReturnFalse; if(!cHyperCurvatures.Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindows = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iCentroids = (int)FileReadInteger(file_handle); SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronHyperboloids layer. | //+------------------------------------------------------------------+ void CNeuronHyperboloids::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); CNeuronBaseOCL *neuron = NULL; //--- for(int i = 0; i < cHyperCentroids.Total(); i++) { neuron = cHyperCentroids[i]; if(!neuron) continue; neuron.TrainMode(bTrain); } //--- for(int i = 0; i < cHyperCurvatures.Total(); i++) { neuron = cHyperCurvatures[i]; if(!neuron) continue; neuron.TrainMode(bTrain); } } //+------------------------------------------------------------------+ //| Initializes the CNeuronHypDiff layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronHypDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); int layer = 0; //--- Projection CNeuronHyperProjection *lorenz = new CNeuronHyperProjection(); if(!lorenz || !lorenz.Init(0, layer, OpenCL, window, units_count, optimization, iBatch) || !cLayers.Add(lorenz)) { DeleteObj(lorenz); ReturnFalse; } layer++; //--- Encoder CNeuronRMAT *rmat = new CNeuronRMAT(); if(!rmat || !rmat.Init(0, layer, OpenCL, window + 1, window_key, units_count, heads, layers, optimization, iBatch) || !cLayers.Add(rmat)) { DeleteObj(rmat); ReturnFalse; } layer++; //--- CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, window + 1, window + 1, 2 * window, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } layer++; conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, 2 * window, 2 * window, 3, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } layer++; //--- LogMap projecction CNeuronHyperboloids *logmap = new CNeuronHyperboloids(); if(!logmap || !logmap.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) || !cLayers.Add(logmap)) { DeleteObj(logmap); ReturnFalse; } layer++; //--- Diffusion model CNeuronDiffusion *diff = new CNeuronDiffusion(); if(!diff || !diff.Init(0, layer, OpenCL, 3, window_key, heads, units_count * centroids, 2, layers, optimization, iBatch) || !cLayers.Add(diff)) { DeleteObj(diff); ReturnFalse; } layer++; //--- Pooling CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling(); if(!pooling || !pooling.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) || !cLayers.Add(pooling)) { DeleteObj(pooling); ReturnFalse; } layer++; //--- Resize to source size conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, 3, 3, window, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- if(!SetOutput(conv.getOutput(), true) || !SetGradient(conv.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPSformer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPSformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint segments, float rho, ENUM_OPTIMIZATION optimization_type, uint batch ) { if((window * units_count) % segments > 0) ReturnFalse; if(!CNeuronBaseSAMOCL::Init(numOutputs, myIndex, open_cl, window * units_count, rho, optimization_type, batch)) ReturnFalse; //--- uint count = Neurons() / segments; //--- if(!acTranspose[0].Init(0, 0, OpenCL, segments, count, optimization, iBatch)) ReturnFalse; acTranspose[0].SetActivationFunction(None); //--- if(!acPSBlocks[0].Init(0, 1, OpenCL, segments, segments, units_count / segments, 1, fRho, optimization, iBatch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!acAttention[i].Init(0, i + 2, OpenCL, segments, segments, units_count / segments, 2, optimization, iBatch)) ReturnFalse; if(!acPSBlocks[i + 1].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject())) ReturnFalse; } //--- if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acAttention[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation()); //--- if(!acTranspose[1].Init(0, 5, OpenCL, count, segments, optimization, iBatch)) ReturnFalse; acTranspose[1].SetActivationFunction((ENUM_ACTIVATION)acPSBlocks[2].Activation()); //--- if(!SetOutput(acTranspose[1].getOutput(), true) || !SetGradient(acTranspose[1].getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPSformer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronPSformer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Dimension Transformation if(!acTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; //--- Segment Attention CObject* prev = acTranspose[0].AsObject(); for(int i = 0; i < 2; i++) { if(!acPSBlocks[i].FeedForward(prev)) ReturnFalse; if(!acAttention[i].FeedForward(acPSBlocks[i].AsObject())) ReturnFalse; prev = acAttention[i].AsObject(); } //--- Residual Add if(!SumAndNormalize(acTranspose[0].getOutput(), acAttention[1].getOutput(), cResidual.getOutput(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- PS Block if(!acPSBlocks[2].FeedForward(cResidual.AsObject())) ReturnFalse; //--- Inverse Transformation if(!acTranspose[1].FeedForward(acPSBlocks[2].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronPSformer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronPSformer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!acPSBlocks[2].CalcHiddenGradients(acTranspose[1].AsObject())) ReturnFalse; //--- if(!cResidual.CalcHiddenGradients(acPSBlocks[2].AsObject())) ReturnFalse; //--- if(!acPSBlocks[1].CalcHiddenGradients(acAttention[1].AsObject())) ReturnFalse; if(!acAttention[0].CalcHiddenGradients(acPSBlocks[1].AsObject())) ReturnFalse; if(!acPSBlocks[0].CalcHiddenGradients(acAttention[0].AsObject())) ReturnFalse; //--- if(!acTranspose[0].CalcHiddenGradients(acPSBlocks[0].AsObject())) ReturnFalse; if(acTranspose[0].Activation() == None) { if(!SumAndNormalize(acTranspose[0].getGradient(), cResidual.getGradient(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(acTranspose[0].getOutput(), cResidual.getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].Activation()) || !SumAndNormalize(acTranspose[0].getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPSformer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPSformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!acPSBlocks[2].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; //--- CObject* prev = acAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { if(!acAttention[i].UpdateInputWeights(acPSBlocks[i].AsObject())) ReturnFalse; if(!acPSBlocks[i].UpdateInputWeights(prev)) ReturnFalse; prev = acTranspose[0].AsObject(); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPSformer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPSformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseSAMOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) { acTranspose[i].SetOpenCL(OpenCL); acPSBlocks[i].SetOpenCL(OpenCL); acAttention[i].SetOpenCL(OpenCL); } acPSBlocks[2].SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronPSformer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPSformer::Save(const int file_handle) { if(!CNeuronBaseSAMOCL::Save(file_handle)) ReturnFalse; if(!acPSBlocks[0].Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!acTranspose[i].Save(file_handle) || !acAttention[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPSformer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPSformer::Load(const int file_handle) { if(!CNeuronBaseSAMOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, acPSBlocks[0].AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject()) || !LoadInsideLayer(file_handle, acAttention[i].AsObject())) ReturnFalse; //--- for(int i = 1; i < 3; i++) if(!acPSBlocks[i].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject())) ReturnFalse; //--- if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acAttention[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation()); //--- if(!SetOutput(acTranspose[1].getOutput(), true) || !SetGradient(acTranspose[1].getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAdaBN layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAdaBN::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint & bottlenecks[], uint top_k, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; int index = 0; if(!cGates.Init(0, index, OpenCL, window, units_count * variables, bottlenecks.Size(), top_k, optimization, iBatch)) ReturnFalse; //--- index++; if(!cXProj.Init(0, index, OpenCL, window, window, window, units_count, optimization, iBatch)) ReturnFalse; cXProj.SetActivationFunction(None); //--- cExperts.Clear(); cExperts.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronMultiWindowsConvOCL *mwconv = NULL; CNeuronTransposeRCDOCL *transp = NULL; //--- uint bn_size = 0; for(uint i = 0; i < bottlenecks.Size(); i++) bn_size += bottlenecks[i]; //--- index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, bn_size, units_count, variables, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(LReLU); //--- index++; mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv || !mwconv.Init(0, index, OpenCL, bottlenecks, window_out, units_count, variables, optimization, iBatch) || !cExperts.Add(mwconv)) { DeleteObj(conv); ReturnFalse; } mwconv.SetActivationFunction(LReLU); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, units_count * variables, bottlenecks.Size(), window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_out, window_out, window, units_count * variables, bottlenecks.Size(), optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, bottlenecks.Size(), units_count * variables, window, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTempEmbedding layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint embed_dim1, uint period1, uint frame1, uint embed_dim2, uint period2, uint frame2, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + embed_dim1 + embed_dim2)*units, optimization_type, batch)) ReturnFalse; //--- if(!caEmbeddings[0].Init(0, 0, OpenCL, embed_dim1 * period1, optimization, iBatch)) ReturnFalse; if(!caEmbeddings[1].Init(0, 1, OpenCL, embed_dim2 * period2, optimization, iBatch)) ReturnFalse; //--- iWindow = window; iUnits = units; aiEmbeddingDim[0] = embed_dim1; aiEmbeddingDim[1] = embed_dim2; aiFrames[0] = MathMax(1, frame1); aiFrames[1] = MathMax(1, frame2); aiPeriod[0] = period1; aiPeriod[1] = period2; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronTempEmbedding concat by label operation. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < int(iUnits * iWindow) || SecondInput.Total() < (int)iUnits) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]), 3 }; uint kernel = def_k_ConcatByLabel; setBuffer(kernel, def_k_cbl_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex()) setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getOutputIndex()) setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getOutputIndex()) setBuffer(kernel, def_k_cbl_output, getOutputIndex()) setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow) setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0]) setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1]) setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0]) setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1]) setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0]) setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1]) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronTempEmbedding concat by label grad operation. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < int(iUnits * iWindow) || SecondInput.Total() < (int)iUnits) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {MathMax(iUnits, aiPeriod[ArrayMaximum(aiPeriod)]), MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]), 3 }; uint kernel = def_k_ConcatByLabelGrad; setBuffer(kernel, def_k_cbl_data, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex()) setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getGradientIndex()) setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getGradientIndex()) setBuffer(kernel, def_k_cbl_output, getGradientIndex()) setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow) setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0]) setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1]) setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0]) setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1]) setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0]) setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1]) setArgument(kernel, def_k_cbl_gr_units, (int)iUnits) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTempEmbedding feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(bTrain) for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].FeedForward()) ReturnFalse; //--- return(ConcatByLabel(NeuronOCL, SecondInput)); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTempEmbedding layer. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!ConcatByLabelGrad(NeuronOCL, SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTempEmbedding trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].UpdateInputWeights()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronTempEmbedding weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTempEmbedding* Source = source; for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTempEmbedding state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; for(uint i = 0; i < caEmbeddings.Size(); i++) { if(FileWriteInteger(file_handle, (int)aiEmbeddingDim[i]) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiFrames[i]) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiPeriod[i]) < INT_VALUE) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTempEmbedding state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); for(uint i = 0; i < caEmbeddings.Size(); i++) { if(FileIsEnding(file_handle)) ReturnFalse; aiEmbeddingDim[i] = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; aiFrames[i] = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; aiPeriod[i] = (uint)FileReadInteger(file_handle); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTempEmbedding GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTempEmbedding::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- for(uint i = 0; i < caEmbeddings.Size(); i++) caEmbeddings[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiScaleExcitation layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint scales, uint bottleneck, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units * scales, optimization_type, batch)) ReturnFalse; //--- cExcitation.Clear(); cExcitation.SetOpenCL(OpenCL); CNeuronSpikeConvBlock* conv = NULL; CNeuronTransposeOCL* transp = NULL; CNeuronSoftMaxOCL* softmax = NULL; uint index = 0; //--- [Scale, Unit, Dimension] transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, scales, units * dimension, optimization, iBatch) || !cExcitation.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; //--- [Unit, Dimension, Scale] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, scales, scales, 1, dimension * units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Dimension] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension, dimension, bottleneck, units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Bottleneck] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, scales * dimension, units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Dimension, Scale] softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) || !cExcitation.Add(softmax)) { DeleteObj(softmax) ReturnFalse; } softmax.SetHeads(dimension * units); index++; //--- [Unit, Dimension, Scale] transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units * dimension, scales, optimization, iBatch) || !cExcitation.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; //--- [Scale, Unit, Dimension] return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiScaleExcitation feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!ElementMult(NeuronOCL.getOutput(), prev.getOutput(), Output)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiScaleExcitation layer. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* next = cExcitation[-1]; CNeuronBaseOCL* curr = NULL; //--- if(!NeuronOCL || !next) ReturnFalse; if(!ElementMultGrad(NeuronOCL.getOutput(), PrevOutput, next.getOutput(), next.getGradient(), Gradient, NeuronOCL.Activation(), next.Activation())) ReturnFalse; //--- for(int i = cExcitation.Total() - 2; i >= 0; i--) { curr = cExcitation[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiScaleExcitation trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMultiScaleExcitation weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMultiScaleExcitation* Source = source; if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiScaleExcitation state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cExcitation.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiScaleExcitation state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cExcitation.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronMultiScaleExcitation execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cExcitation.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultiScaleExcitation GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultiScaleExcitation::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cExcitation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMultiScaleExcitation layer. | //+------------------------------------------------------------------+ void CNeuronMultiScaleExcitation::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cExcitation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Sets the sample of the CNeuronBayesianSampler. | //+------------------------------------------------------------------+ bool CNeuronBayesianSampler::SetSample(const vector &sample) { if(!m_cRandom || m_cRandom.Total() != sample.Size() || !m_cRandom.AssignArray(sample) || !m_cRandom.BufferWrite()) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Clears one retained Bayesian draw without changing its contract. | //+------------------------------------------------------------------+ bool CNeuronBayesianSampler::Clear(void) { //--- Clear inherited output and gradient buffers. if(!CNeuronBaseOCL::Clear()) ReturnFalseEx("sampler output"); //--- The local reparameterization noise is execution-only state. if(!m_cRandom || !m_cRandom.Fill(0.0f)) ReturnFalseEx("sampler random"); //--- Finalize the sampler scratch reset. return(true); } //+------------------------------------------------------------------+ //| Feed the existing VAE kernel from a caller-owned local draw. | //+------------------------------------------------------------------+ bool CNeuronBayesianSampler::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || !m_cRandom || NeuronOCL.Neurons() % 2 != 0 || NeuronOCL.Neurons() / 2 != Neurons()) ReturnFalse; return(VAEFeedForward(NeuronOCL, NULL)); } //+------------------------------------------------------------------+ //| Runs the existing Bayesian local draw with numeric checking. | //+------------------------------------------------------------------+ bool CNeuronBayesianSampler::FeedForwardChecked(CNeuronBaseOCL *NeuronOCL, CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar, bool &rejected) { rejected = false; //--- Preserve the externally prepared local draw without regeneration. if(!ValidateCheckedScratch(NeuronOCL, numeric_flags, numeric_ones, numeric_scalar)) return(false); //--- Dispatch the caller-supplied draw and reduce its complete output prefix. if(!VAEFeedForward(NeuronOCL, numeric_flags)) return(false); //--- Finalize the checked Bayesian sampler dispatch. return(ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, (uint)Neurons(), rejected)); } //+------------------------------------------------------------------+ //| Constructor: initializes the CNeuronBayesianLinearOCL layer | //| defaults. | //+------------------------------------------------------------------+ CNeuronBayesianLinearOCL::CNeuronBayesianLinearOCL(void) : iVariables(0), iInputs(0), iOutputs(0), iSamples(0), iSampleState(1), m_fKLDMult(0.01f), bRegularizersEnabled(true), bSamplesGradientReady(false), bSamplesPrepared(false) { cRandomSamplers.FreeMode(true); } //+------------------------------------------------------------------+ //| Release the runtime-only posterior samplers | //| (CNeuronBayesianLinearOCL). | //+------------------------------------------------------------------+ CNeuronBayesianLinearOCL::~CNeuronBayesianLinearOCL(void) { cRandomSamplers.Clear(); } //+------------------------------------------------------------------+ //| Sets the kld mult of the CNeuronBayesianLinearOCL. | //+------------------------------------------------------------------+ void CNeuronBayesianLinearOCL::SetKLDMult(float value) { if(!MathIsValidNumber(value) || value < 0.0f) return; m_fKLDMult = value; cMeanSampler.SetKLDMult(bRegularizersEnabled ? m_fKLDMult : 0.0f); } //+------------------------------------------------------------------+ //| Enables or suppresses the runtime posterior regularizers. | //+------------------------------------------------------------------+ void CNeuronBayesianLinearOCL::SetRegularizersEnabled(bool value) { if(bRegularizersEnabled == value) return; //--- Keep the configured multiplier while changing only its effective runtime value. bRegularizersEnabled = value; cMeanSampler.SetKLDMult(bRegularizersEnabled ? m_fKLDMult : 0.0f); bSamplesGradientReady = false; bSamplesPrepared = false; } //+------------------------------------------------------------------+ //| Handles the CNeuronBayesianLinearOCL create temporary operation. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::CreateTemporary(void) { //--- Allocate the deterministic posterior and posterior-sample work layers. uint w = iVariables * iInputs * iOutputs; uint b = iVariables * iOutputs; if(!OpenCL) ReturnFalseEx("Bayesian temporary: OpenCL context"); if(!iSamples) ReturnFalseEx("Bayesian temporary: sample count"); if(!cPosterior.Init(0, 0, OpenCL, 2 * (w + b), optimization, iBatch)) ReturnFalseEx("Bayesian temporary: posterior"); if(!cSampleWeights.Init(0, 0, OpenCL, w, optimization, iBatch)) ReturnFalseEx("Bayesian temporary: sampled weights"); if(!cSampleBias.Init(0, 0, OpenCL, b, optimization, iBatch)) ReturnFalseEx("Bayesian temporary: sampled bias"); if(!cSampleOutput.Init(0, 0, OpenCL, b, optimization, iBatch)) ReturnFalseEx("Bayesian temporary: sampled output"); if(!cInputGradient.Init(0, 0, OpenCL, iVariables * iInputs, optimization, iBatch)) ReturnFalseEx("Bayesian temporary: input gradient"); if(!cPosteriorGradient.Init(0, 0, OpenCL, 2 * (w + b), optimization, iBatch)) ReturnFalseEx("Bayesian temporary: posterior gradient"); if(!cMeanSampler.Init(0, 0, OpenCL, w + b, optimization, iBatch)) ReturnFalseEx("Bayesian temporary: mean sampler"); //--- The production forward path always evaluates the posterior mean. cMeanSampler.SetSampleMode(VAE_SAMPLE_MEAN); cMeanSampler.SetKLDMult(bRegularizersEnabled ? m_fKLDMult : 0.0f); cRandomSamplers.Clear(); //--- Create the stochastic samplers retained exclusively for regularization. for(uint i = 0; i < iSamples; i++) { CNeuronBayesianSampler *s = new CNeuronBayesianSampler(); if(!s) ReturnFalseEx("Bayesian temporary: random sampler allocation"); if(!s.Init(0, 0, OpenCL, w + b, optimization, iBatch)) { delete s; ReturnFalseEx("Bayesian temporary: random sampler initialization"); } if(!cRandomSamplers.Add(s)) { delete s; ReturnFalseEx("Bayesian temporary: random sampler registry"); } s.SetKLDMult(0.0f); } //--- Finalize the complete temporary state. return(true); } //+------------------------------------------------------------------+ //| Clears Bayesian execution scratch and retained stochastic draws. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::Clear(void) { //--- No stale sample or gradient cache may survive the public Clear boundary. bSamplesGradientReady = false; bSamplesPrepared = false; if(!CNeuronBaseOCL::Clear()) ReturnFalseEx("output"); //--- Keep cached posterior Outputs for inference while clearing only gradients. CBufferFloat *weights_gradient = cWeights.getGradient(); if(!weights_gradient || !weights_gradient.Fill(0.0f)) ReturnFalseEx("weight gradient"); CBufferFloat *bias_gradient = cBias.getGradient(); if(!bias_gradient || !bias_gradient.Fill(0.0f)) ReturnFalseEx("bias gradient"); if(!cPosterior.Clear()) ReturnFalseEx("posterior"); if(!cSampleWeights.Clear()) ReturnFalseEx("sample weights"); if(!cSampleBias.Clear()) ReturnFalseEx("sample bias"); if(!cSampleOutput.Clear()) ReturnFalseEx("sample output"); if(!cInputGradient.Clear()) ReturnFalseEx("input gradient"); if(!cPosteriorGradient.Clear()) ReturnFalseEx("posterior gradient"); if(!cMeanSampler.Clear()) ReturnFalseEx("mean sampler"); //--- Random sampler overrides also clear their owned reparameterization noise. for(int i = 0; i < cRandomSamplers.Total(); i++) { CVAE *sampler = cRandomSamplers.At(i); if(!sampler || !sampler.Clear()) ReturnFalseEx("random sampler"); } //--- Batch-local buffers are optional and remain allocated for reuse. if(cBatchSampleGradient.Total() > 0 && !cBatchSampleGradient.Fill(0.0f)) ReturnFalseEx("batch sample gradient"); if(cBatchInputGradient.Total() > 0 && !cBatchInputGradient.Fill(0.0f)) ReturnFalseEx("batch input gradient"); if(cBatchInputGradientAccum.Total() > 0 && !cBatchInputGradientAccum.Fill(0.0f)) ReturnFalseEx("batch input accumulation"); //--- Finalize the retained Bayesian execution-state reset. return(true); } //+------------------------------------------------------------------+ //| Create GPU-only accumulation buffers for an internal history. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::CreateBatchTemporary(uint rows_per_group) { if(!OpenCL || rows_per_group == 0) ReturnFalseEx("Bayesian batch temporary: arguments"); //--- Allocate GPU-only buffers used to accumulate the sample gradients. uint sample_total = iVariables * rows_per_group * iOutputs; uint input_total = iVariables * rows_per_group * iInputs; if(!cBatchSampleGradient.BufferInit(sample_total, 0.0f)) ReturnFalseEx("Bayesian batch temporary: sample gradient host buffer"); if(!cBatchSampleGradient.BufferCreate(OpenCL)) ReturnFalseEx("Bayesian batch temporary: sample gradient GPU buffer"); if(!cBatchInputGradient.BufferInit(input_total, 0.0f)) ReturnFalseEx("Bayesian batch temporary: input gradient host buffer"); if(!cBatchInputGradient.BufferCreate(OpenCL)) ReturnFalseEx("Bayesian batch temporary: input gradient GPU buffer"); if(!cBatchInputGradientAccum.BufferInit(input_total, 0.0f)) ReturnFalseEx("Bayesian batch temporary: accumulated gradient host buffer"); if(!cBatchInputGradientAccum.BufferCreate(OpenCL)) ReturnFalseEx("Bayesian batch temporary: accumulated gradient GPU buffer"); //--- Finalize the batch-local accumulation state. return(true); } //+------------------------------------------------------------------+ //| Assemble the posterior parameter tensor (BuildPosterior). | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::BuildPosterior(void) { //--- Refresh the trainable posterior parameters before concatenation. if(bTrain && !cWeights.FeedForward()) ReturnFalseEx("Bayesian posterior: weights forward"); if(bTrain && !cBias.FeedForward()) ReturnFalseEx("Bayesian posterior: bias forward"); //--- Assemble [mu | logvar] for weights and bias. if(!Concat(cWeights.getOutput(), cBias.getOutput(), cPosterior.getOutput(), iVariables * iInputs * iOutputs, iVariables * iOutputs, 2)) ReturnFalseEx("Bayesian posterior: parameter concatenation"); //--- Finalize the assembled posterior tensor. return(true); } //+------------------------------------------------------------------+ //| Assemble posterior parameters and reduce each checked producer. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::BuildPosteriorChecked(CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar, bool &rejected) { rejected = false; //--- Preserve cached inference parameters while checking each raw posterior slice. if(!bTrain) { const uint weights = iVariables * iInputs * iOutputs; const uint bias = iVariables * iOutputs; CBufferFloat *cached_weights = cWeights.getOutput(); CBufferFloat *cached_bias = cBias.getOutput(); CBufferFloat *posterior = cPosterior.getOutput(); if(!cached_weights || !cached_bias || !posterior) return(false); if(!MatMul(numeric_ones, cached_weights, posterior, 1, 1, weights, 1, false, 0, 0, 0, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, weights, rejected)) return(false); if(rejected) return(true); if(!MatMul(numeric_ones, cached_bias, posterior, 1, 1, bias, 1, false, 0, 0, weights, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, bias, rejected)) return(false); if(rejected) return(true); if(!MatMul(numeric_ones, cached_weights, posterior, 1, 1, weights, 1, false, 0, weights, weights + bias, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, weights, rejected)) return(false); if(rejected) return(true); if(!MatMul(numeric_ones, cached_bias, posterior, 1, 1, bias, 1, false, 0, bias, 2 * weights + bias, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, bias, rejected)) return(false); return(true); } //--- Refresh both trainable parameter groups before concatenation. if(!cWeights.FeedForwardChecked(numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) return(true); if(!cBias.FeedForwardChecked(numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) return(true); //--- Concatenate the existing posterior layout without changing its ABI. if(!Concat(cWeights.getOutput(), cBias.getOutput(), cPosterior.getOutput(), iVariables * iInputs * iOutputs, iVariables * iOutputs, 2)) return(false); //--- Finalize the checked posterior assembly. return(true); } //+------------------------------------------------------------------+ //| Validates caller scratch before any Bayesian checked dispatch. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::ValidateCheckedScratch(CBufferFloat *source_buffer, CBufferFloat *output, uint required_count, CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar) { //--- Validate external operands and the complete caller-owned scratch contract. if(!OpenCL || !source_buffer || !output || !numeric_flags || !numeric_ones || !numeric_scalar || required_count == 0 || required_count > INT_MAX || source_buffer == output || source_buffer.GetOpenCL() != OpenCL || output.GetOpenCL() != OpenCL || source_buffer.GetIndex() < 0 || output.GetIndex() < 0 || source_buffer.GetIndex() == output.GetIndex() || numeric_flags.GetOpenCL() != OpenCL || numeric_ones.GetOpenCL() != OpenCL || numeric_scalar.GetOpenCL() != OpenCL || numeric_flags.GetIndex() < 0 || numeric_ones.GetIndex() < 0 || numeric_scalar.GetIndex() < 0 || numeric_flags.GetIndex() == numeric_ones.GetIndex() || numeric_flags.GetIndex() == numeric_scalar.GetIndex() || numeric_ones.GetIndex() == numeric_scalar.GetIndex() || numeric_flags.Total() < int(required_count) || numeric_ones.Total() < int(required_count) || numeric_scalar.Total() != 1) return(false); //--- Reject aliases with every operand that the checked forward can read or overwrite. int blocked[] = {source_buffer.GetIndex(), output.GetIndex(), cWeights.getOutput().GetIndex(), cWeights.getWeightsParams().GetIndex(), cBias.getOutput().GetIndex(), cBias.getWeightsParams().GetIndex(), cPosterior.getOutput().GetIndex(), cSampleWeights.getOutput().GetIndex(), cSampleBias.getOutput().GetIndex(), cSampleOutput.getOutput().GetIndex(), cMeanSampler.getOutput().GetIndex() }; for(int i = 0; i < ArraySize(blocked); i++) if(blocked[i] < 0 || numeric_flags.GetIndex() == blocked[i] || numeric_ones.GetIndex() == blocked[i] || numeric_scalar.GetIndex() == blocked[i]) return(false); //--- Reject aliases with every retained stochastic sampler output before mutation. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); CBufferFloat *sample_output = (sample ? sample.getOutput() : NULL); if(!sample_output || sample_output.GetOpenCL() != OpenCL || sample_output.GetIndex() < 0 || numeric_flags.GetIndex() == sample_output.GetIndex() || numeric_ones.GetIndex() == sample_output.GetIndex() || numeric_scalar.GetIndex() == sample_output.GetIndex()) return(false); } //--- Finalize the pre-dispatch alias validation. return(true); } //+------------------------------------------------------------------+ //| Advance the deterministic sampler noise generator. | //+------------------------------------------------------------------+ uint CNeuronBayesianLinearOCL::NextSampleRandom(void) { iSampleState ^= iSampleState << 13; iSampleState ^= iSampleState >> 17; iSampleState ^= iSampleState << 5; return(iSampleState); } //+------------------------------------------------------------------+ //| Set reparameterization noise for one posterior sampler. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::SetSample(CVAE *sampler) { if(!sampler) ReturnFalseEx("Bayesian sample: sampler"); //--- Fill the sampler's reparameterization noise with deterministic pseudo-random normals. vector sample = vector::Zeros(iVariables * (iInputs * iOutputs + iOutputs)); for(uint i = 0; i < sample.Size(); i += 2) { double u1 = (double(NextSampleRandom()) + 1.0) / 4294967297.0; double u2 = (double(NextSampleRandom()) + 1.0) / 4294967297.0; double radius = MathSqrt(-2.0 * MathLog(u1)); sample[i] = (float)(radius * MathCos(2.0 * M_PI * u2)); if(i + 1 < sample.Size()) sample[i + 1] = (float)(radius * MathSin(2.0 * M_PI * u2)); } CNeuronBayesianSampler *random_sampler = sampler; if(!random_sampler.SetSample(sample)) ReturnFalseEx("Bayesian sample: reparameterization noise"); //--- The sampler holds the noise for the next posterior draw. return(true); } //+------------------------------------------------------------------+ //| Apply one grouped posterior affine transformation. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::Affine(CNeuronBaseOCL *n, CVAE *s, CNeuronBaseOCL *r) { if(!n || !s || !r) ReturnFalseEx("Bayesian affine: layer arguments"); //--- Split the sampled posterior into grouped affine weights and bias. uint w = iVariables * iInputs * iOutputs; uint b = iVariables * iOutputs; if(!DeConcat(cSampleWeights.getOutput(), cSampleBias.getOutput(), s.getOutput(), w, b, 1)) ReturnFalseEx("Bayesian affine: sampled posterior split"); if(!MatMul(n.getOutput(), cSampleWeights.getOutput(), r.getOutput(), 1, iInputs, iOutputs, iVariables, true)) ReturnFalseEx("Bayesian affine: grouped matrix multiplication"); if(!SumVecMatrix(cSampleBias.getOutput(), r.getOutput(), r.getOutput(), iOutputs, iVariables)) ReturnFalseEx("Bayesian affine: bias addition"); //--- The sampled affine output is ready for the caller. return(true); } //+------------------------------------------------------------------+ //| Apply one checked grouped posterior affine transformation. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::AffineChecked(CNeuronBaseOCL *n, CVAE *s, CNeuronBaseOCL *r, CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar, bool &rejected) { rejected = false; if(!n || !s || !r) return(false); //--- Read the raw sampled posterior weights directly from its prefix. uint w = iVariables * iInputs * iOutputs; if(!MatMul(n.getOutput(), s.getOutput(), r.getOutput(), 1, iInputs, iOutputs, iVariables, true, 0, 0, 0, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, (uint)r.Neurons(), rejected)) return(false); if(rejected) return(true); if(!SumVecMatrix(s.getOutput(), r.getOutput(), r.getOutput(), iOutputs, iVariables, w, 0, 0, 1.0f, 0, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, (uint)r.Neurons(), rejected)) return(false); //--- Finalize the checked affine transformation. return(true); } //+------------------------------------------------------------------+ //| Initialize grouped posterior parameters and runtime samplers. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::Init(uint no, uint mi, COpenCLMy *oc, uint v, uint in, uint out, uint k, ENUM_OPTIMIZATION opt, uint batch) { if(!v || !in || !out || !k) ReturnFalseEx("Bayesian init: dimensions"); if(!CNeuronBaseOCL::Init(no, mi, oc, v * out, opt, batch)) ReturnFalseEx("Bayesian init: base layer"); //--- Store the grouped affine contract before allocating posterior parameters. iVariables = v; iInputs = in; iOutputs = out; iSamples = k; iSampleState = 1; bRegularizersEnabled = true; bSamplesGradientReady = false; bSamplesPrepared = false; activation = None; uint w = v * in * out; uint b = v * out; if(!cWeights.Init(0, 0, OpenCL, 2 * w, opt, iBatch)) ReturnFalseEx("Bayesian init: posterior weights"); if(!cBias.Init(0, 0, OpenCL, 2 * b, opt, iBatch)) ReturnFalseEx("Bayesian init: posterior bias"); if(!cWeights.getWeightsParams().Fill(0.0f)) ReturnFalseEx("Bayesian init: posterior weight initialization"); if(!cBias.getWeightsParams().Fill(0.0f)) ReturnFalseEx("Bayesian init: posterior bias initialization"); if(!cWeights.FeedForward()) ReturnFalseEx("Bayesian init: posterior weights forward"); if(!cBias.FeedForward()) ReturnFalseEx("Bayesian init: posterior bias forward"); if(!CreateTemporary()) ReturnFalseEx("Bayesian init: temporary state"); //--- Finalize the initialized posterior layer. return(true); } //+------------------------------------------------------------------+ //| Generate deterministic output and optional posterior samples. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::feedForward(CNeuronBaseOCL *n) { if(!n || n.Neurons() != int(iVariables * iInputs)) ReturnFalseEx("Bayesian forward: input shape"); if(!BuildPosterior()) ReturnFalseEx("Bayesian forward: posterior assembly"); //--- Every forward invalidates prior stochastic samples and their gradients. bSamplesGradientReady = false; bSamplesPrepared = false; //--- Generate the deterministic production output from the posterior mean. CVAE *mean = GetPointer(cMeanSampler); if(!mean) ReturnFalseEx("Bayesian forward: mean sampler"); if(!mean.FeedForward(cPosterior.AsObject())) ReturnFalseEx("Bayesian forward: mean sample"); if(!Affine(n, mean, GetPointer(this))) ReturnFalseEx("Bayesian forward: mean affine"); if(!bTrain || !bRegularizersEnabled) return(true); //--- Stochastic states are retained exclusively for regularizers. //--- The CNet output remains the posterior mean. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); if(!sample) ReturnFalseEx("Bayesian forward: random sampler"); if(!SetSample(sample)) ReturnFalseEx("Bayesian forward: random sample"); if(!sample.FeedForward(cPosterior.AsObject())) ReturnFalseEx("Bayesian forward: random sampler forward"); } //--- Mark the regularizer samples valid for the next backward pass. bSamplesPrepared = true; return(true); } //+------------------------------------------------------------------+ //| Generates checked mean output and retained posterior samples. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::FeedForwardChecked(CNeuronBaseOCL *n, CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar, bool &rejected) { rejected = false; if(!n || !n.getOutput()) return(false); //--- Derive bounded checked spans before any sampling or kernel dispatch. const ulong source_total = ulong(iVariables) * iInputs; const ulong weight_total = ulong(iVariables) * iInputs * iOutputs; const ulong bias_total = ulong(iVariables) * iOutputs; const ulong posterior_total = 2 * (weight_total + bias_total); const ulong output_total = ulong(iVariables) * iOutputs; const ulong scratch_total = (posterior_total > output_total ? posterior_total : output_total); if(!n || !numeric_flags || !numeric_ones || !numeric_scalar || iVariables == 0 || iInputs == 0 || iOutputs == 0 || iSamples == 0 || weight_total > INT_MAX || bias_total > INT_MAX || posterior_total > INT_MAX || output_total > INT_MAX || scratch_total > INT_MAX || source_total > INT_MAX || n.Neurons() != int(source_total) || n.getOutput().GetOpenCL() != OpenCL || n.getOutput().GetIndex() < 0 || Output.GetIndex() < 0 || numeric_flags.GetOpenCL() != OpenCL || numeric_ones.GetOpenCL() != OpenCL || numeric_scalar.GetOpenCL() != OpenCL || numeric_flags.GetIndex() < 0 || numeric_ones.GetIndex() < 0 || numeric_scalar.GetIndex() < 0 || numeric_flags.GetIndex() == numeric_ones.GetIndex() || numeric_flags.GetIndex() == numeric_scalar.GetIndex() || numeric_ones.GetIndex() == numeric_scalar.GetIndex() || numeric_flags.Total() < int(scratch_total) || numeric_ones.Total() < int(scratch_total) || numeric_scalar.Total() != 1 || numeric_flags.GetIndex() == n.getOutput().GetIndex() || numeric_flags.GetIndex() == Output.GetIndex() || numeric_ones.GetIndex() == n.getOutput().GetIndex() || numeric_ones.GetIndex() == Output.GetIndex() || numeric_scalar.GetIndex() == n.getOutput().GetIndex() || numeric_scalar.GetIndex() == Output.GetIndex() || numeric_ones.GetIndex() == cWeights.getOutput().GetIndex() || numeric_ones.GetIndex() == cBias.getOutput().GetIndex() || numeric_ones.GetIndex() == cPosterior.getOutput().GetIndex() || numeric_ones.GetIndex() == cSampleWeights.getOutput().GetIndex() || numeric_ones.GetIndex() == cSampleBias.getOutput().GetIndex() || numeric_scalar.GetIndex() == cWeights.getOutput().GetIndex() || numeric_scalar.GetIndex() == cBias.getOutput().GetIndex() || numeric_scalar.GetIndex() == cPosterior.getOutput().GetIndex() || numeric_scalar.GetIndex() == cSampleWeights.getOutput().GetIndex() || numeric_scalar.GetIndex() == cSampleBias.getOutput().GetIndex()) return(false); if(!ValidateCheckedScratch(n.getOutput(), Output, (uint)scratch_total, numeric_flags, numeric_ones, numeric_scalar)) return(false); //--- Invalidate retained stochastic state until the complete checked forward succeeds. bSamplesGradientReady = false; bSamplesPrepared = false; if(!BuildPosteriorChecked(numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) return(true); //--- Evaluate the deterministic mean through the checked VAE and affine paths. CVAE *mean = GetPointer(cMeanSampler); if(!mean || !mean.FeedForwardChecked(cPosterior.AsObject(), numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) return(true); if(!AffineChecked(n, mean, GetPointer(this), numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected || !bTrain || !bRegularizersEnabled) return(true); //--- Prepare each stochastic draw once and retain it for batch reference use. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); if(!sample) return(false); if(!SetSample(sample)) return(false); if(!sample.FeedForwardChecked(cPosterior.AsObject(), numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) return(true); } //--- Mark only fully checked stochastic samples prepared for later regularizers. bSamplesPrepared = true; return(true); } //+------------------------------------------------------------------+ //| Backpropagate the deterministic posterior-mean path. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::calcInputGradients(CNeuronBaseOCL *n) { if(!bTrain) return(true); //--- Accumulate the mean affine path into the posterior gradient. uint w = iVariables * iInputs * iOutputs; uint b = iVariables * iOutputs; uint p = w + b; if(!n) ReturnFalseEx("Bayesian backward: input layer"); if(!n.getGradient().Fill(0.0f)) ReturnFalseEx("Bayesian backward: input gradient reset"); if(!bSamplesGradientReady && !cPosteriorGradient.getGradient().Fill(0.0f)) ReturnFalseEx("Bayesian backward: posterior gradient reset"); CVAE *mean = GetPointer(cMeanSampler); if(!mean) ReturnFalseEx("Bayesian backward: mean sampler"); if(!DeConcat(cSampleWeights.getOutput(), cSampleBias.getOutput(), mean.getOutput(), w, b, 1)) ReturnFalseEx("Bayesian backward: mean posterior split"); if(!CopyBufferRaw(Gradient, cSampleOutput.getGradient(), b)) ReturnFalseEx("output gradient device copy"); if(!SumVecMatrixGrad(cSampleBias.getGradient(), cSampleOutput.getGradient(), cSampleOutput.getGradient(), iOutputs, iVariables)) ReturnFalseEx("Bayesian backward: bias gradient"); if(!MatMulGrad(n.getOutput(), cInputGradient.getGradient(), cSampleWeights.getOutput(), cSampleWeights.getGradient(), cSampleOutput.getGradient(), 1, iInputs, iOutputs, iVariables, true)) ReturnFalseEx("Bayesian backward: grouped affine gradient"); if(!SumAndNormalize(cInputGradient.getGradient(), n.getGradient(), n.getGradient(), iInputs, false, 0, 0, 0, 1.0f)) ReturnFalseEx("Bayesian backward: input gradient accumulation"); if(!Concat(cSampleWeights.getGradient(), cSampleBias.getGradient(), mean.getGradient(), w, b, 1)) ReturnFalseEx("Bayesian backward: mean gradient assembly"); if(!cPosterior.CalcHiddenGradients(mean)) ReturnFalseEx("Bayesian backward: mean sampler gradient"); if(!SumAndNormalize(cPosterior.getGradient(), cPosteriorGradient.getGradient(), cPosteriorGradient.getGradient(), p, false, 0, 0, 0, 1.0f)) ReturnFalseEx("Bayesian backward: posterior gradient accumulation"); if(!DeConcat(cWeights.getGradient(), cBias.getGradient(), cPosteriorGradient.getGradient(), w, b, 2)) ReturnFalseEx("Bayesian backward: posterior gradient split"); //--- Invalidate the stochastic-gradient cache and pass the mean gradient upstream. bSamplesGradientReady = false; if(n.Activation() != None && !DeActivation(n.getOutput(), n.getGradient(), n.getGradient(), n.Activation())) ReturnFalseEx("Bayesian backward: input deactivation"); return(true); } //+------------------------------------------------------------------+ //| Generate K posterior-affine samples for one internal history. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::ForwardSamplesBatch(CBufferFloat *batch_input, CBufferFloat *output, uint rows_per_group) { //--- Validate the grouped history buffers before using their OpenCL offsets. uint w = iVariables * iInputs * iOutputs; uint b = iVariables * iOutputs; uint sample_total = iVariables * rows_per_group * iOutputs; if(!bTrain || !bRegularizersEnabled || !bSamplesPrepared || !batch_input || !output || rows_per_group == 0 || batch_input.GetOpenCL() != OpenCL || output.GetOpenCL() != OpenCL || batch_input.GetIndex() < 0 || output.GetIndex() < 0 || batch_input.Total() != int(iVariables * rows_per_group * iInputs) || output.Total() != int(iSamples * sample_total)) ReturnFalseEx("Bayesian batch forward: input or output contract"); if(!BuildPosterior()) ReturnFalseEx("Bayesian batch forward: posterior assembly"); //--- Evaluate every posterior sample into its strided output segment. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); if(!sample) ReturnFalseEx("Bayesian batch forward: random sampler"); if(!DeConcat(cSampleWeights.getOutput(), cSampleBias.getOutput(), sample.getOutput(), w, b, 1)) ReturnFalseEx("Bayesian batch forward: sampled posterior split"); if(!MatMul(batch_input, cSampleWeights.getOutput(), output, rows_per_group, iInputs, iOutputs, iVariables, true, 0, 0, i * sample_total)) ReturnFalseEx("Bayesian batch forward: grouped matrix multiplication"); if(!SumVecMatrix(cSampleBias.getOutput(), output, output, iOutputs, iVariables, 0, i * sample_total, i * sample_total, 1.0f, rows_per_group)) ReturnFalseEx("Bayesian batch forward: bias addition"); } //--- The next backward pass must derive a fresh stochastic-gradient cache. bSamplesGradientReady = false; return(true); } //+------------------------------------------------------------------+ //| Evaluates retained posterior samples with checked affine output. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::ForwardSamplesBatchChecked(CBufferFloat *batch_input, CBufferFloat *output, uint rows_per_group, CBufferFloat *numeric_flags, CBufferFloat *numeric_ones, CBufferFloat *numeric_scalar, bool &rejected) { rejected = false; //--- Derive all grouped spans before narrowing OpenCL offsets. const ulong input_total = ulong(iVariables) * rows_per_group * iInputs; const ulong sample_total = ulong(iVariables) * rows_per_group * iOutputs; const ulong output_total = ulong(iSamples) * sample_total; const ulong weight_total = ulong(iVariables) * iInputs * iOutputs; const ulong bias_total = ulong(iVariables) * iOutputs; const ulong parameter_total = 2 * (weight_total + bias_total); const ulong scratch_total = (sample_total > parameter_total ? sample_total : parameter_total); if(!bTrain || !bRegularizersEnabled || !bSamplesPrepared || !batch_input || !output || !numeric_flags || !numeric_ones || !numeric_scalar || rows_per_group == 0 || input_total > INT_MAX || sample_total > INT_MAX || output_total > INT_MAX || weight_total > INT_MAX || bias_total > INT_MAX || parameter_total > INT_MAX || scratch_total > INT_MAX || batch_input.GetOpenCL() != OpenCL || output.GetOpenCL() != OpenCL || batch_input.GetIndex() < 0 || output.GetIndex() < 0 || batch_input.Total() != int(input_total) || output.Total() != int(output_total) || numeric_flags.GetOpenCL() != OpenCL || numeric_ones.GetOpenCL() != OpenCL || numeric_scalar.GetOpenCL() != OpenCL || numeric_flags.GetIndex() < 0 || numeric_ones.GetIndex() < 0 || numeric_scalar.GetIndex() < 0 || numeric_flags.GetIndex() == numeric_ones.GetIndex() || numeric_flags.GetIndex() == numeric_scalar.GetIndex() || numeric_ones.GetIndex() == numeric_scalar.GetIndex() || numeric_flags.Total() < int(scratch_total) || numeric_ones.Total() < int(scratch_total) || numeric_scalar.Total() != 1 || numeric_flags.GetIndex() == batch_input.GetIndex() || numeric_flags.GetIndex() == output.GetIndex() || numeric_ones.GetIndex() == batch_input.GetIndex() || numeric_ones.GetIndex() == output.GetIndex() || numeric_scalar.GetIndex() == batch_input.GetIndex() || numeric_scalar.GetIndex() == output.GetIndex()) return(false); if(!ValidateCheckedScratch(batch_input, output, (uint)scratch_total, numeric_flags, numeric_ones, numeric_scalar)) return(false); if(!BuildPosteriorChecked(numeric_flags, numeric_ones, numeric_scalar, rejected)) return(false); if(rejected) { bSamplesPrepared = false; return(true); } //--- Reuse the retained draws without regenerating stochastic samples. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); if(!sample) return(false); if(!MatMul(batch_input, sample.getOutput(), output, int(rows_per_group), int(iInputs), int(iOutputs), int(iVariables), true, 0, 0, (uint)(ulong(i) * sample_total), numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, (uint)sample_total, rejected)) return(false); if(rejected) { bSamplesPrepared = false; return(true); } if(!SumVecMatrix(sample.getOutput(), output, output, iOutputs, iVariables, (uint)weight_total, (uint)(ulong(i) * sample_total), (uint)(ulong(i) * sample_total), 1.0f, rows_per_group, numeric_flags)) return(false); if(!ReduceNumericFlags(numeric_flags, numeric_ones, numeric_scalar, (uint)sample_total, rejected)) return(false); if(rejected) { bSamplesPrepared = false; return(true); } } //--- Invalidate only the gradient cache after a successful checked batch. bSamplesGradientReady = false; return(true); } //+------------------------------------------------------------------+ //| Accumulate posterior sample gradients for one internal history. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::BackwardSamplesBatch(CBufferFloat *batch_input, CBufferFloat *sample_gradients, uint rows_per_group) { //--- Validate and reset per-history buffers before accumulating all samples. uint w = iVariables * iInputs * iOutputs; uint b = iVariables * iOutputs; uint p = w + b; uint sample_total = iVariables * rows_per_group * iOutputs; if(!bTrain || !bRegularizersEnabled || !bSamplesPrepared || !batch_input || !sample_gradients || rows_per_group == 0 || batch_input.GetOpenCL() != OpenCL || sample_gradients.GetOpenCL() != OpenCL || batch_input.GetIndex() < 0 || sample_gradients.GetIndex() < 0 || batch_input.Total() != int(iVariables * rows_per_group * iInputs) || sample_gradients.Total() != int(iSamples * sample_total)) ReturnFalseEx("Bayesian batch backward: input or gradient contract"); if(!CreateBatchTemporary(rows_per_group)) ReturnFalseEx("Bayesian batch backward: temporary buffers"); if(!bSamplesGradientReady && !cPosteriorGradient.getGradient().Fill(0.0f)) ReturnFalseEx("Bayesian batch backward: posterior gradient reset"); if(!cBatchInputGradientAccum.Fill(0.0f)) ReturnFalseEx("Bayesian batch backward: input gradient reset"); //--- Accumulate every random posterior sample into the common posterior gradient. for(uint i = 0; i < iSamples; i++) { CVAE *sample = cRandomSamplers.At(i); if(!sample) ReturnFalseEx("Bayesian batch backward: random sampler"); if(!DeConcat(cSampleWeights.getOutput(), cSampleBias.getOutput(), sample.getOutput(), w, b, 1)) ReturnFalseEx("Bayesian batch backward: sampled posterior split"); if(!SumVecMatrixGrad(cSampleBias.getGradient(), &cBatchSampleGradient, sample_gradients, iOutputs, iVariables, 0, 0, i * sample_total, 1.0f, rows_per_group)) ReturnFalseEx("Bayesian batch backward: bias gradient"); //--- Restore the affine bias sum without scaling the per-row matrix gradient. if(rows_per_group > 1 && !SumAndNormalize(cSampleBias.getGradient(), cSampleBias.getGradient(), cSampleBias.getGradient(), iOutputs, false, 0, 0, 0, 0.5f * float(rows_per_group))) ReturnFalseEx("Bayesian batch backward: bias gradient reduction"); if(!MatMulGrad(batch_input, &cBatchInputGradient, cSampleWeights.getOutput(), cSampleWeights.getGradient(), &cBatchSampleGradient, rows_per_group, iInputs, iOutputs, iVariables, true)) ReturnFalseEx("Bayesian batch backward: grouped affine gradient"); if(!SumAndNormalize(&cBatchInputGradient, &cBatchInputGradientAccum, &cBatchInputGradientAccum, iInputs, false, 0, 0, 0, 1.0f)) ReturnFalseEx("Bayesian batch backward: input gradient accumulation"); if(!Concat(cSampleWeights.getGradient(), cSampleBias.getGradient(), sample.getGradient(), w, b, 1)) ReturnFalseEx("Bayesian batch backward: sampled gradient assembly"); if(!cPosterior.CalcHiddenGradients(sample)) ReturnFalseEx("Bayesian batch backward: random sampler gradient"); //--- Preserve exact affine mean gradients while retaining the VAE log-variance path. if(!DeConcat(cSampleWeights.getGradient(), cSampleBias.getGradient(), cSampleWeights.getOutput(), cSampleBias.getOutput(), cPosterior.getGradient(), w, b, w, b, 1)) ReturnFalseEx("Bayesian batch backward: posterior gradient partition"); if(!DeConcat(cSampleWeights.getGradient(), cSampleBias.getGradient(), sample.getGradient(), w, b, 1)) ReturnFalseEx("Bayesian batch backward: affine mean gradient restore"); if(!Concat(cSampleWeights.getGradient(), cSampleBias.getGradient(), cSampleWeights.getOutput(), cSampleBias.getOutput(), cPosterior.getGradient(), w, b, w, b, 1)) ReturnFalseEx("Bayesian batch backward: posterior gradient restore"); if(!SumAndNormalize(cPosterior.getGradient(), cPosteriorGradient.getGradient(), cPosteriorGradient.getGradient(), p, false, 0, 0, 0, 1.0f)) ReturnFalseEx("Bayesian batch backward: posterior gradient accumulation"); } if(!DeConcat(cWeights.getGradient(), cBias.getGradient(), cPosteriorGradient.getGradient(), w, b, 2)) ReturnFalseEx("Bayesian batch backward: posterior gradient split"); //--- Mark the accumulated sample gradients ready for the parameter update. bSamplesGradientReady = true; return(true); } //+------------------------------------------------------------------+ //| Update posterior parameters after gradient accumulation. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::updateInputWeights(CNeuronBaseOCL *n) { if(!bTrain) return(true); //--- Apply the posterior updates once after all sample gradients are accumulated. if(!cWeights.UpdateInputWeights()) ReturnFalseEx("Bayesian update: posterior weights"); if(!cBias.UpdateInputWeights()) ReturnFalseEx("Bayesian update: posterior bias"); //--- Refresh the deterministic caches before switching to inference. if(!cWeights.FeedForward()) ReturnFalseEx("updated posterior weights cache"); if(!cBias.FeedForward()) ReturnFalseEx("updated posterior bias cache"); //--- Finalize the inherited update contract. return(true); } //+------------------------------------------------------------------+ //| Save posterior parameters and deterministic configuration. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::Save(const int h) { //--- Persist only posterior parameters and deterministic layer configuration. if(!CNeuronBaseOCL::Save(h)) ReturnFalseEx("Bayesian save: base layer"); if(FileWriteInteger(h, iVariables, INT_VALUE) < INT_VALUE) ReturnFalseEx("Bayesian save: variables"); if(FileWriteInteger(h, iInputs, INT_VALUE) < INT_VALUE) ReturnFalseEx("Bayesian save: inputs"); if(FileWriteInteger(h, iOutputs, INT_VALUE) < INT_VALUE) ReturnFalseEx("Bayesian save: outputs"); if(FileWriteInteger(h, iSamples, INT_VALUE) < INT_VALUE) ReturnFalseEx("Bayesian save: samples"); if(FileWriteFloat(h, m_fKLDMult) < sizeof(float)) ReturnFalseEx("Bayesian save: KLD multiplier"); if(!cWeights.Save(h)) ReturnFalseEx("Bayesian save: posterior weights"); if(!cBias.Save(h)) ReturnFalseEx("Bayesian save: posterior bias"); //--- Finalize the posterior checkpoint. return(true); } //+------------------------------------------------------------------+ //| Load posterior parameters and rebuild transient samplers. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::Load(const int h) { if(!CNeuronBaseOCL::Load(h)) ReturnFalseEx("Bayesian load: base layer"); //--- Restore the deterministic dimensions and reset runtime-only sampling state. iVariables = (uint)FileReadInteger(h, INT_VALUE); iInputs = (uint)FileReadInteger(h, INT_VALUE); iOutputs = (uint)FileReadInteger(h, INT_VALUE); iSamples = (uint)FileReadInteger(h, INT_VALUE); m_fKLDMult = FileReadFloat(h); iSampleState = 1; bRegularizersEnabled = true; bSamplesGradientReady = false; bSamplesPrepared = false; if(!iVariables || !iInputs || !iOutputs || !iSamples) ReturnFalseEx("Bayesian load: dimensions"); if(!MathIsValidNumber(m_fKLDMult) || m_fKLDMult < 0.0f) ReturnFalseEx("KLD multiplier is nonfinite or negative"); if(iSamples > INT_MAX) ReturnFalseEx("sample count"); //--- Derive every persisted posterior span with staged signed-buffer bounds. const ulong bias_total = (ulong)iVariables * (ulong)iOutputs; if(bias_total > ulong(INT_MAX) / 4) ReturnFalseEx("bias tensor size"); const ulong weights_total = bias_total * (ulong)iInputs; if(weights_total > ulong(INT_MAX) / 4) ReturnFalseEx("weight tensor size"); const int expected_output = (int)bias_total; const int expected_weight_logical = (int)(2 * weights_total); const int expected_weight_physical = (int)(4 * weights_total); const int expected_bias_logical = (int)(2 * bias_total); const int expected_bias_physical = (int)(4 * bias_total); //--- Validate the outer layer before accepting nested parameter owners. CBufferFloat *output_buffer = getOutput(); CBufferFloat *gradient_buffer = getGradient(); if(!output_buffer || !gradient_buffer) ReturnFalseEx("outer buffers"); if(output_buffer.Total() != expected_output || gradient_buffer.Total() != expected_output) ReturnFalseEx("outer buffer shape"); if(!LoadInsideLayer(h, cWeights.AsObject())) ReturnFalseEx("Bayesian load: posterior weights"); //--- Confirm all serialized weight owners match the restored affine contract. CBufferFloat *weights_output = cWeights.getOutput(); CBufferFloat *weights_gradient = cWeights.getGradient(); CBufferFloat *weights_physical = cWeights.getWeightsParams(); if(!weights_output || !weights_gradient || !weights_physical) ReturnFalseEx("weight buffers"); if(weights_output.Total() != expected_weight_logical || weights_gradient.Total() != expected_weight_logical || weights_physical.Total() != expected_weight_physical) ReturnFalseEx("weight buffer shape"); if(!LoadInsideLayer(h, cBias.AsObject())) ReturnFalseEx("Bayesian load: posterior bias"); //--- Confirm all serialized bias owners match the restored affine contract. CBufferFloat *bias_output = cBias.getOutput(); CBufferFloat *bias_gradient = cBias.getGradient(); CBufferFloat *bias_physical = cBias.getWeightsParams(); if(!bias_output || !bias_gradient || !bias_physical) ReturnFalseEx("bias buffers"); if(bias_output.Total() != expected_bias_logical || bias_gradient.Total() != expected_bias_logical || bias_physical.Total() != expected_bias_physical) ReturnFalseEx("bias buffer shape"); if(!CreateTemporary()) ReturnFalseEx("Bayesian load: temporary state"); //--- Recreate transient samplers; stochastic samples are not serialized. return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronBayesianLinearOCL GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronBayesianLinearOCL::SetOpenCL(COpenCLMy *o) { //--- Rebind every persistent buffer and sampler to the replacement device. CNeuronBaseOCL::SetOpenCL(o); cWeights.SetOpenCL(OpenCL); cBias.SetOpenCL(OpenCL); cPosterior.SetOpenCL(OpenCL); cSampleWeights.SetOpenCL(OpenCL); cSampleBias.SetOpenCL(OpenCL); cSampleOutput.SetOpenCL(OpenCL); cInputGradient.SetOpenCL(OpenCL); cPosteriorGradient.SetOpenCL(OpenCL); cMeanSampler.SetOpenCL(OpenCL); if(cBatchSampleGradient.Total() > 0) cBatchSampleGradient.BufferCreate(OpenCL); if(cBatchInputGradient.Total() > 0) cBatchInputGradient.BufferCreate(OpenCL); if(cBatchInputGradientAccum.Total() > 0) cBatchInputGradientAccum.BufferCreate(OpenCL); for(int i = 0; i < cRandomSamplers.Total(); i++) { CVAE *s = cRandomSamplers.At(i); if(s) s.SetOpenCL(OpenCL); } //--- Existing stochastic gradients cannot cross device contexts. bSamplesGradientReady = false; bSamplesPrepared = false; } //+------------------------------------------------------------------+ //| Checks checked-transfer support of the complete Bayesian tree. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::SupportsOpenCLChecked(void) const { //--- Validate the layer identity and complete stochastic sampler registry. if(Type() != defNeuronBayesianLinearOCL || iSamples == 0 || (uint)cRandomSamplers.Total() != iSamples) return(false); //--- Validate every embedded owner before a checked transfer can begin. if(!cWeights.SupportsOpenCLChecked() || !cBias.SupportsOpenCLChecked() || !cPosterior.SupportsOpenCLChecked() || !cSampleWeights.SupportsOpenCLChecked() || !cSampleBias.SupportsOpenCLChecked() || !cSampleOutput.SupportsOpenCLChecked() || !cInputGradient.SupportsOpenCLChecked() || !cPosteriorGradient.SupportsOpenCLChecked() || !cMeanSampler.SupportsOpenCLChecked()) return(false); for(int i = 0; i < cRandomSamplers.Total(); i++) { CObject *object = cRandomSamplers.At(i); if(!object || object.Type() != defNeuronVAEOCL) return(false); CVAE *sampler = object; if(!sampler.SupportsOpenCLChecked()) return(false); } //--- Finalize only when the complete embedded hierarchy is transferable. return(true); } //+------------------------------------------------------------------+ //| Transfers Bayesian children before inherited owners publish | //| last. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::SetOpenCLChecked(COpenCLMy *obj) { if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) return(false); //--- Valid same-context calls also discard stale sampled-gradient state. bSamplesGradientReady = false; bSamplesPrepared = false; //--- Transfer each embedded checked owner before the inherited base owner. if(!cWeights.SetOpenCLChecked(obj)) ReturnFalseEx("posterior weights context"); if(!cBias.SetOpenCLChecked(obj)) ReturnFalseEx("posterior bias context"); if(!cPosterior.SetOpenCLChecked(obj)) ReturnFalseEx("posterior context"); if(!cSampleWeights.SetOpenCLChecked(obj)) ReturnFalseEx("sample weights context"); if(!cSampleBias.SetOpenCLChecked(obj)) ReturnFalseEx("sample bias context"); if(!cSampleOutput.SetOpenCLChecked(obj)) ReturnFalseEx("sample output context"); if(!cInputGradient.SetOpenCLChecked(obj)) ReturnFalseEx("input gradient context"); if(!cPosteriorGradient.SetOpenCLChecked(obj)) ReturnFalseEx("posterior gradient context"); if(!cMeanSampler.SetOpenCLChecked(obj)) ReturnFalseEx("mean sampler context"); //--- Synchronize direct batch owners independently for safe same-B retry. if(cBatchSampleGradient.Total() > 0 && cBatchSampleGradient.GetIndex() >= 0 && cBatchSampleGradient.GetOpenCL() != obj && !cBatchSampleGradient.BufferRead()) ReturnFalseEx("batch sample gradient read"); if(cBatchSampleGradient.Total() > 0 && (cBatchSampleGradient.GetOpenCL() != obj || cBatchSampleGradient.GetIndex() < 0) && !cBatchSampleGradient.BufferCreate(obj)) ReturnFalseEx("batch sample gradient create"); if(cBatchInputGradient.Total() > 0 && cBatchInputGradient.GetIndex() >= 0 && cBatchInputGradient.GetOpenCL() != obj && !cBatchInputGradient.BufferRead()) ReturnFalseEx("batch input gradient read"); if(cBatchInputGradient.Total() > 0 && (cBatchInputGradient.GetOpenCL() != obj || cBatchInputGradient.GetIndex() < 0) && !cBatchInputGradient.BufferCreate(obj)) ReturnFalseEx("batch input gradient create"); if(cBatchInputGradientAccum.Total() > 0 && cBatchInputGradientAccum.GetIndex() >= 0 && cBatchInputGradientAccum.GetOpenCL() != obj && !cBatchInputGradientAccum.BufferRead()) ReturnFalseEx("batch input accumulation read"); if(cBatchInputGradientAccum.Total() > 0 && (cBatchInputGradientAccum.GetOpenCL() != obj || cBatchInputGradientAccum.GetIndex() < 0) && !cBatchInputGradientAccum.BufferCreate(obj)) ReturnFalseEx("batch input accumulation create"); //--- Transfer every validated stochastic sampler before outer publication. for(int i = 0; i < cRandomSamplers.Total(); i++) { CObject *object = cRandomSamplers.At(i); if(!object || object.Type() != defNeuronVAEOCL) ReturnFalseEx("random sampler type"); CVAE *sampler = object; if(!sampler.SetOpenCLChecked(obj)) ReturnFalseEx("random sampler context"); } //--- Publish the inherited base owner only after every child succeeded. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("inherited base context"); //--- Finalize the checked context adoption after all owners succeeded. return(true); } //+------------------------------------------------------------------+ //| Switch posterior parameters between training and inference. | //+------------------------------------------------------------------+ void CNeuronBayesianLinearOCL::TrainMode(bool f) { //--- Propagate the mode to posterior parameters and discard stochastic state. CNeuronBaseOCL::TrainMode(f); cWeights.TrainMode(bTrain); cBias.TrainMode(bTrain); bSamplesGradientReady = false; bSamplesPrepared = false; } //+------------------------------------------------------------------+ //| Copy posterior parameters from a compatible source layer. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalseEx("Bayesian weight copy: source layer type"); //--- Synchronize the posterior parameters and invalidate stochastic runtime state. CNeuronBayesianLinearOCL *Source = source; if(!cWeights.WeightsUpdate(Source.cWeights.AsObject(), tau)) ReturnFalseEx("Bayesian weight copy: posterior weights"); if(!cBias.WeightsUpdate(Source.cBias.AsObject(), tau)) ReturnFalseEx("Bayesian weight copy: posterior bias"); iSampleState = Source.iSampleState; bSamplesGradientReady = false; bSamplesPrepared = false; //--- Finalize the inherited weight-copy contract. return(true); } //+------------------------------------------------------------------+ //| Append posterior state to the parameter fingerprint. | //+------------------------------------------------------------------+ bool CNeuronBayesianLinearOCL::AppendParameterFingerprint(ulong &f) { //--- Include the Bayesian type tag before the posterior parameter state. f = (f ^ ulong(Type())) * ulong(1099511628211); if(!cWeights.AppendParameterFingerprint(f)) ReturnFalseEx("Bayesian fingerprint: posterior weights"); if(!cBias.AppendParameterFingerprint(f)) ReturnFalseEx("Bayesian fingerprint: posterior bias"); //--- Finalize the posterior fingerprint. return(true); } //+------------------------------------------------------------------+ //| Returns the layer info of the CNeuronBayesianLinearOCL. | //+------------------------------------------------------------------+ CLayerDescription *CNeuronBayesianLinearOCL::GetLayerInfo(void) { //--- Report the grouped affine and sampling contract to the graph description. CLayerDescription *r = CNeuronBaseOCL::GetLayerInfo(); if(r) { r.count = iOutputs; r.window = iInputs; r.variables = iVariables; r.layers = iSamples; } //--- Return the inherited description when its allocation succeeded. return(r); } //+------------------------------------------------------------------+