//+------------------------------------------------------------------+ //| NeuroNet_Attention.mqh | //| Copyright 2026, DNG | //| https://www.mql5.com/ru/users/dng | //+------------------------------------------------------------------+ #ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE //+------------------------------------------------------------------+ //| Logical method implementations generated from NeuroNet.mqh: | //| Attention. | //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronAttentionOCL GPU resources. | //+------------------------------------------------------------------+ CNeuronAttentionOCL::~CNeuronAttentionOCL(void) { DeleteObj(Querys); DeleteObj(Values); DeleteObj(Scores); DeleteObj(AttentionOut); DeleteObj(FF1); DeleteObj(FF2); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAttentionOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) { Querys = new CNeuronConvOCL(); if(CheckPointer(Querys) == POINTER_INVALID) ReturnFalse; if(!Querys.Init(0, 0, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys.SetActivationFunction(None); } //--- if(CheckPointer(Values) == POINTER_INVALID) { Values = new CNeuronConvOCL(); if(CheckPointer(Values) == POINTER_INVALID) ReturnFalse; if(!Values.Init(0, 2, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values.SetActivationFunction(None); } //--- if(CheckPointer(Scores) == POINTER_INVALID) { Scores = new CBufferFloat(); if(CheckPointer(Scores) == POINTER_INVALID) ReturnFalse; } if(!Scores.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) { AttentionOut = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut) == POINTER_INVALID) ReturnFalse; if(!AttentionOut.Init(0, 3, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); } //--- if(CheckPointer(FF1) == POINTER_INVALID) { FF1 = new CNeuronConvOCL(); if(CheckPointer(FF1) == POINTER_INVALID) ReturnFalse; if(!FF1.Init(0, 4, open_cl, window, window, window * 4, units_count, optimization_type, batch)) ReturnFalse; FF1.SetActivationFunction(LReLU); } //--- if(CheckPointer(FF2) == POINTER_INVALID) { FF2 = new CNeuronConvOCL(); if(CheckPointer(FF2) == POINTER_INVALID) ReturnFalse; if(!FF2.Init(0, 5, open_cl, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; FF2.SetActivationFunction(None); FF2.SetGradientIndex(Gradient.GetIndex()); } //--- iWindow = window; iUnits = units_count; activation = (ENUM_ACTIVATION)FF2.Activation(); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAttentionOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) //---if(!prevLayer.Output.BufferRead()) //--- ReturnFalse; #ifdef _DEBUG if(!prevLayer.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, (int)iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, AttentionOut.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, AttentionOut.Neurons()); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(!FF1.FeedForward(AttentionOut)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, AttentionOut.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronAttentionOCL layer. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!AttentionOut.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, AttentionOut.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, AttentionOut.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, Querys.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, Values.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, Values.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, Scores.GetIndex()); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!Querys.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, AttentionOut.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Querys)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Values)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, prevLayer.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)(iWindow + 1)) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.1f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, prevLayer.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getGradientIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); //---Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAttentionOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Values.UpdateInputWeights(prevLayer)) ReturnFalse; if(!FF1.UpdateInputWeights(AttentionOut)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronAttentionOCL weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronAttentionOCL *Source = source; dWeightsUpdate(Querys, Source, tau); dWeightsUpdate(Values, Source, tau); dWeightsUpdate(FF1, Source, tau); dWeightsUpdate(FF2, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronAttentionOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(CheckPointer(Querys) == POINTER_INVALID || !Querys.Save(file_handle)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores) == POINTER_INVALID || !Scores.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut) == POINTER_INVALID || !AttentionOut.Save(file_handle)) ReturnFalse; if(CheckPointer(FF1) == POINTER_INVALID || !FF1.Save(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID || !FF2.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iUnits, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAttentionOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) Querys = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values) == POINTER_INVALID) Values = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores) == POINTER_INVALID) Scores = new CBufferFloat(); if(Scores.GetIndex() >= 0) Scores.BufferFree(); if(!Scores.Load(file_handle)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) AttentionOut = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(FF1) == POINTER_INVALID) FF1 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF1.Load(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID) FF2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF2.Load(file_handle)) ReturnFalse; iWindow = FileReadInteger(file_handle); iUnits = FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Returns the layer info of the CNeuronAttentionOCL. | //+------------------------------------------------------------------+ CLayerDescription* CNeuronAttentionOCL::GetLayerInfo() { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return(result); result.window = (int)iWindow; result.count = (int)iUnits; return(result); } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronMHAttentionOCL GPU resources. | //+------------------------------------------------------------------+ CNeuronMHAttentionOCL::~CNeuronMHAttentionOCL(void) { DeleteObj(Querys2); DeleteObj(Querys3); DeleteObj(Querys4); DeleteObj(Keys2); DeleteObj(Keys3); DeleteObj(Keys4); DeleteObj(Values2); DeleteObj(Values3); DeleteObj(Values4); DeleteObj(Scores2); DeleteObj(Scores3); DeleteObj(Scores4); DeleteObj(Weights0); DeleteObj(AttentionOut2); DeleteObj(AttentionOut3); DeleteObj(AttentionOut4); DeleteObj(AttentionConcatenate); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHAttentionOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronAttentionOCL::Init(numOutputs, myIndex, open_cl, window, units_count, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) { Querys2 = new CNeuronConvOCL(); if(CheckPointer(Querys2) == POINTER_INVALID) ReturnFalse; if(!Querys2.Init(0, 6, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys2.SetActivationFunction(None); } //--- if(CheckPointer(Querys3) == POINTER_INVALID) { Querys3 = new CNeuronConvOCL(); if(CheckPointer(Querys3) == POINTER_INVALID) ReturnFalse; if(!Querys3.Init(0, 7, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys3.SetActivationFunction(None); } //--- if(CheckPointer(Querys4) == POINTER_INVALID) { Querys4 = new CNeuronConvOCL(); if(CheckPointer(Querys4) == POINTER_INVALID) ReturnFalse; if(!Querys4.Init(0, 8, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys4.SetActivationFunction(None); } //--- if(CheckPointer(Values2) == POINTER_INVALID) { Values2 = new CNeuronConvOCL(); if(CheckPointer(Values2) == POINTER_INVALID) ReturnFalse; if(!Values2.Init(0, 9, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values2.SetActivationFunction(None); } //--- if(CheckPointer(Values3) == POINTER_INVALID) { Values3 = new CNeuronConvOCL(); if(CheckPointer(Values3) == POINTER_INVALID) ReturnFalse; if(!Values3.Init(0, 10, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values3.SetActivationFunction(None); } //--- if(CheckPointer(Values4) == POINTER_INVALID) { Values4 = new CNeuronConvOCL(); if(CheckPointer(Values4) == POINTER_INVALID) ReturnFalse; if(!Values4.Init(0, 11, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values4.SetActivationFunction(None); } //--- if(CheckPointer(Scores2) == POINTER_INVALID) { Scores2 = new CBufferFloat(); if(CheckPointer(Scores2) == POINTER_INVALID) ReturnFalse; } if(!Scores2.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) { Scores3 = new CBufferFloat(); if(CheckPointer(Scores3) == POINTER_INVALID) ReturnFalse; } if(!Scores3.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) { Scores4 = new CBufferFloat(); if(CheckPointer(Scores4) == POINTER_INVALID) ReturnFalse; } if(!Scores4.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) { AttentionOut2 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut2) == POINTER_INVALID) ReturnFalse; if(!AttentionOut2.Init(0, 12, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut2.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) { AttentionOut3 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut3) == POINTER_INVALID) ReturnFalse; if(!AttentionOut3.Init(0, 13, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut3.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) { AttentionOut4 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut4) == POINTER_INVALID) ReturnFalse; if(!AttentionOut4.Init(0, 14, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut4.SetActivationFunction(None); } //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) { AttentionConcatenate = new CNeuronBaseOCL(); if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) ReturnFalse; if(!AttentionConcatenate.Init(0, 15, open_cl, 4 * window * units_count, optimization_type, batch)) ReturnFalse; AttentionConcatenate.SetActivationFunction(None); } //--- if(CheckPointer(Weights0) == POINTER_INVALID) { Weights0 = new CNeuronConvOCL(); if(CheckPointer(Weights0) == POINTER_INVALID) ReturnFalse; if(!Weights0.Init(0, 16, open_cl, 4 * window, 4 * window, window, units_count, optimization_type, batch)) ReturnFalse; Weights0.SetActivationFunction(None); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHAttentionOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {1}; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.FeedForward(prevLayer)) ReturnFalse; //--- Scores Head 1 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()) setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()) kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 2 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys2.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores2.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores2.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores2.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values2.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut2.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut2.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 3 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys3.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores3.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores3.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores3.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values3.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut3.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut3.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 4 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys4.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores4.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores4.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores4.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values4.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut4.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut4.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iUnits, iWindow }; setBuffer(def_k_ConcatenateMatrix, def_k_conc_input1, AttentionOut.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window1, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input2, AttentionOut2.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window2, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input3, AttentionOut3.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window3, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input4, AttentionOut4.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window4, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_out, AttentionConcatenate.getOutputIndex()); kernelExecute(def_k_ConcatenateMatrix, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionConcatenate.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.FeedForward(AttentionConcatenate)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, Weights0.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, Weights0.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, Weights0.Neurons()); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(!FF1.FeedForward(Weights0)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHAttentionOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!Weights0.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, Weights0.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!AttentionConcatenate.CalcHiddenGradients((CObject *)Weights0)) ReturnFalse; //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iWindow}; setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output1, AttentionOut.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window1, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output2, AttentionOut2.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window2, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output3, AttentionOut3.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window3, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output4, AttentionOut4.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window4, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_inputs, AttentionConcatenate.getGradientIndex()); kernelExecute(def_k_DeconcatenateMatrix, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!calcHeadGradient(Querys, Values, Scores, AttentionOut, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys2, Values2, Scores2, AttentionOut2, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys3, Values3, Scores3, AttentionOut3, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys4, Values4, Scores4, AttentionOut4, prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix1, AttentionOut.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix2, AttentionOut2.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix3, AttentionOut3.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix4, AttentionOut4.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix5, Weights0.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix_out, prevLayer.getGradientIndex()); setArgument(def_k_Matrix5Sum, def_k_sum5_dimension, (int)iWindow); setArgument(def_k_Matrix5Sum, def_k_sum5_multiplyer, (float)0.2); kernelExecute(def_k_Matrix5Sum, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHAttentionOCL calc head gradient operation. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcHeadGradient(CNeuronConvOCL *query, CNeuronConvOCL *value, CBufferFloat *score, CNeuronBaseOCL *attention, CNeuronBaseOCL *prevLayer ) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, attention.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, query.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, query.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, value.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, value.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, score.GetIndex()); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!query.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)query)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)value)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow + 1) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.33f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHAttentionOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer) || !Querys2.UpdateInputWeights(prevLayer) || !Querys3.UpdateInputWeights(prevLayer) || !Querys4.UpdateInputWeights(prevLayer)) ReturnFalse; //--- if(!Values.UpdateInputWeights(prevLayer) || !Values2.UpdateInputWeights(prevLayer) || !Values3.UpdateInputWeights(prevLayer) || !Values4.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Weights0.UpdateInputWeights(AttentionConcatenate)) ReturnFalse; if(!FF1.UpdateInputWeights(Weights0)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHAttentionOCL weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMHAttentionOCL *Source = source; if(!Querys.WeightsUpdate(Source.Querys, tau) || !Querys2.WeightsUpdate(Source.Querys2, tau) || !Querys3.WeightsUpdate(Source.Querys3, tau) || !Querys4.WeightsUpdate(Source.Querys4, tau)) ReturnFalse; //--- if(!Values.WeightsUpdate(Source.Values, tau) || !Values2.WeightsUpdate(Source.Values2, tau) || !Values3.WeightsUpdate(Source.Values3, tau) || !Values4.WeightsUpdate(Source.Values4, tau)) ReturnFalse; if(!Weights0.WeightsUpdate(Source.Weights0, tau)) ReturnFalse; if(!FF1.WeightsUpdate(Source.FF1, tau)) ReturnFalse; if(!FF2.WeightsUpdate(Source.FF2, tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHAttentionOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Save(const int file_handle) { if(!CNeuronAttentionOCL::Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.Save(file_handle)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores2) == POINTER_INVALID || !Scores2.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut2) == POINTER_INVALID || !AttentionOut2.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.Save(file_handle)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores3) == POINTER_INVALID || !Scores3.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut3) == POINTER_INVALID || !AttentionOut3.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.Save(file_handle)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores4) == POINTER_INVALID || !Scores4.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut4) == POINTER_INVALID || !AttentionOut4.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID || !AttentionConcatenate.Save(file_handle)) ReturnFalse; if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHAttentionOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Load(const int file_handle) { if(!CNeuronAttentionOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) Querys2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values2) == POINTER_INVALID) Values2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores2) == POINTER_INVALID) Scores2 = new CBufferFloat(); if(Scores2.GetIndex() >= 0) Scores2.BufferFree(); if(!Scores2.Load(file_handle)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) AttentionOut2 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID) Querys3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values3) == POINTER_INVALID) Values3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) Scores3 = new CBufferFloat(); if(Scores3.GetIndex() >= 0) Scores3.BufferFree(); if(!Scores3.Load(file_handle)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) AttentionOut3 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID) Querys4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values4) == POINTER_INVALID) Values4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) Scores4 = new CBufferFloat(); if(Scores4.GetIndex() >= 0) Scores4.BufferFree(); if(!Scores4.Load(file_handle)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) AttentionOut4 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) AttentionConcatenate = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionConcatenate.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Weights0) == POINTER_INVALID) Weights0 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Weights0.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Returns the layer info of the CNeuronMHAttentionOCL. | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronAttentionOCL::GetLayerInfo(); if(!result) return(result); result.step = 4; //--- return(result); } //+------------------------------------------------------------------+ //| Constructor: initializes the CNeuronMLMHAttentionOCL layer | //| defaults. | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::CNeuronMLMHAttentionOCL(void) : iLayers(0), iHeads(0), iWindow(0), iWindowKey(0), iUnits(0) { QKV_Tensors = new CCollection(); QKV_Weights = new CCollection(); S_Tensors = new CCollection(); AO_Tensors = new CCollection(); FF_Tensors = new CCollection(); FF_Weights = new CCollection(); } //+------------------------------------------------------------------+ //| Destructor: releases the CNeuronMLMHAttentionOCL GPU resources. | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::~CNeuronMLMHAttentionOCL(void) { DeleteObj(QKV_Tensors); DeleteObj(QKV_Weights); DeleteObj(S_Tensors); DeleteObj(AO_Tensors); DeleteObj(FF_Tensors); DeleteObj(FF_Weights); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMLMHAttentionOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- //--- Size of QKV tensor uint num = 3 * iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of QKV tenzor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits; //--- Size of our tensore uint out = iWindow * iUnits; //--- Size W0 tensor uint w0 = (iWindowKey + 1) * iHeads * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? qkv_weights : 3 * iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? w0 : out), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMLMHAttentionOCL feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL convolution forward | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0, uint variables = 1 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = { outputs.Total() / (window_out * variables), window_out, variables }; setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_w, weights.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_i, inputs.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_o, outputs.GetIndex()); setArgument(def_k_FeedForwardConv, def_k_ffc_inputs, (int)(inputs.Total() / variables)); setArgument(def_k_FeedForwardConv, def_k_ffc_step, (int)step); setArgument(def_k_FeedForwardConv, def_k_ffc_window_in, (int)window); setArgument(def_k_FeedForwardConv, def_k_ffс_window_out, (int)window_out); setArgument(def_k_FeedForwardConv, def_k_ffc_activation, (int)activ); kernelExecute(def_k_FeedForwardConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL attention score operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHAttentionScore, def_k_mhas_mask, (int)mask); kernelExecute(def_k_MHAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMLMHAttentionOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL convolution input gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionInputGradients(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0, uint variables = 1 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total(); global_work_size[1] = variables; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, weights.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, gradient.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, inputs.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, gradient.Total() - shift_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)step); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)window); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)window_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)activ); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)shift_out); kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!inp_gradient.BufferRead()) ReturnFalse; #endif //--- //--- inp_gradient.BufferRead(); return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionInsideGradients(CBufferFloat* qkv, CBufferFloat* qkv_g, CBufferFloat* scores, CBufferFloat* gradient ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_score, scores.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()); kernelExecute(def_k_MHAttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv_g.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMLMHAttentionOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads, 0, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL convolutuion update weights | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* momentum1, CBufferFloat* momentum2, uint window, uint window_out, uint step = 0, uint heads = 0, uint variables = 1 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = weights.Total(); uint global_work_offset_am[3] = {0, 0, 0}; uint global_work_size_am[3] = {window, window_out, variables}; uint local_work_size_am[3] = {window, (heads > 0 ? window_out / heads : 1), variables}; if(weights.GetIndex() < 0) ReturnFalse; float lt = 0; switch(optimization) { case SGD: if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_inputs, inputs.Total()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_learning_rates, lr); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_momentum, alpha); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_in, (int)window); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_out, (int)window_out); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_step, (int)step); ResetLastError(); kernelExecute(def_k_UpdateWeightsConvMomentum, global_work_offset, global_work_size) break; case ADAM: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsConvAdam, global_work_offset, global_work_size) t++; break; case ADAM_MINI: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_v, momentum2.GetIndex()) lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_l, lt) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b1, b1) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b2, b2) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_step, (int)step) ResetLastError(); kernelExecuteLoc(def_k_UpdateWeightsConvAdamMini, global_work_offset_am, global_work_size_am, local_work_size_am) t++; break; //--- default: printf("Error of optimization type %s: %s", __FUNCSIG__, EnumToString(optimization)); ReturnFalse; } #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- global_work_size[0] = window_out; setBuffer(def_k_NormalizeWeights, def_k_norm_buffer, weights.GetIndex()); setArgument(def_k_NormalizeWeights, def_k_norm_dimension, (int)window + 1); kernelExecute(def_k_NormalizeWeights, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- //--- weights.BufferRead(); return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMLMHAttentionOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayers, INT_VALUE) || !FileWriteInteger(file_handle, iHeads, INT_VALUE) || !FileWriteInteger(file_handle, iWindow, INT_VALUE) || !FileWriteInteger(file_handle, iUnits, INT_VALUE) || !FileWriteInteger(file_handle, iWindowKey, INT_VALUE)) ReturnFalse; //--- Saving objects if(!QKV_Tensors.Save(file_handle) || !QKV_Weights.Save(file_handle) || !S_Tensors.Save(file_handle) || !AO_Tensors.Save(file_handle) || !FF_Tensors.Save(file_handle) || !FF_Weights.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMLMHAttentionOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 6 - 4); Gradient = FF_Tensors.At(iLayers * 6 - 1); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMLMHAttentionOCL GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); QKV_Tensors.SetOpenCL(OpenCL); QKV_Weights.SetOpenCL(OpenCL); S_Tensors.SetOpenCL(OpenCL); AO_Tensors.SetOpenCL(OpenCL); FF_Tensors.SetOpenCL(OpenCL); FF_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Returns the layer info of the CNeuronMLMHAttentionOCL. | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMLMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return(result); //--- result.window = (int)iWindow; result.step = (int)iHeads; result.window_out = (int)iWindowKey; result.count = (int)iUnits; result.layers = (int)iLayers; //--- return(result); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHSparseAttention attention score | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionScore(CBufferFloat* qkv, CBufferFloat* scores, bool mask = true) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_sparse, m_dSparse); kernelExecute(def_k_MHSparseAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHSparseAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionOut(CBufferFloat* qkv, CBufferFloat* scores, CBufferFloat* out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHSparseAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHSparseAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMLMHSparseAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteFloat(file_handle, m_dSparse) < sizeof(float)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMLMHSparseAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; m_dSparse = FileReadFloat(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMLMHAttentionOCL weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionOCL convolutuion update weights | //| operation (ConvolutuionUpdateWeights). | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* source, CBufferFloat* momentum1, CBufferFloat* momentum2, float tau ) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {weights.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, weights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, source.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, momentum1.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, momentum2.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, weights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, source.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Constructor: initializes the CNeuronMH2AttentionOCL layer | //| defaults. | //+------------------------------------------------------------------+ CNeuronMH2AttentionOCL::CNeuronMH2AttentionOCL(void) : iHeads(0), iWindow(0), iUnits(0), iWindowKey(0), ScoreIndex(INVALID_HANDLE) { activation = None; } //+------------------------------------------------------------------+ //| Releases the raw MH2 score buffer while its context is live. | //+------------------------------------------------------------------+ CNeuronMH2AttentionOCL::~CNeuronMH2AttentionOCL(void) { if(!!OpenCL && ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = INVALID_HANDLE; } //+------------------------------------------------------------------+ //| Initializes the CNeuronMH2AttentionOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Release the old raw score slot before the base changes its context. if(ScoreIndex >= 0 && !!OpenCL) { if(!OpenCL.BufferFree(ScoreIndex)) ReturnFalse; ScoreIndex = INVALID_HANDLE; } //--- Initialize the inherited owners before allocating MH2 children. if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Transpose.Init(0, 0, OpenCL, iUnits, iWindow, optimization_type, batch)) ReturnFalse; Transpose.SetActivationFunction(None); //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iUnits, iUnits, 2 * iWindowKey * iHeads, iWindow, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); //--- if(iUnits > uint(INT_MAX) / iWindow) ReturnFalse; const uint score_count = iUnits * iWindow; if(score_count > uint(INT_MAX) / iHeads) ReturnFalse; const uint score_total = score_count * iHeads; if(score_total > UINT_MAX / sizeof(float)) ReturnFalse; ScoreIndex = OpenCL.AddBuffer(score_total * sizeof(float), CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) FF[i].SetActivationFunction(None); //--- CBufferFloat *ff1_gradient = FF[1].getGradient(); if(Gradient != ff1_gradient) { Gradient.BufferFree(); DeleteObj(Gradient); Gradient = ff1_gradient; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMH2AttentionOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Transpose.FeedForward(NeuronOCL) || !KV_Embedding.FeedForward(Transpose.AsObject())) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMH2AttentionOCL attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindow/*K units*/, iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMH2AttentionOCL attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!MHAttentionOut.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMH2AttentionOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(GetPointer(Transpose))) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMH2AttentionOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Transpose.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getGradient(), AttentionOut.getGradient(), iWindow, false)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(Transpose.AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), AttentionOut.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMH2AttentionOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!Q_Embedding.Save(file_handle)) ReturnFalse; if(!KV_Embedding.Save(file_handle)) ReturnFalse; if(!Transpose.Save(file_handle)) ReturnFalse; if(!W0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Append the trainable multi-head attention graph. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::AppendParameterFingerprint(ulong &fingerprint) { if(!CNeuronBaseOCL::AppendParameterFingerprint(fingerprint) || !Q_Embedding.AppendParameterFingerprint(fingerprint) || !KV_Embedding.AppendParameterFingerprint(fingerprint) || !W0.AppendParameterFingerprint(fingerprint)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].AppendParameterFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMH2AttentionOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, Q_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, KV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, Transpose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, W0.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, FF[i].AsObject())) ReturnFalse; if(!SetGradient(FF[1].getGradient(), true)) ReturnFalse; //--- iUnits = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization, iBatch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; AttentionOut.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMH2AttentionOCL GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMH2AttentionOCL::SetOpenCL(COpenCLMy * obj) { if(CheckPointer(obj) == POINTER_INVALID || OpenCL == obj) return; //--- Release the old transient scores before any owner pointer changes. if(!!OpenCL && ScoreIndex >= 0) { if(!OpenCL.BufferFree(ScoreIndex)) return; ScoreIndex = INVALID_HANDLE; } //--- Rebind persistent children; legacy callers intentionally discard scores. CNeuronBaseOCL::SetOpenCL(obj); if(OpenCL != obj) return; Q_Embedding.SetOpenCL(OpenCL); KV_Embedding.SetOpenCL(OpenCL); if(Transpose.Neurons() > 0) Transpose.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); W0.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL); FF[0].SetOpenCL(OpenCL); FF[1].SetOpenCL(OpenCL); if(iUnits == 0 || iWindow == 0 || iHeads == 0) return; if(iUnits > uint(INT_MAX) / iWindow) return; const uint score_count = iUnits * iWindow; if(score_count > uint(INT_MAX) / iHeads) return; const uint score_total = score_count * iHeads; if(score_total > UINT_MAX / sizeof(float)) return; ScoreIndex = OpenCL.AddBuffer(score_total * sizeof(float), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Reports checked transfer support for exact MH2 attention only. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::SupportsOpenCLChecked(void) const { //--- Require exact MH2 identity and a live, shape-matching raw score owner. if(Type() != defNeuronMH2AttentionOCL || CheckPointer(OpenCL) == POINTER_INVALID || ScoreIndex < 0 || iUnits == 0 || iWindow == 0 || iWindowKey == 0 || iHeads == 0 || iUnits > uint(INT_MAX) / iWindow) return(false); const uint score_count = iUnits * iWindow; if(score_count > uint(INT_MAX) / iHeads) return(false); const uint score_total = score_count * iHeads; if(score_total > UINT_MAX / sizeof(float) || OpenCL.GetBufferSize(ScoreIndex) != score_total * sizeof(float)) return(false); //--- Require every nested owner before any checked transfer mutates it. return(Q_Embedding.SupportsOpenCLChecked() && KV_Embedding.SupportsOpenCLChecked() && Transpose.SupportsOpenCLChecked() && MHAttentionOut.SupportsOpenCLChecked() && W0.SupportsOpenCLChecked() && AttentionOut.SupportsOpenCLChecked() && FF[0].SupportsOpenCLChecked() && FF[1].SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers MH2 children and device scores to a checked context. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the complete MH2 owner set before creating a candidate. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or MH2 tree"); int candidate = INVALID_HANDLE; COpenCLMy *old_owner = OpenCL; int old_score = ScoreIndex; //--- Read and stage scores only when the target differs from the source. if(obj != OpenCL) { const uint score_total = (uint)(old_owner.GetBufferSize(old_score) / sizeof(float)); float scores[]; if(ArrayResize(scores, (int)score_total) != (int)score_total) ReturnFalseEx("score snapshot allocation"); if(!old_owner.BufferRead(old_score, scores, 0, 0, score_total)) ReturnFalseEx("score snapshot read"); candidate = obj.AddBufferFromArray(scores, 0, score_total, CL_MEM_READ_WRITE); if(candidate == INVALID_HANDLE) ReturnFalseEx("score candidate create"); } //--- Transfer children before publishing the parent base context. if(!Q_Embedding.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("query embedding context"); } if(!KV_Embedding.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("key-value embedding context"); } if(!Transpose.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("transpose context"); } if(!MHAttentionOut.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("attention output context"); } if(!W0.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("projection context"); } if(!AttentionOut.SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("residual context"); } if(!FF[0].SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("feed-forward zero context"); } if(!FF[1].SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("feed-forward one context"); } if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) { if(candidate >= 0 && !obj.BufferFree(candidate)) ReturnFalseEx("score candidate cleanup"); ReturnFalseEx("base context"); } if(candidate == INVALID_HANDLE) return(true); //--- Associate the candidate with B even if releasing A reports failure. ScoreIndex = candidate; if(!old_owner.BufferFree(old_score)) ReturnFalseEx("source score release"); //--- Finalize the checked transfer with the score owned by the published context. return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMH2AttentionOCL weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMH2AttentionOCL *src = source; if(!Q_Embedding.WeightsUpdate(GetPointer(src.Q_Embedding), tau)) ReturnFalse; if(!KV_Embedding.WeightsUpdate(GetPointer(src.KV_Embedding), tau)) ReturnFalse; if(!W0.WeightsUpdate(GetPointer(src.W0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].WeightsUpdate(GetPointer(src.FF[i]), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMFTOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint features, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * features, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(features, 1); //--- MHSA //--- Size of QKV tensor uint num = 3 * iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits; //--- Size of output tensor uint out = iWindow * iUnits; //--- Size W0 tensor uint w0 = (iWindowKey + 1) * iHeads * iWindow; //--- MHCA //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits; //--- Size of KV tensor uint num_kv = 2 * iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //--- Size of weights' matrix of KV tenzor uint kv_weights = 2 * (iUnits + 1) * iWindowKey * iHeads; //--- Size of Score tensor uint scores_ca = iUnits * iWindow * iHeads; //--- FF //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- MHSA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cTranspose.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores_ca, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads cross attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- MHSA //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; for(uint w = 0; w < q_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float kv = (float)(1 / sqrt(iUnits + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* kv)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- MHSA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Transposes the internal tensors of the CNeuronMFTOCL layer. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Transpose(CBufferFloat* in, CBufferFloat* out, int rows, int cols) { if(!in || !out) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {rows, cols}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, in.GetIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, out.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMFTOCL mhca operation. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCA(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* score, CBufferFloat* out) { if(!q || !kv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindow, iHeads}; uint local_work_size[3] = {1, iWindow, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, score.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMFTOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- MHSA //--- Calculate Queries, Keys, Values CBufferFloat *inputs = NeuronOCL.getOutput(); CBufferFloat *qkv = QKV_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 4); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 4); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 8); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- MHCA inputs = temp; CBufferFloat *q = QKV_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2); if(IsStopped() || !Transpose(inputs, tr, iUnits, iWindow)) ReturnFalse; CBufferFloat *kv = QKV_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), tr, kv, iUnits, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Multi-heads cross attention calculation temp = S_Tensors.At(i * 4 + 1); out = AO_Tensors.At(i * 4 + 1); if(IsStopped() || !MHCA(q, kv, temp, out)) ReturnFalse; //--- Cross Attention out calculation temp = FF_Tensors.At(i * 8 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 8 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 8 + 3); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, Output, iWindow, true, 0, 0, i * inputs.Total())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMFTOCL mhca inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCAInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* score, CBufferFloat* aog ) { if(!q || !qg || !kv || !kvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, score.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, aog.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!aog.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMFTOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *inp = prevLayer.getOutput(); CBufferFloat *grad = prevLayer.getGradient(); //--- for(int i = 0; (i < (int)iLayers && !IsStopped()); i++) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), Gradient, FF_Tensors.At(i * 8 + 2), FF_Tensors.At(i * 8 + 6), 4 * iWindow, iWindow, None, i * inp.Total())) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 8 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(i * 8 + 6), FF_Tensors.At(i * 8 + 1), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradient if(IsStopped() || !SumAndNormalize(Gradient, temp, temp, iWindow, false, i * inp.Total(), 0, 0)) ReturnFalse; out_grad = temp; //--- MHCA //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out_grad, AO_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3), iWindowKey * iHeads, iWindow, None)) ReturnFalse; if(IsStopped() || !MHCAInsideGradients(QKV_Tensors.At(i * 6 + 1), QKV_Tensors.At(i * 6 + 4), QKV_Tensors.At(i * 6 + 2), QKV_Tensors.At(i * 6 + 5), S_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3))) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2 + 1); if(IsStopped() || !Transpose(QKV_Tensors.At(i * 6 + 5), tr, iWindow, iUnits)) ReturnFalse; //--- Sum temp = FF_Tensors.At(i * 8 + 4); if(IsStopped() || !SumAndNormalize(QKV_Tensors.At(i * 6 + 4), tr, temp, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; //--- MHSA //--- Split gradient to multi-heads out_grad = temp; if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out_grad, AO_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 2), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 6), QKV_Tensors.At(i * 6 + 3), S_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 1))) ReturnFalse; //--- if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(i * 6 + 3), inp, tr, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum gradients if(i > 0) { if(IsStopped() || !SumAndNormalize(grad, tr, grad, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormalize(out_grad, grad, grad, iWindow, false)) ReturnFalse; } else if(IsStopped() || !SumAndNormalize(out_grad, tr, grad, iWindow, false)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMFTOCL trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(l * 6 + 3), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), iWindow, 3 * iWindowKey * iHeads, 0, 3 * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), QKV_Tensors.At(l * 6 + 4), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), QKV_Tensors.At(l * 6 + 5), cTranspose.At(l * 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), iUnits, 2 * iWindowKey * iHeads, 0, 2 * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), FF_Tensors.At(l * 8 + 4), AO_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), FF_Tensors.At(l * 8 + 5), AO_Tensors.At(l * 4 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(l * 8 + 6), FF_Tensors.At(l * 8 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), FF_Tensors.At(l * 8 + 7), FF_Tensors.At(l * 8 + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMFTOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMFTOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle) || !cTranspose.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL) || !cTranspose.SetOpenCL(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMFTOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMFTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Averages the CNeuronMFTOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMFTOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12)), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 3), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronXCiTOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iUnits = fmax(units_count, 1); iHeads = fmax(fmin(heads, iWindow), 1); iWindowKey = fmax((window + iHeads - 1) / iHeads, 1); iLayers = fmax(layers, 1); iLPIWindow = fmax(lpi_window, 1); iLPIStep = 1; //--- XCA //--- Size of QKV tensor uint num = 3 * iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //--- Size of Score tensor uint scores = iWindowKey * iWindowKey * iHeads; //--- Size of output tensor uint out = iWindow * iUnits; //--- LPI //--- Size of LPI1 tensor uint lpi1_num = iWindow * iHeads * iUnits; //--- Size of weights' matrix of LPI1 tenzor uint lpi1_weights = (iLPIWindow + 1) * iHeads; //--- Size of weights' matrix of LPI2 tenzor uint lpi2_weights = (iHeads + 1) * 2; //--- FF //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- LPI //--- Initilize LPI tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; //--- LPI1 if(!cLPI.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; //--- LPI Normalize if(!cLPI.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; //--- LPI2 if(!cLPI.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi1_weights)) ReturnFalse; for(uint w = 0; w < lpi1_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Normalization int count = (int)lpi1_num * (optimization_type == SGD ? 7 : 9); temp = new CBufferFloat(); if(!temp.BufferInit(count, 0.0f)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi2_weights)) ReturnFalse; for(uint w = 0; w < lpi2_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- LPI temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi2_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } iBatchCount = 1; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronXCiTOCL x ci t operation. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiT(CBufferFloat* qkv, CBufferFloat* score, CBufferFloat* out) { if(!OpenCL || !qkv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; uint local_work_size[3] = {iWindowKey, iUnits, 1}; setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_score, score.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_out, out.GetIndex()) ResetLastError(); kernelExecuteLoc(def_k_XCiTFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronXCiTOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(qkv, temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronXCiTOCL batch norm operation. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNorm(CBufferFloat* inputs, CBufferFloat* options, CBufferFloat* out) { if(!OpenCL || !inputs || !options || !out) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); iBatchCount = MathMin(iBatchCount, iBatch); setBuffer(def_k_BatchFeedForward, def_k_bff_inputs, inputs.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_options, options.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_output, out.GetIndex()) setArgument(def_k_BatchFeedForward, def_k_bff_batch, (int)iBatchCount) setArgument(def_k_BatchFeedForward, def_k_bff_optimization, (int)optimization) setArgument(def_k_BatchFeedForward, def_k_bff_activation, (int)None) ResetLastError(); kernelExecute(def_k_BatchFeedForward, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronXCiTOCL batch norm inside gradient operation. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormInsideGradient(CBufferFloat* inputs, CBufferFloat* inputs_g, CBufferFloat* options, CBufferFloat* out, CBufferFloat* out_g, ENUM_ACTIVATION activ) { if(!OpenCL || !inputs || !inputs_g || !options || !out || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_i, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_options, options.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_g, out_g.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_ig, inputs_g.GetIndex()) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_activation, (int)activ) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_batch, (int)iBatchCount) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_optimization, (int)optimization) ResetLastError(); kernelExecute(def_k_CalcHiddenGradientBatch, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronXCiTOCL batch norm update weights operation. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormUpdateWeights(CBufferFloat* options, CBufferFloat* out_g) { if(!OpenCL || !options || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = out_g.Total(); //--- if(optimization == SGD) { setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_learning_rates, lr) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_momentum, alpha) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsMomentum, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif } else { setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_l, lr) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b1, b1) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b2, b2) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronXCiTOCL x ci t inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiTInsideGradients(CBufferFloat* qkv, CBufferFloat* qkvg, CBufferFloat* score, CBufferFloat* aog) { if(!OpenCL || !qkv || !qkvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv_g, qkvg.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_scores, score.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_gradient, aog.GetIndex()) ResetLastError(); kernelExecute(def_k_XCiTInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!aog.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronXCiTOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronXCiTOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronXCiTOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronXCiTOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLPIWindow, INT_VALUE) || !FileWriteInteger(file_handle, iLPIStep, INT_VALUE) || !FileWriteInteger(file_handle, iBatchCount, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cLPI.Save(file_handle) || !cLPI_Weights.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronXCiTOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLPIWindow = FileReadInteger(file_handle, INT_VALUE); iLPIStep = FileReadInteger(file_handle, INT_VALUE); iBatchCount = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!cLPI.Load(file_handle) || !cLPI_Weights.Load(file_handle)) ReturnFalse; if(!cLPI.SetOpenCL(OpenCL) || !cLPI_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 4 - 3); Gradient = FF_Tensors.At(iLayers * 4 - 1); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronXCiTOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cLPI.SetOpenCL(OpenCL); cLPI_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDOTOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint dimension, uint heads, uint units_count, uint prev_window, ENUM_OPTIMIZATION optimization_type, uint batch ) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- if(prev_window != window) { if(!cProjInput.Init(0, 0, OpenCL, prev_window, prev_window, window, units_count, optimization_type, batch)) ReturnFalse; } //--- iWindowSize = window; iPrevWindowSize = prev_window; iDimension = dimension; iHeads = heads; iUnits = units_count; //--- if(!cQKV.Init(0, 1, OpenCL, window, window, dimension * heads, units_count, optimization_type, batch)) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- if(!cRelativePositionsBias.Init(1, 2, OpenCL, iUnits * iHeads * 3, optimization_type, batch)) ReturnFalse; if(!MHAttentionOut.Init(0, 3, OpenCL, iUnits * iHeads * iDimension, optimization_type, batch)) ReturnFalse; if(!cProj.Init(0, 4, OpenCL, iHeads * iDimension, iHeads * iDimension, window, iUnits, optimization_type, batch)) ReturnFalse; if(!AttentionOut.Init(0, 5, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cFF1.Init(0, 6, OpenCL, window, window, 4 * window, units_count, optimization_type, batch)) ReturnFalse; if(!cFF2.Init(0, 7, OpenCL, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; if(!SAttenOut.Init(0, 8, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cCAtten.Init(0, 9, OpenCL, window, MathMax(window / 2, 3), 8, iUnits, 1, optimization_type, batch)) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDOTOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; if(iPrevWindowSize != iWindowSize) { if(!cProjInput.FeedForward(inputs) || !cQKV.FeedForward(GetPointer(cProjInput))) ReturnFalse; inputs = GetPointer(cProjInput); } else if(!cQKV.FeedForward(inputs)) ReturnFalse; if(!DOT()) ReturnFalse; if(!cProj.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; if(!SumAndNormalize(inputs.getOutput(), cProj.getOutput(), AttentionOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cFF1.FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.FeedForward(GetPointer(cFF1))) ReturnFalse; if(!SumAndNormalize(AttentionOut.getOutput(), cFF2.getOutput(), SAttenOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cCAtten.FeedForward(GetPointer(SAttenOut))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDOTOCL dot operation. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOT(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iUnits, iHeads}; uint local_work_size[3] = {iDimension, 1, 1}; setBuffer(def_k_DOTFeedForward, def_k_dot_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_score, iScoreBuffer) setBuffer(def_k_DOTFeedForward, def_k_dot_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_out, MHAttentionOut.getOutputIndex()) ResetLastError(); kernelExecuteLoc(def_k_DOTFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDOTOCL dot inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOTInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iDimension, iHeads}; setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_scores, iScoreBuffer) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb_g, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_gradient, MHAttentionOut.getGradientIndex()) ResetLastError(); kernelExecute(def_k_DOTInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronDOTOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cCAtten.CalcHiddenGradients((CObject *)GetPointer(SAttenOut))) ReturnFalse; if(!cFF2.CalcHiddenGradients((CObject *)GetPointer(cFF1))) ReturnFalse; if(!cFF1.CalcHiddenGradients((CObject *)GetPointer(AttentionOut))) ReturnFalse; if(!SumAndNormalize(AttentionOut.getGradient(), SAttenOut.getGradient(), cProj.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProj.CalcHiddenGradients((CObject *)GetPointer(MHAttentionOut))) ReturnFalse; if(!DOTInsideGradients()) ReturnFalse; //--- if(iPrevWindowSize != iWindowSize) { if(!cQKV.CalcHiddenGradients((CObject *)GetPointer(cProjInput))) ReturnFalse; if(!SumAndNormalize(cProjInput.getGradient(), cProj.getGradient(), cProjInput.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProjInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; } else { if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cProj.getGradient(), prevLayer.getGradient(), iWindowSize, false)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDOTOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronDOTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjInput.SetOpenCL(OpenCL); cQKV.SetOpenCL(OpenCL); iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); cRelativePositionsBias.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); cProj.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL);; cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); SAttenOut.SetOpenCL(OpenCL); cCAtten.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronDOTOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrevWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- ResetLastError(); if(iWindowSize != iPrevWindowSize) if(!cProjInput.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cQKV.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cRelativePositionsBias.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!MHAttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cProj.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!AttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF1.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF2.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!SAttenOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cCAtten.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDOTOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindowSize = (uint)FileReadInteger(file_handle); iPrevWindowSize = (uint)FileReadInteger(file_handle); iDimension = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); //--- if(iWindowSize != iPrevWindowSize) if(!LoadInsideLayer(file_handle, GetPointer(cProjInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cRelativePositionsBias))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cProj))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(AttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF1))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF2))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(SAttenOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cCAtten))) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDOTOCL trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(iWindowSize != iPrevWindowSize) { if(!cProjInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(GetPointer(cProjInput))) ReturnFalse; } else { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; } //--- if(!cProj.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!cFF1.UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.UpdateInputWeights(GetPointer(cFF1))) ReturnFalse; if(!cCAtten.UpdateInputWeights(GetPointer(SAttenOut))) ReturnFalse; if(!updateRelativePositionsBias()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronDOTOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronDOTOCL *Source = source; if(iWindowSize != iPrevWindowSize) { if(!cProjInput.WeightsUpdate(GetPointer(Source.cProjInput), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } else { if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } //--- if(!cProj.WeightsUpdate(GetPointer(Source.cProj), tau)) ReturnFalse; if(!cFF1.WeightsUpdate(GetPointer(Source.cFF1), tau)) ReturnFalse; if(!cFF2.WeightsUpdate(GetPointer(Source.cFF2), tau)) ReturnFalse; if(!cCAtten.WeightsUpdate(GetPointer(Source.cCAtten), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDOTOCL update relative positions bias | //| operation. | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateRelativePositionsBias(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {cRelativePositionsBias.Neurons()}; setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_gradient, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_m, cRelativePositionsBias.getFirstMomentumIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_v, cRelativePositionsBias.getSecondMomentumIndex()) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b1, b1) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b2, b2) ResetLastError(); kernelExecute(def_k_RPBUpdateAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!cRelativePositionsBias.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronFAQOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint heads, uint units_count, uint input_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- activation = None; //--- if(!cF.Init(0, 0, OpenCL, 3 * window, window, 8, fmax((int)input_units - 2, 1), optimization_type, batch)) ReturnFalse; cF.SetActivationFunction(None); if(!cNormV.Init(8, 1, OpenCL, fmax((int)input_units - 2, 1) * 8, batch, optimization_type)) ReturnFalse; cNormV.SetActivationFunction(None); if(!cWv.Init(units_count * window_out, 2, OpenCL, 8, optimization_type, batch)) ReturnFalse; cWv.SetActivationFunction(SIGMOID); if(!cQd.Init(0, 4, OpenCL, units_count * window_out, optimization_type, batch)) ReturnFalse; cQd.SetActivationFunction(None); if(!cDQd.Init(0, 5, OpenCL, window_out, 3, heads, units_count, 3, optimization_type, batch)) ReturnFalse; cDQd.SetActivationFunction(None); //--- if(Output != cDQd.getOutput()) { Output.BufferFree(); DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { Gradient.BufferFree(); DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronFAQOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- if(!cF.FeedForward(NeuronOCL)) ReturnFalse; if(!cNormV.FeedForward(GetPointer(cF))) ReturnFalse; if(!cWv.FeedForward(GetPointer(cNormV))) ReturnFalse; if(!cQd.FeedForward(GetPointer(cWv))) ReturnFalse; if(!cDQd.FeedForward(GetPointer(cQd))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronFAQOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cDQd.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cWv.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cWv))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cF))) ReturnFalse; if(!cF.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronFAQOCL trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cDQd.UpdateInputWeights(GetPointer(cQd))) ReturnFalse; if(!cQd.UpdateInputWeights(GetPointer(cWv))) ReturnFalse; if(!cWv.UpdateInputWeights(GetPointer(cNormV))) ReturnFalse; if(!cNormV.UpdateInputWeights(GetPointer(cF))) ReturnFalse; if(!cF.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronFAQOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cF.Save(file_handle)) ReturnFalse; if(!cNormV.Save(file_handle)) ReturnFalse; if(!cWv.Save(file_handle)) ReturnFalse; if(!cQd.Save(file_handle)) ReturnFalse; if(!cDQd.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronFAQOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cF))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNormV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cWv))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQd))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cDQd))) ReturnFalse; //--- if(Output != cDQd.getOutput()) { DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronFAQOCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cF.WeightsUpdate(source, tau)) ReturnFalse; if(!cNormV.WeightsUpdate(source, tau)) ReturnFalse; if(!cWv.WeightsUpdate(source, tau)) ReturnFalse; if(!cQd.WeightsUpdate(source, tau)) ReturnFalse; if(!cDQd.WeightsUpdate(source, tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronFAQOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronFAQOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cF.SetOpenCL(OpenCL); cNormV.SetOpenCL(OpenCL); cWv.SetOpenCL(OpenCL); cQd.SetOpenCL(OpenCL); cDQd.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCrossAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iWindow_K = fmax(window_k, 1); iUnits_K = fmax(units_k, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iWindow_K, iWindow_K, 2 * iWindowKey * iHeads, iUnits_K, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); if(!Transpose.Init(0, 0, OpenCL, iUnits, iWindow, optimization_type, batch)) ReturnFalse; Transpose.SetActivationFunction(None); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[0].SetActivationFunction(LReLU); if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[1].SetActivationFunction(None); //--- Gradient.BufferFree(); DeleteObj(Gradient); Gradient = FF[1].getGradient(); SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Reports checked transfer support for exact CrossAttention only. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::SupportsOpenCLChecked(void) const { //--- Validate the CrossAttention geometry and its rectangular raw-score owner. if(Type() != defNeuronCrossAttenOCL || CheckPointer(OpenCL) == POINTER_INVALID || ScoreIndex < 0 || iWindow == 0 || iWindowKey == 0 || iHeads == 0 || iUnits == 0 || iWindow_K == 0 || iUnits_K == 0 || iUnits > uint(INT_MAX) / iUnits_K || iWindow_K > uint(INT_MAX) / iUnits_K) return(false); const uint score_count = iUnits * iUnits_K; if(score_count > uint(INT_MAX) / iHeads) return(false); const uint score_total = score_count * iHeads; if(score_total > UINT_MAX / sizeof(float) || OpenCL.GetBufferSize(ScoreIndex) != score_total * sizeof(float)) return(false); //--- Require the optional context base owner and its fixed rectangular buffers. if(cContext != NULL) { if(CheckPointer(cContext) == POINTER_INVALID || cContext.Type() != defNeuronBaseOCL || !cContext.SupportsOpenCLChecked()) return(false); const uint context_total = iWindow_K * iUnits_K; CBufferFloat *context_output = cContext.getOutput(); CBufferFloat *context_gradient = cContext.getGradient(); if(CheckPointer(context_output) == POINTER_INVALID || CheckPointer(context_gradient) == POINTER_INVALID || context_output.Total() != context_total || context_gradient.Total() != context_total) return(false); } //--- Require every nested owner before any checked transfer mutates it. return(Q_Embedding.SupportsOpenCLChecked() && KV_Embedding.SupportsOpenCLChecked() && Transpose.SupportsOpenCLChecked() && MHAttentionOut.SupportsOpenCLChecked() && W0.SupportsOpenCLChecked() && AttentionOut.SupportsOpenCLChecked() && FF[0].SupportsOpenCLChecked() && FF[1].SupportsOpenCLChecked()); } //+------------------------------------------------------------------+ //| Transfers CrossAttention context before the inherited MH2 tree. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and complete CrossAttention ownership contract. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or CrossAttention tree"); //--- Transfer the optional context before the inherited child sequence. if(cContext != NULL && !cContext.SetOpenCLChecked(obj)) ReturnFalseEx("CrossAttention context"); //--- Finalize through MH2, preserving its child order and base-last publication. return(CNeuronMH2AttentionOCL::SetOpenCLChecked(obj)); } //+------------------------------------------------------------------+ //| Handles the CNeuronCrossAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits_K/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits_K, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCrossAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { //--- Validate the fixed CrossAttention context shape. if(!Context || iWindow_K == 0 || iUnits_K == 0 || iWindow_K > UINT_MAX / iUnits_K) ReturnFalse; const uint context_total = iWindow_K * iUnits_K; if(Context.Total() != context_total) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, context_total, optimization, iBatch)) ReturnFalse; } //--- Resolve the cContext-owned buffers before temporary binding. CBufferFloat *inside = cContext.getOutput(); CBufferFloat *gradient = cContext.getGradient(); if(!inside || !gradient || inside.Total() != context_total || gradient.Total() != context_total) ReturnFalse; if(Context.GetIndex() >= 0) { //--- The caller owns Context; bind it only for this synchronous call. if(!cContext.SetOutput(Context, false)) ReturnFalse; const bool calculated = feedForward(NeuronOCL, cContext); const bool restored = cContext.SetOutput(inside, false); if(!restored || !calculated) ReturnFalse; return(true); } //--- Copy host Context into cContext-owned output. if(!inside.AssignArray(Context)) ReturnFalse; if(!inside.BufferWrite()) ReturnFalse; //--- Run forward with cContext's owned host copy. return(feedForward(NeuronOCL, cContext)); } //+------------------------------------------------------------------+ //| Runs the CNeuronCrossAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!KV_Embedding.FeedForward(Context)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronCrossAttention attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits_K) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!Q_Embedding.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCrossAttention layer. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL* prevLayer, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { //--- Validate the fixed CrossAttention context shape and caller buffers. if(!SecondInput || !SecondGradient || iWindow_K == 0 || iUnits_K == 0 || iWindow_K > UINT_MAX / iUnits_K) ReturnFalse; const uint context_total = iWindow_K * iUnits_K; if(SecondInput.Total() != context_total || SecondGradient.Total() != context_total) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, context_total, optimization, iBatch)) ReturnFalse; } CBufferFloat *inside_output = cContext.getOutput(); CBufferFloat *inside_gradient = cContext.getGradient(); if(!inside_output || !inside_gradient || inside_output.Total() != context_total || inside_gradient.Total() != context_total) ReturnFalse; cContext.SetActivationFunction(SecondActivation); //--- Prepare cContext output for the supplied input. bool output_bound = false; if(SecondInput.GetIndex() >= 0) { //--- The supplied input remains owned by the caller. if(!cContext.SetOutput(SecondInput, false)) ReturnFalse; output_bound = true; } else { if(!inside_output.AssignArray(SecondInput)) ReturnFalse; if(!inside_output.BufferWrite()) ReturnFalse; } if(SecondGradient.GetIndex() >= 0) { //--- The supplied gradient remains owned by the caller. if(!cContext.SetGradient(SecondGradient, false)) { if(output_bound && !cContext.SetOutput(inside_output, false)) ReturnFalse; ReturnFalse; } const bool calculated = calcInputGradients(prevLayer, cContext); const bool gradient_restored = cContext.SetGradient(inside_gradient, false); const bool output_restored = !output_bound || cContext.SetOutput(inside_output, false); if(!gradient_restored || !output_restored || !calculated) ReturnFalse; } else { const bool calculated = calcInputGradients(prevLayer, cContext); const bool output_restored = !output_bound || cContext.SetOutput(inside_output, false); if(!output_restored || !calculated) ReturnFalse; if(cContext.getGradient().GetData(SecondGradient) <= 0) ReturnFalse; } //--- Both caller-owned bindings are restored before return. return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCrossAttention layer. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL* prevLayer, CNeuronBaseOCL* Context) { if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Context.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCrossAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { //--- Validate the fixed CrossAttention context shape. if(!Context || iWindow_K == 0 || iUnits_K == 0 || iWindow_K > UINT_MAX / iUnits_K) ReturnFalse; const uint context_total = iWindow_K * iUnits_K; if(Context.Total() != context_total) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, context_total, optimization, iBatch)) ReturnFalse; } //--- Resolve the cContext-owned buffers before temporary binding. CBufferFloat *inside = cContext.getOutput(); CBufferFloat *gradient = cContext.getGradient(); if(!inside || !gradient || inside.Total() != context_total || gradient.Total() != context_total) ReturnFalse; if(Context.GetIndex() >= 0) { //--- The caller owns Context; bind it only for this synchronous call. if(!cContext.SetOutput(Context, false)) ReturnFalse; const bool updated = updateInputWeights(NeuronOCL, cContext); const bool restored = cContext.SetOutput(inside, false); if(!restored || !updated) ReturnFalse; return(true); } //--- Copy host Context into cContext-owned output. if(!inside.AssignArray(Context)) ReturnFalse; if(!inside.BufferWrite()) ReturnFalse; //--- Update weights from cContext's owned host copy. return(updateInputWeights(NeuronOCL, cContext)); } //+------------------------------------------------------------------+ //| Updates the CNeuronCrossAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CNeuronBaseOCL* Context) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(Context)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Exposes the exact compact-policy geometry and validates | //| children. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::GetDPCCPolicyGeometry(uint &query_window, uint &attention_dimension, uint &heads, uint &query_units, uint &context_window, uint &context_units) const { query_window = 0; attention_dimension = 0; heads = 0; query_units = 0; context_window = 0; context_units = 0; if(Type() != defNeuronCrossAttenOCL || iWindow == 0 || iWindowKey == 0 || iHeads == 0 || iUnits == 0 || iWindow_K == 0 || iUnits_K == 0 || iWindow > uint(INT_MAX) / iUnits || iWindow_K > uint(INT_MAX) / iUnits_K || iWindowKey > UINT_MAX / iHeads || iWindow > UINT_MAX / 4) return(false); const uint projection = iWindowKey * iHeads; if(projection > UINT_MAX / 2 || Neurons() != (int)(iWindow * iUnits) || Q_Embedding.Type() != defNeuronConvOCL || KV_Embedding.Type() != defNeuronConvOCL || W0.Type() != defNeuronConvOCL || FF[0].Type() != defNeuronConvOCL || FF[1].Type() != defNeuronConvOCL || Q_Embedding.Activation() != None || KV_Embedding.Activation() != None || W0.Activation() != None || FF[0].Activation() != LReLU || FF[1].Activation() != None) return(false); if(Q_Embedding.GetWindow() != iWindow || Q_Embedding.GetInputStride() != iWindow || Q_Embedding.GetFilters() != projection || Q_Embedding.GetVariables() != 1 || Q_Embedding.GetUnits() != iUnits || KV_Embedding.GetWindow() != iWindow_K || KV_Embedding.GetInputStride() != iWindow_K || KV_Embedding.GetFilters() != 2 * projection || KV_Embedding.GetVariables() != 1 || KV_Embedding.GetUnits() != iUnits_K || W0.GetWindow() != projection || W0.GetInputStride() != projection || W0.GetFilters() != iWindow || W0.GetVariables() != 1 || W0.GetUnits() != iUnits || FF[0].GetWindow() != iWindow || FF[0].GetInputStride() != iWindow || FF[0].GetFilters() != 4 * iWindow || FF[0].GetVariables() != 1 || FF[0].GetUnits() != iUnits || FF[1].GetWindow() != 4 * iWindow || FF[1].GetInputStride() != 4 * iWindow || FF[1].GetFilters() != iWindow || FF[1].GetVariables() != 1 || FF[1].GetUnits() != iUnits) return(false); query_window = iWindow; attention_dimension = iWindowKey; heads = iHeads; query_units = iUnits; context_window = iWindow_K; context_units = iUnits_K; return(true); } //+------------------------------------------------------------------+ //| Saves only CrossAttention weights, Adam state and exact | //| geometry. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::SaveDPCCPolicyTrainingStateOnly(const int file_handle) { uint query_window = 0; uint attention_dimension = 0; uint heads = 0; uint query_units = 0; uint context_window = 0; uint context_units = 0; if(file_handle == INVALID_HANDLE || !GetDPCCPolicyGeometry(query_window, attention_dimension, heads, query_units, context_window, context_units) || !SaveDPCCPolicyTrainingBaseOnly(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, 1, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, 5, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)query_window, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)attention_dimension, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)heads, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)query_units, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)context_window, INT_VALUE) != INT_VALUE || FileWriteInteger(file_handle, (int)context_units, INT_VALUE) != INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, Q_Embedding.Type(), INT_VALUE) != INT_VALUE || !Q_Embedding.SaveDPCCPolicyTrainingStateOnly(file_handle) || FileWriteInteger(file_handle, KV_Embedding.Type(), INT_VALUE) != INT_VALUE || !KV_Embedding.SaveDPCCPolicyTrainingStateOnly(file_handle) || FileWriteInteger(file_handle, W0.Type(), INT_VALUE) != INT_VALUE || !W0.SaveDPCCPolicyTrainingStateOnly(file_handle) || FileWriteInteger(file_handle, FF[0].Type(), INT_VALUE) != INT_VALUE || !FF[0].SaveDPCCPolicyTrainingStateOnly(file_handle) || FileWriteInteger(file_handle, FF[1].Type(), INT_VALUE) != INT_VALUE || !FF[1].SaveDPCCPolicyTrainingStateOnly(file_handle)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Validates/restores compact CrossAttention training state only. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::LoadDPCCPolicyTrainingStateOnly(const int file_handle, const bool apply) { uint query_window = 0; uint attention_dimension = 0; uint heads = 0; uint query_units = 0; uint context_window = 0; uint context_units = 0; if(file_handle == INVALID_HANDLE || !GetDPCCPolicyGeometry(query_window, attention_dimension, heads, query_units, context_window, context_units) || !LoadDPCCPolicyTrainingBaseOnly(file_handle, apply)) ReturnFalse; ResetLastError(); if(FileReadInteger(file_handle, INT_VALUE) != 1 || FileReadInteger(file_handle, INT_VALUE) != 5 || FileReadInteger(file_handle, INT_VALUE) != (int)query_window || FileReadInteger(file_handle, INT_VALUE) != (int)attention_dimension || FileReadInteger(file_handle, INT_VALUE) != (int)heads || FileReadInteger(file_handle, INT_VALUE) != (int)query_units || FileReadInteger(file_handle, INT_VALUE) != (int)context_window || FileReadInteger(file_handle, INT_VALUE) != (int)context_units || GetLastError() != 0) ReturnFalse; if(FileReadInteger(file_handle, INT_VALUE) != Q_Embedding.Type() || !Q_Embedding.LoadDPCCPolicyTrainingStateOnly(file_handle, apply) || FileReadInteger(file_handle, INT_VALUE) != KV_Embedding.Type() || !KV_Embedding.LoadDPCCPolicyTrainingStateOnly(file_handle, apply) || FileReadInteger(file_handle, INT_VALUE) != W0.Type() || !W0.LoadDPCCPolicyTrainingStateOnly(file_handle, apply) || FileReadInteger(file_handle, INT_VALUE) != FF[0].Type() || !FF[0].LoadDPCCPolicyTrainingStateOnly(file_handle, apply) || FileReadInteger(file_handle, INT_VALUE) != FF[1].Type() || !FF[1].LoadDPCCPolicyTrainingStateOnly(file_handle, apply) || GetLastError() != 0) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Fingerprints compact CrossAttention weights, Adam and geometry. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::AppendDPCCPolicyTrainingFingerprint(ulong &fingerprint) { uint query_window = 0; uint attention_dimension = 0; uint heads = 0; uint query_units = 0; uint context_window = 0; uint context_units = 0; if(!GetDPCCPolicyGeometry(query_window, attention_dimension, heads, query_units, context_window, context_units) || !AppendDPCCPolicyTrainingBaseFingerprint(fingerprint)) ReturnFalse; const ulong identity[] = {ulong(1), ulong(5), ulong(query_window), ulong(attention_dimension), ulong(heads), ulong(query_units), ulong(context_window), ulong(context_units) }; for(int i = 0; i < ArraySize(identity); i++) fingerprint = (fingerprint ^ identity[i]) * ulong(1099511628211); if(!Q_Embedding.AppendDPCCPolicyTrainingFingerprint(fingerprint) || !KV_Embedding.AppendDPCCPolicyTrainingFingerprint(fingerprint) || !W0.AppendDPCCPolicyTrainingFingerprint(fingerprint) || !FF[0].AppendDPCCPolicyTrainingFingerprint(fingerprint) || !FF[1].AppendDPCCPolicyTrainingFingerprint(fingerprint)) ReturnFalse; return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCrossAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Save(const int file_handle) { if(!CNeuronMH2AttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow_K) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits_K) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCrossAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Load(const int file_handle) { if(!CNeuronMH2AttentionOCL::Load(file_handle)) ReturnFalse; iWindow_K = (uint)FileReadInteger(file_handle); iUnits_K = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCrossAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronCrossAttention::SetOpenCL(COpenCLMy *obj) { if(OpenCL == obj) return; COpenCLMy *previous = OpenCL; if(!!previous && ScoreIndex >= 0) previous.BufferFree(ScoreIndex); ScoreIndex = INVALID_HANDLE; CNeuronBaseOCL::SetOpenCL(obj); if(!OpenCL) return; Q_Embedding.SetOpenCL(OpenCL); KV_Embedding.SetOpenCL(OpenCL); if(Transpose.Neurons() > 0) Transpose.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); W0.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL); FF[0].SetOpenCL(OpenCL); FF[1].SetOpenCL(OpenCL); if(!!cContext) cContext.SetOpenCL(OpenCL); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGTE layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGTE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!cQKV.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * 3 * iHeads, iUnits, optimization, iBatch)) ReturnFalse; if(!cSoftMax.Init(0, 1, OpenCL, iWindowKey * 3 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(3 * iHeads * iUnits); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!cMHAttentionOut.Init(0, 2, OpenCL, iWindowKey * 2 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cW0.Init(0, 3, OpenCL, 2 * iWindowKey * iHeads, 2 * iWindowKey * iHeads, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cAttentionOut.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Init(0, 5 + i, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; if(!cFF[i].Init(0, 7 + i, OpenCL, (i == 0 ? iWindow : 4 * iWindow), (i == 0 ? iWindow : 4 * iWindow), (i == 1 ? iWindow : 4 * iWindow), iUnits, optimization, iBatch)) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGTE feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronGTE::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cSoftMax.FeedForward(GetPointer(cQKV))) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cMHAttentionOut))) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), iWindow, true)) ReturnFalse; if(!cGraphConv[0].FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].FeedForward(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].FeedForward(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].FeedForward(GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormalize(cAttentionOut.getOutput(), cFF[1].getOutput(), Output, iWindow, true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronGTE | //| layer. | //+------------------------------------------------------------------+ bool CNeuronGTE::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cFF[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cGraphConv[1]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormalize(cAttentionOut.getGradient(), Gradient, cW0.getGradient(), iWindow, false)) ReturnFalse; if(!cW0.CalcHiddenGradients((CObject *)GetPointer(cMHAttentionOut))) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!cSoftMax.CalcHiddenGradients((CObject *)GetPointer(cQKV))) ReturnFalse; if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGTE trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronGTE::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cMHAttentionOut))) ReturnFalse; if(!cGraphConv[0].UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].UpdateInputWeights(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].UpdateInputWeights(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].UpdateInputWeights(GetPointer(cFF[0]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronGTE weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronGTE::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronGTE *Source = source; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].WeightsUpdate(GetPointer(Source.cGraphConv[i]), tau)) ReturnFalse; if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronGTE attention out operation. | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_GTEFeedForward, def_k_gteff_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEFeedForward, def_k_gteff_score, ScoreIndex) setBuffer(def_k_GTEFeedForward, def_k_gteff_out, cAttentionOut.getOutputIndex()) setArgument(def_k_GTEFeedForward, def_k_gteff_dimension, (int)iWindowKey) kernelExecuteLoc(def_k_GTEFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronGTE attention inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_scores, ScoreIndex) setBuffer(def_k_GTEInsideGradients, def_k_gteig_gradient, cAttentionOut.getGradientIndex()) kernelExecute(def_k_GTEFeedForward, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGTE GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronGTE::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- if(!!OpenCL) if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) { cGraphConv[i].SetOpenCL(OpenCL); cFF[i].SetOpenCL(OpenCL); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Saves the CNeuronGTE state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGTE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Save(file_handle)) ReturnFalse; if(!cFF[i].Save(file_handle)) ReturnFalse; } //--- if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGTE state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronGTE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cSoftMax))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cMHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cAttentionOut))) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, GetPointer(cGraphConv[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; } //--- iHeads = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronGTE layer. | //+------------------------------------------------------------------+ void CNeuronGTE::TrainMode(bool flag) { //--- CNeuronBaseOCL::TrainMode(flag); cQKV.TrainMode(flag); cSoftMax.TrainMode(flag); cMHAttentionOut.TrainMode(flag); cW0.TrainMode(flag); cAttentionOut.TrainMode(flag); for(int i = 0; i < 2; i++) { cGraphConv[i].TrainMode(flag); cFF[i].TrainMode(flag); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Initializes the CCCMREncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CCCMREncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!cInput.Init(0, 0, OpenCL, window, window, 32, count, optimization, iBatch)) ReturnFalse; if(!cNorm.Init(0, 1, OpenCL, 32 * count, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(LReLU); //--- if(!cResidual[0].Init(0, 2, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[1].Init(0, 3, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[2].Init(0, 4, OpenCL, 32, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[3].Init(0, 5, OpenCL, 64, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[4].Init(0, 6, OpenCL, 64, 128, count, optimization, iBatch)) ReturnFalse; if(!cResidual[5].Init(0, 7, OpenCL, 128, 128, count, optimization, iBatch)) ReturnFalse; //--- if(!cOutput.Init(0, 8, OpenCL, 128, 128, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CCCMREncoder feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CCCMREncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cInput.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].FeedForward(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].FeedForward(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.FeedForward(GetPointer(cResidual[5]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CCCMREncoder | //| layer. | //+------------------------------------------------------------------+ bool CCCMREncoder::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cOutput.CalcHiddenGradients((CObject *)GetPointer(cResidual[5]))) ReturnFalse; for(int i = 5; i > 0; i--) if(!cResidual[i].CalcHiddenGradients((CObject *)GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cResidual[0].CalcHiddenGradients((CObject *)GetPointer(cNorm))) ReturnFalse; if(!cNorm.CalcHiddenGradients((CObject *)GetPointer(cInput))) ReturnFalse; if(!cInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CCCMREncoder trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CCCMREncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].UpdateInputWeights(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].UpdateInputWeights(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.UpdateInputWeights(GetPointer(cResidual[5]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CCCMREncoder weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CCCMREncoder::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CCCMREncoder *Source = source; if(!cInput.WeightsUpdate(GetPointer(Source.cInput), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(GetPointer(Source.cNorm), tau)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].WeightsUpdate(GetPointer(Source.cResidual[i]), tau)) ReturnFalse; if(!cOutput.WeightsUpdate(GetPointer(Source.cOutput), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CCCMREncoder state to the file handle. | //+------------------------------------------------------------------+ bool CCCMREncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].Save(file_handle)) ReturnFalse; if(!cOutput.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CCCMREncoder state from the file handle. | //+------------------------------------------------------------------+ bool CCCMREncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNorm))) ReturnFalse; for(int i = 0; i < 6; i++) if(!LoadInsideLayer(file_handle, GetPointer(cResidual[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cOutput))) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CCCMREncoder GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CCCMREncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInput.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cOutput.SetOpenCL(OpenCL); for(int i = 0; i < 6; i++) cResidual[i].SetOpenCL(OpenCL); //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } } //+------------------------------------------------------------------+ //| Sets the training flag of the CCCMREncoder layer. | //+------------------------------------------------------------------+ void CCCMREncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cInput.TrainMode(flag); cNorm.TrainMode(flag); cOutput.TrainMode(flag); for(int i = 0; i < 6; i++) cResidual[i].TrainMode(flag); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCrossXCiTOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window1, uint window2, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronXCiTOCL::Init(numOutputs, myIndex, open_cl, window1, lpi_window, heads, units_count, layers, optimization_type, batch)) ReturnFalse; //--- Cross XCA iWindow2 = fmax(window2, 1); //--- Size of V tensor uint num = iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of V tensor uint v_weights = (iWindow2 + 1) * iWindowKey * iHeads; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize V tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cValue.Add(temp)) ReturnFalse; //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize V weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(v_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < v_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(v_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCrossXCiTOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Motion) { if(!NeuronOCL || !Motion) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *v = cValue.At(i * 2); if(IsStopped() || !ConvolutionForward(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), Motion, v, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !Concat(qkv, v, cConcat.At(2 * i), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(cConcat.At(2 * i), temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCrossXCiTOCL layer. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer, CNeuronBaseOCL* Motion) { if(!prevLayer || !Motion) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(cConcat.At(i * 2), cConcat.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; if(IsStopped() || !DeConcat(QKV_Tensors.At(i * 2 + 1), cValue.At(i * 2 + 1), cConcat.At(i * 2 + 1), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) out_grad = temp; if(i == iLayers - 1) { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), Motion.getGradient(), iWindow, iWindowKey * iHeads, None)) ReturnFalse; } else { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), GetPointer(TempBuffer), iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(TempBuffer), Motion.getGradient(), Motion.getGradient(), iWindow2, false)) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCrossXCiTOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Motion) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 2 * iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), cValue.At(l * 2 + 1), inputs, (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronCrossXCiTOCL weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronCrossXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.cV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCrossXCiTOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronCrossXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronXCiTOCL::SetOpenCL(obj); cConcat.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cV_Weights.SetOpenCL(OpenCL); TempBuffer.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronCrossXCiTOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Save(const int file_handle) { if(!CNeuronXCiTOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, (int)iWindow2, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cValue.Save(file_handle) || !cV_Weights.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCrossXCiTOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Load(const int file_handle) { if(!CNeuronXCiTOCL::Load(file_handle)) ReturnFalse; //--- Loading constant iWindow2 = (uint)FileReadInteger(file_handle); //--- Loading objects if(!cValue.Load(file_handle) || !cV_Weights.Load(file_handle)) ReturnFalse; //--- //--- Size of V tensor uint num = iWindowKey * iHeads * iUnits; //--- CBufferFloat *temp = NULL; for(uint i = 0; i < iLayers; i++) { for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCCMROCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!FeatureExtractor.Init(0, 0, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!PrevFeatures.Init(0, 1, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(!Motion.Init(0, 2, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); //--- if(!Temp.Init(0, 3, OpenCL, window * count, optimization, iBatch)) ReturnFalse; if(!LocalContext.Init(0, 4, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!GlobalContext.Init(0, 5, OpenCL, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!MotionContext.Init(0, 6, OpenCL, 16, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!RecurentUnit.Init(0, 7, OpenCL, 16 * count, optimization, iBatch) || !RecurentUnit.SetInputs(16 * count)) ReturnFalse; if(!UpScale.Init(0, 8, OpenCL, 16, 16, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCCMROCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Delta Features if(!SumAndNormalize(FeatureExtractor.getOutput(), FeatureExtractor.getOutput(), PrevFeatures.getOutput(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!FeatureExtractor.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormalize(FeatureExtractor.getOutput(), PrevFeatures.getOutput(), Motion.getOutput(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- Context if(Temp.getOutputIndex() != NeuronOCL.getOutputIndex()) { CBufferFloat* temp = Temp.getOutput(); temp.BufferSet(NeuronOCL.getOutputIndex()); Temp.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); } if(!LocalContext.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalContext.FeedForward(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.FeedForward(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; //--- Flow if(!RecurentUnit.FeedForward(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.FeedForward(GetPointer(RecurentUnit))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronCCMROCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!UpScale.CalcHiddenGradients((CObject *)GetPointer(RecurentUnit))) ReturnFalse; if(!RecurentUnit.CalcHiddenGradients((CObject *)GetPointer(MotionContext))) ReturnFalse; //--- if(!MotionContext.calcInputGradients(GetPointer(GlobalContext), GetPointer(Motion))) ReturnFalse; if(!GlobalContext.CalcHiddenGradients((CObject *)GetPointer(LocalContext))) ReturnFalse; if(!LocalContext.CalcHiddenGradients((CObject *)GetPointer(Temp))) ReturnFalse; //--- if(!FeatureExtractor.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), Temp.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCCMROCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!FeatureExtractor.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!LocalContext.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!GlobalContext.UpdateInputWeights(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.UpdateInputWeights(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; if(!RecurentUnit.UpdateInputWeights(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.UpdateInputWeights(GetPointer(RecurentUnit))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronCCMROCL weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronCCMROCL *Source = source; if(!FeatureExtractor.WeightsUpdate(GetPointer(Source.FeatureExtractor), tau)) ReturnFalse; if(!LocalContext.WeightsUpdate(GetPointer(Source.LocalContext), tau)) ReturnFalse; if(!GlobalContext.WeightsUpdate(GetPointer(Source.GlobalContext), tau)) ReturnFalse; if(!MotionContext.WeightsUpdate(GetPointer(Source.MotionContext), tau)) ReturnFalse; if(!RecurentUnit.WeightsUpdate(GetPointer(Source.RecurentUnit), tau)) ReturnFalse; if(!UpScale.WeightsUpdate(GetPointer(Source.UpScale), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronCCMROCL execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Clear(void) { if(!RecurentUnit.Clear()) ReturnFalse; //--- CBufferFloat *temp = FeatureExtractor.getOutput(); temp.BufferInit(temp.Total(), 0); if(!temp.BufferWrite()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronCCMROCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!FeatureExtractor.Save(file_handle)) ReturnFalse; if(!PrevFeatures.Save(file_handle)) ReturnFalse; if(!Motion.Save(file_handle)) ReturnFalse; if(!Temp.Save(file_handle)) ReturnFalse; if(!LocalContext.Save(file_handle)) ReturnFalse; if(!GlobalContext.Save(file_handle)) ReturnFalse; if(!MotionContext.Save(file_handle)) ReturnFalse; if(!RecurentUnit.Save(file_handle)) ReturnFalse; if(!UpScale.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronCCMROCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(FeatureExtractor))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(PrevFeatures))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Motion))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Temp))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(LocalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(GlobalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MotionContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(RecurentUnit))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(UpScale))) ReturnFalse; //--- if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronCCMROCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronCCMROCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); FeatureExtractor.SetOpenCL(OpenCL); PrevFeatures.SetOpenCL(OpenCL); Motion.SetOpenCL(OpenCL); Temp.SetOpenCL(OpenCL); LocalContext.SetOpenCL(OpenCL); GlobalContext.SetOpenCL(OpenCL); MotionContext.SetOpenCL(OpenCL); RecurentUnit.SetOpenCL(OpenCL); UpScale.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronCCMROCL layer. | //+------------------------------------------------------------------+ void CNeuronCCMROCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); FeatureExtractor.TrainMode(bTrain); PrevFeatures.TrainMode(bTrain); Motion.TrainMode(bTrain); Temp.TrainMode(bTrain); LocalContext.TrainMode(bTrain); GlobalContext.TrainMode(bTrain); MotionContext.TrainMode(bTrain); RecurentUnit.TrainMode(bTrain); UpScale.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronNODEOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint variables, uint lenth, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * variables * lenth, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iVariables = variables; iLenth = lenth; //--- uint mult = 2; uint weights = (iDimension + 2) * iDimension * iVariables; //--- if(ArrayResize(iBuffersK, 18) < 18) ReturnFalse; if(ArrayResize(iInputsK, 18) < 18) ReturnFalse; if(ArrayResize(iMeadl, 12) < 12) ReturnFalse; CBufferFloat *temp = NULL; //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iInputsK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Meadl Output and Gradient buffers iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- Initilize Weights for(int i = 0; i < 2; i++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(weights)) ReturnFalse; float k = (float)(1 / sqrt(iDimension + 2)); for(uint w = 0; w < weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; //--- for(uint d = 0; d < 2; d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; } } //--- Constants //--- Alpha { float temp_ar[] = {0, 0.2f, 0.3f, 0.8f, 8.0f / 9, 1, 1}; if(!cAlpha.AssignArray(temp_ar)) ReturnFalse; if(!cAlpha.BufferCreate(OpenCL)) ReturnFalse; } //--- Beta K1 { float temp_ar[] = {0, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K2 { float temp_ar[] = {0.2f, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K3 { float temp_ar[] = {3.0f / 40, 9.0f / 40, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K4 { float temp_ar[] = {44.0f / 44, -56.0f / 15, 32.0f / 9, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K5 { float temp_ar[] = {19372.0f / 6561, -25360 / 2187.0f, 64448 / 6561.0f, -212.0f / 729, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K6 { float temp_ar[] = {9017 / 3168.0f, -355 / 33.0f, 46732 / 5247.0f, 49.0f / 176, -5103.0f / 18656, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Yt+1 { float temp_ar[] = {35.0f / 384, 0, 500.0f / 1113, 125.0f / 192, -2187.0f / 6784, 11.0f / 84}; if(!cSolution.AssignArray(temp_ar)) ReturnFalse; if(!cSolution.BufferCreate(OpenCL)) ReturnFalse; } //--- if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate input k operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputK(CBufferFloat* inputs, int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- if(k == 0) { if(iInputsK[k] != inputs.GetIndex()) { OpenCL.BufferFree(iInputsK[k]); iInputsK[k] = inputs.GetIndex(); } return(true); } //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k]) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat *)cBeta.At(k)).BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate k buffer operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBuffer(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iInputsK[k]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iMeadl[k * 2]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, float(cAlpha.At(k))) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(LReLU)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iBuffersK[k]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, cAlpha.At(k)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(None)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //-- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate output operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutput(CBufferFloat* inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, Output.GetIndex()) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- if(!SumAndNormalize(Output, inputs, Output, iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronNODEOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { for(int k = 0; k < 6; k++) { if(!CalculateInputK(NeuronOCL.getOutput(), k)) ReturnFalse; if(!CalculateKBuffer(k)) ReturnFalse; } //--- return(CalculateOutput(NeuronOCL.getOutput())); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate output gradient operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutputGradient(CBufferFloat* inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[6]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[7]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[8]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[9]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[10]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[11]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, Gradient.GetIndex()) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!Gradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate k buffer gradient | //| operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBufferGradient(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iMeadl[k * 2 + 1]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iBuffersK[k + 6]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(LReLU)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iInputsK[k]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iInputsK[k + 12]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iMeadl[k * 2 + 1]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(None)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //-- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNODEOCL calculate input k gradient operation. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputKGradient(CBufferFloat* inputs, int k) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[12]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[13]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[14]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[15]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[16]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[17]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k + 6]) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat *)cBeta.At(k)).BufferRead()) ReturnFalse; #endif //--- for(int i = k - 1; i >= 0; i--) { float mult = 1.0f / (i == (k - 1) ? 6 - k : 1); uint global_work_offset[1] = {0}; uint global_work_size[1] = {iLenth * iVariables}; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, iBuffersK[k + 6]) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, iBuffersK[k + 12]) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, iBuffersK[k + 6]) setArgument(def_k_MatrixSum, def_k_sum_dimension, iDimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, mult) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, iBuffersK[k + 6]) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!inputs.BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronNODEOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!CalculateOutputGradient(prevLayer.getGradient())) ReturnFalse; for(int k = 5; k >= 0; k--) { if(!CalculateKBufferGradient(k)) ReturnFalse; if(!CalculateInputKGradient(GetPointer(cTemp), k)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getOutput(), iDimension, false, 0, 0, 0, 1.0f / (k == 0 ? 6 : 1))) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronNODEOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension + 2, iDimension, iVariables}; ResetLastError(); setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iInputsK[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iMeadl[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iInputsK[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iMeadl[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iInputsK[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iMeadl[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iInputsK[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iMeadl[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iInputsK[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iMeadl[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iInputsK[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iMeadl[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(1)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(2)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iMeadl[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iBuffersK[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iMeadl[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iBuffersK[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iMeadl[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iBuffersK[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iMeadl[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iBuffersK[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iMeadl[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iBuffersK[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iMeadl[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iBuffersK[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(4)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(5)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //-- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronNODEOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWeights.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iVariables), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iLenth), INT_VALUE) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronNODEOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cWeights.Load(file_handle)) ReturnFalse; cWeights.SetOpenCL(OpenCL); //--- iDimension = (int)FileReadInteger(file_handle); iVariables = (int)FileReadInteger(file_handle); iLenth = (int)FileReadInteger(file_handle); //--- CBufferFloat *temp = NULL; for(uint i = 0; i < 18; i++) { OpenCL.BufferFree(iBuffersK[i]); OpenCL.BufferFree(iInputsK[i]); //--- iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Output and Gradient buffers OpenCL.BufferFree(iMeadl[i]); iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- cTemp.BufferFree(); if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronNODEOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronNODEOCL::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- CNeuronBaseOCL::SetOpenCL(obj); cWeights.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); cAlpha.BufferCreate(OpenCL); cBeta.SetOpenCL(OpenCL); cSolution.BufferCreate(OpenCL); //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(i > 11) continue; iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); } //--- } //+------------------------------------------------------------------+ //| Initializes the CNeuronConformer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronConformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint variables, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * variables * units_count, optimization_type, batch)) ReturnFalse; if(!cQKV.Init(0, 0, OpenCL, window, window, 3 * window_key * heads, variables * units_count, optimization, iBatch)) ReturnFalse; //--- iWindow = int(fmax(window, 1)); iDimension = int(fmax(window_key, 1)); iHeads = int(fmax(heads, 1)); iVariables = int(fmax(variables, 1)); iCount = int(fmax(units_count, 1)); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cW0.Init(0, 3, OpenCL, iDimension * iHeads, iDimension * iHeads, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 3; i++) if(!cNODE[i].Init(0, 4 + i, OpenCL, iWindow, iVariables, iCount, optimization, iBatch)) ReturnFalse; //--- if(!cFF[0].Init(0, 7, OpenCL, iWindow, iWindow, 4 * iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cFF[1].Init(0, 8, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronConformer attention out operation. | //+------------------------------------------------------------------+ bool CNeuronConformer::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- Time Derivative { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_TimeDerivative, def_k_tdqkv, cQKV.getOutputIndex()) setBuffer(def_k_TimeDerivative, def_k_tddqkv, cdQKV.getOutputIndex()) setArgument(def_k_TimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_TimeDerivative, global_work_offset, global_work_size) #ifdef _DEBUG if(!cdQKV.getOutput().BufferRead()) ReturnFalse; #endif } //--- MH Attention Out { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iCount, iVariables}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_FeedForwardContAtt, def_k_caqkv, cQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cascore, iScore) setBuffer(def_k_FeedForwardContAtt, def_k_caout, cAttentionOut.getOutputIndex()) setArgument(def_k_FeedForwardContAtt, def_k_cadimension, int(iDimension)) setArgument(def_k_FeedForwardContAtt, def_k_caheads, int(iHeads)) kernelExecuteLoc(def_k_FeedForwardContAtt, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronConformer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronConformer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Generate Query, Key, Value if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //--- MH Continuas Attention if(!attentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cW0.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } if(!SumAndNormalize(prev.getOutput(), cW0.getOutput(), prev.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } if(!SumAndNormalize(prev.getOutput(), cNODE[2].getOutput(), getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronConformer attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronConformer::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- MH Attention Out Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv, cQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv_g, cQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv_g, cdQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcascore, iScore) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaout_g, cAttentionOut.getGradientIndex()) setArgument(def_k_HiddenGradientContAtt, def_k_hgcadimension, int(iDimension)) kernelExecute(def_k_HiddenGradientContAtt, global_work_offset, global_work_size) #ifdef _DEBUG if(!cAttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- Time Derivative Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HGTimeDerivative, def_k_tdqkv, cQKV.getGradientIndex()) setBuffer(def_k_HGTimeDerivative, def_k_tddqkv, cdQKV.getGradientIndex()) setArgument(def_k_HGTimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_HGTimeDerivative, global_work_offset, global_work_size) #ifdef _DEBUG if(!cdQKV.getGradient().BufferRead()) ReturnFalse; #endif } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronConformer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronConformer::calcInputGradients(CNeuronBaseOCL* prevLayer) { //--- Feed Forward Gradient if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cFF[1]))) ReturnFalse; if(!cNODE[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormalize(Gradient, cNODE[2].getGradient(), cNODE[2].getGradient(), iDimension, false)) ReturnFalse; //--- Neural ODE Gradient CNeuronBaseOCL *prev = cNODE[1].AsObject(); for(int i = 2; i > 0; i--) { if(!prev.CalcHiddenGradients(cNODE[i].AsObject())) ReturnFalse; prev = cNODE[i - 1].AsObject(); } if(!cW0.CalcHiddenGradients(prev.AsObject())) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), cNODE[2].getGradient(), cW0.getGradient(), iDimension, false)) ReturnFalse; //--- MH Attention Gradient if(!cAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; //--- Query, Key, Value Graddients if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iDimension, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronConformer weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronConformer::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronConformer *Source = source; //--- MH Attention if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronConformer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronConformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- MH Attention if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronConformer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronConformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) if(!cNODE[i].Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFF[i].Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronConformer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronConformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; for(int i = 0; i < 3; i++) if(!LoadInsideLayer(file_handle, GetPointer(cNODE[i]))) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; //--- iWindow = FileReadInteger(file_handle); iDimension = FileReadInteger(file_handle); iHeads = FileReadInteger(file_handle); iVariables = FileReadInteger(file_handle); iCount = FileReadInteger(file_handle); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(iScore >= 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronConformer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronConformer::SetOpenCL(COpenCLMy * obj) { if(obj == OpenCL) return; if(iScore > 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cdQKV.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); for(int i = 0; i < 3; i++) cNODE[i].SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cFF[i].SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); //--- if(getGradientIndex() != cFF[1].getGradientIndex()) SetGradientIndex(cFF[1].getGradientIndex()); } //+------------------------------------------------------------------+ //| Initializes the CNeuronClientOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint at_layers, uint count, uint & mlp[], ENUM_OPTIMIZATION optimization_type, uint batch) { uint mlp_layers = mlp.Size(); if(mlp_layers == 0) ReturnFalse; if(ArrayResize(cLinearModel, mlp_layers + 1) != (mlp_layers + 1)) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, mlp[mlp_layers - 1] * count, optimization_type, batch)) ReturnFalse; if(!cTransformerEncoder.Init(0, 0, OpenCL, window, window_key, heads, count, at_layers, optimization, iBatch)) ReturnFalse; if(!cInput.Init(0, 1, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- uint w = window; for(uint i = 0; i < mlp_layers; i++) { if(!cLinearModel[i].Init(0, i + 2, OpenCL, w, w, mlp[i], count, optimization, iBatch)) ReturnFalse; cLinearModel[i].SetActivationFunction(LReLU); w = mlp[i]; } if(!cLinearModel[mlp_layers].Init(0, mlp_layers + 2, OpenCL, 1, 1, 1, w * count, optimization, iBatch)) ReturnFalse; cLinearModel[mlp_layers].SetActivationFunction(TANH); if(!cProjection.Init(0, mlp_layers + 3, OpenCL, window, window, w, count, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); SetActivationFunction(TANH); //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[mlp_layers].SetGradient(Gradient)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronClientOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTransformerEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjection.FeedForward(GetPointer(cTransformerEncoder))) ReturnFalse; if(cInput.getOutputIndex() != NeuronOCL.getOutputIndex()) cInput.getOutput().BufferSet(NeuronOCL.getOutputIndex()); //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].FeedForward(neuron)) ReturnFalse; neuron = GetPointer(cLinearModel[i]); } //--- if(!SumAndNormalize(neuron.getOutput(), cProjection.getOutput(), Output, 1, false, 0, 0, 0, 0.5)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronClientOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cTransformerEncoder.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cTransformerEncoder.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *neuron = NULL; int total = (int)cLinearModel.Size() - 1; for(int i = total; i >= 0; i--) { neuron = (i > 0 ? cLinearModel[i - 1] : cInput).AsObject(); if(!neuron.CalcHiddenGradients(cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SumAndNormalize(neuron.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronClientOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cTransformerEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cProjection.UpdateInputWeights(cTransformerEncoder.AsObject())) ReturnFalse; //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = cLinearModel[i].AsObject(); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronClientOCL GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronClientOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cTransformerEncoder.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cInput.SetOpenCL(OpenCL); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronClientOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cTransformerEncoder.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; uint total = cLinearModel.Size(); if(FileWriteInteger(file_handle, int(total), INT_VALUE) != INT_VALUE) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronClientOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Load(const int file_handle) { ArrayFree(cLinearModel); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransformerEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInput.AsObject())) ReturnFalse; int total = FileReadInteger(file_handle); if(total <= 0) ReturnFalse; if(ArrayResize(cLinearModel, total) < total) ReturnFalse; for(int i = 0; i < total; i++) { cLinearModel[i].SetOpenCL(OpenCL); if(!LoadInsideLayer(file_handle, cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[total - 1].SetGradient(Gradient)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronClientOCL layer. | //+------------------------------------------------------------------+ void CNeuronClientOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cTransformerEncoder.TrainMode(bTrain); cProjection.TrainMode(bTrain); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Averages the CNeuronClientOCL weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronClientOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronClientOCL *sour = source; if(!cTransformerEncoder.WeightsUpdate(sour.cTransformerEncoder.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(sour.cProjection.AsObject(), tau)) ReturnFalse; //--- uint total = cLinearModel.Size(); if(total != sour.cLinearModel.Size()) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].WeightsUpdate(sour.cLinearModel[i].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUShapeAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, 4 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronUShapeAttention *temp = new CNeuronUShapeAttention(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, 2 * units_count, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window, window, 2 * units_count, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, 2 * units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, 2 * window, 2 * window, window, units_count, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUShapeAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), Output, 1, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronUShapeAttention layer. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), Gradient, prevLayer.getGradient(), 1, false)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUShapeAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUShapeAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronUShapeAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) { cAttention[i].SetOpenCL(OpenCL); cMergeSplit[i].SetOpenCL(OpenCL); } if(!!cNeck) cNeck.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronUShapeAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cAttention[i].Save(file_handle)) ReturnFalse; if(!cMergeSplit[i].Save(file_handle)) ReturnFalse; } if(!cNeck.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUShapeAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronUShapeAttention: cNeck = new CNeuronUShapeAttention(); if(!cNeck) ReturnFalse; break; case defNeuronConvOCL: cNeck = new CNeuronConvOCL(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronUShapeAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) return(true); //--- CNeuronUShapeAttention *temp = source; for(int i = 0; i < 2; i++) { if(!cAttention[i].WeightsUpdate(temp.cAttention[i].AsObject(), tau)) ReturnFalse; if(!cMergeSplit[i].WeightsUpdate(temp.cMergeSplit[i].AsObject(), tau)) ReturnFalse; } if(!cNeck.WeightsUpdate(temp.cNeck.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronComplexMLMHAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- //--- Size of QKV tensor uint num = 2 * 3 * iWindowKey * iHeads * iUnits; //--- Size of weights' matrix of QKV tenzor uint qkv_weights = 2 * 3 * (iWindow + 1) * iWindowKey * iHeads; //--- Size of Score tensor uint scores = 2 * iUnits * iUnits * iHeads; //--- Size of multi-heads self-attention uint mh_out = 2 * iWindowKey * iHeads * iUnits; //--- Size of our tensore uint out = 2 * iWindow * iUnits; //--- Size W0 tensor uint w0 = 2 * (iWindowKey + 1) * iHeads * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = 4 * (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention convolution forward | //| operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionForward(CBufferFloat* weights, CBufferFloat* inputs, CBufferFloat* outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = outputs.Total() / (2 * window_out); global_work_size[1] = window_out; global_work_size[2] = 1; //--- setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_w, weights.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_o, outputs.GetIndex()) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_inputs, (int)(inputs.Total() / 2)) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_step, (int)step) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_window_in, (int)window) setArgument(def_k_FeedForwardComplexConv, def_k_ffс_window_out, (int)window_out) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_activation - 1, (int)activ) //--- kernelExecute(def_k_FeedForwardComplexConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention attention score | //| operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionScore(CBufferFloat* qkv, CBufferFloat* scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_score, scores.GetIndex()) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_dimension, (int)iWindowKey) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_mask, (int)mask) kernelExecute(def_k_ComplexMHAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionOut(CBufferFloat* qkv, CBufferFloat* scores, CBufferFloat* out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_out, out.GetIndex()) setArgument(def_k_ComplexMHAttentionOut, def_k_mhao_dimension, (int)iWindowKey) //--- kernelExecute(def_k_ComplexMHAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention convolutuion update | //| weights operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* momentum1, CBufferFloat* momentum2, uint window, uint window_out, uint step = 0 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = weights.Total() / 2; global_work_size[1] = weights.Total() / 2; if(weights.GetIndex() < 0) ReturnFalse; if(optimization == SGD) { if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_learning_rates, lr) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_momentum, alpha) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvMomentum, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif } else { if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) float lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif t++; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention convolution input | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionInputGradients(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0 ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total() / 2; global_work_size[1] = 1; //--- setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_w, weights.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_g, gradient.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_o, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_outputs, int(Neurons() / 2) - shift_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_step, (int)step) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_in, (int)window) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_out, (int)window_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_activation, (int)activ) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_shift_out, (int)shift_out) kernelExecute(def_k_CalcHiddenGradientComplexConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!inp_gradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention attention inside | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionInsideGradients(CBufferFloat* qkv, CBufferFloat* qkv_g, CBufferFloat* scores, CBufferFloat* gradient ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()) //--- kernelExecute(def_k_ComplexMHAttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMLMHAttention sum and normalize | //| operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::SumAndNormalize(CBufferFloat* tensor1, CBufferFloat* tensor2, CBufferFloat* out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, out.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, 2 * dimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 2 * shift_in1) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 2 * shift_in2) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 2 * shift_out) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, multiplyer) setBuffer(def_k_MatrixSum, def_k_sum_numeric_flags, out.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_flags_enabled, 0) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- if(!normilize) return(true); //--- setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, dimension) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMLMHAttentionMLKV layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits; //--- Size of KV tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //--- Size of weights' matrix of KV tenzor uint kv_weights = 2 * (iWindow * iHeadsKV + 1) * iWindowKey; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits; //--- Size of out tensore uint out = iWindow * iUnits; //--- Size W0 tensor uint w0 = (iWindowKey * iHeads + 1) * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionMLKV attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMLMHAttentionMLKV feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, kv, iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLMHAttentionMLKV attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMLMHAttentionMLKV layer. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), kv_g, inp, GetPointer(Temp), iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMLMHAttentionMLKV trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindow, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMLMHAttentionMLKV GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); KV_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronMLMHAttentionMLKV state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle) || !KV_Weights.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMLMHAttentionMLKV state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMLMHAttentionMLKV weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronMLMHAttentionOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionMLKV *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < ((iLayers + iLayersToOneKV - 1) / iLayersToOneKV); l++) { if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.KV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? KV_Weights.At(l * 2 + 1) : KV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMLCrossAttentionMLKV layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = fmax(window_kv, 1); iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits; //--- Size of KV tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnitsKV; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //--- Size of weights' matrix of KV tenzor uint kv_weights = 2 * (iWindowKV + 1) * iWindowKey * iHeadsKV; //--- Size of Score tensor uint scores = iUnits * iUnitsKV * iHeads; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits; //--- Size of our tensore uint out = iWindow * iUnits; //--- Size W0 tensor uint w0 = (iWindowKey + 1) * iHeads * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLCrossAttentionMLKV attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMLCrossAttentionMLKV feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, kv, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMLCrossAttentionMLKV attention inside | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMLCrossAttentionMLKV layer. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, SecondActivation)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(i > 0) out_grad = temp; } //--- if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(0), KV_Tensors.At(1), SecondInput, SecondGradient, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; for(int i = 1; i < KV_Tensors.Total() / 2; i++) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i * (optimization == SGD ? 2 : 3)), KV_Tensors.At(i * 2 + 1), SecondInput, GetPointer(Temp), iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMLCrossAttentionMLKV state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMLCrossAttentionMLKV state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMLCrossAttentionMLKV trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindowKV, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSPyrAttentionOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_key, uint heads, uint heads_kv, uint units_count, uint pam_layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_in * units_count, optimization_type, batch)) ReturnFalse; //--- iWindowIn = window_in; iWindowKey = MathMax(window_key, 1); iHeads = MathMax(heads, 1); iHeadsKV = MathMax(heads_kv, 1); iCount = units_count; iPAMLayers = MathMax(pam_layers, 2); //--- caS3.Clear(); caQuery.Clear(); caKV.Clear(); caScore.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3) ReturnFalse; if(!s3.Init(0, l, OpenCL, iWindowIn, iCount, optimization, iBatch) || !caS3.Add(s3)) ReturnFalse; s3.SetActivationFunction(None); //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, iWindowKey * iHeads, iCount, optimization, iBatch) || !caQuery.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- KV conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 2 * iWindowKey * iHeadsKV, iCount, optimization, iBatch) || !caKV.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- MH Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowKey * iHeadsKV, iWindowKey * iHeadsKV, iWindowIn, iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!cFF1.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 4 * iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF1.SetActivationFunction(LReLU); if(!cFF2.Init(0, 0, OpenCL, 4 * iWindowIn, 4 * iWindowIn, iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF2.SetActivationFunction(None); if(!SetGradient(cFF2.getGradient())) ReturnFalse; //--- SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPyrAttentionOCL attention out operation. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionOut(CBufferFloat* q, CBufferFloat* kv, int scores, CBufferFloat* out, int window) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount/*Q units*/, iCount/*K units*/, iHeads}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_q, q.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_kv, kv.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_score, scores) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_out, out.GetIndex()) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_dimension, (int)iWindowKey) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_window, window) kernelExecuteLoc(def_k_MH2PyrAttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPyrAttentionOCL attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* q_g, CBufferFloat* kv, CBufferFloat* kv_g, int scores, CBufferFloat* gradient) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, q_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kv_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, gradient.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iCount) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!gradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSPyrAttentionOCL feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; CBufferFloat *q = NULL; CBufferFloat *kv = NULL; //--- for(uint l = 0; l < iPAMLayers; l++) { //--- Mix current = caS3.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; //--- Query current = caQuery.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; q = current.getOutput(); //--- Key and Value current = caKV.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; kv = current.getOutput(); //--- PAM current = caAttentionOut.At(l); if(!current || !AttentionOut(q, kv, caScore.At(l), current.getOutput(), iPAMLayers - l)) ReturnFalse; prev = current; //--- W0 current = caW0.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; } //--- Residual current = caW0.At(iPAMLayers); if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), current.getOutput(), iWindowIn, true)) ReturnFalse; //---FeedForward if(!cFF1.FeedForward(current.AsObject()) || !cFF2.FeedForward(cFF1.AsObject()) ) ReturnFalse; //--- Residual if(!SumAndNormalize(current.getOutput(), cFF2.getOutput(), getOutput(), iWindowIn, true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSPyrAttentionOCL layer. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; CNeuronBaseOCL *q = NULL; CNeuronBaseOCL *kv = NULL; //--- FeedForward current = caW0.At(iPAMLayers); if(!current || !cFF1.CalcHiddenGradients(cFF2.AsObject()) || !current.CalcHiddenGradients(cFF1.AsObject()) ) ReturnFalse; next = current; //--- Residual current = caW0.At(iPAMLayers - 1); if(!SumAndNormalize(getGradient(), next.getGradient(), current.getGradient(), iWindowIn, false)) ReturnFalse; CBufferFloat *residual = next.getGradient(); next = current; //--- for(int l = int(iPAMLayers - 1); l >= 0; l--) { //--- W0 current = caAttentionOut.At(l); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; //--- MH Attention q = caQuery.At(l); kv = caKV.At(l); if(!q || !kv || !AttentionInsideGradients(q.getOutput(), q.getGradient(), kv.getOutput(), kv.getGradient(), caScore.At(l), current.getGradient()) ) ReturnFalse; //--- Query current = caS3.At(l); if(!current || !current.CalcHiddenGradients(q.AsObject()) || !Concat(current.getGradient(), current.getGradient(), residual, iWindowIn, 0, iCount) ) ReturnFalse; //--- Key and Value if(!current || !current.CalcHiddenGradients(kv.AsObject()) || !SumAndNormalize(current.getGradient(), residual, current.getGradient(), iWindowIn, false) ) ReturnFalse; next = current; //--- S3 current = (l == 0 ? prevLayer : caW0.At(l - 1)); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; next = current; } //--- Residual current = caW0.At(iPAMLayers - 1); if(!DeActivation(prevLayer.getOutput(), residual, current.getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), residual, prevLayer.getGradient(), iWindowIn, false) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSPyrAttentionOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 current = caS3.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Query prev = current; current = caQuery.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Key and Value current = caKV.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- W0 prev = caAttentionOut.At(l); current = caW0.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current; } //--- FeedForward prev = caW0.At(iPAMLayers); if(!cFF1.UpdateInputWeights(prev) || !cFF2.UpdateInputWeights(cFF1.AsObject()) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPyrAttentionOCL array set open cl operation. | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::ArraySetOpenCL(CArrayObj * array, COpenCLMy * obj) { if(!array || array.Total() <= 0) return; //--- CNeuronBaseOCL *neuron = NULL; for(int l = 0; l < array.Total(); l++) { neuron = array.At(l); if(!neuron) continue; neuron.SetOpenCL(obj); } } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSPyrAttentionOCL GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); ArraySetOpenCL(GetPointer(caS3), OpenCL); ArraySetOpenCL(GetPointer(caQuery), OpenCL); ArraySetOpenCL(GetPointer(caKV), OpenCL); ArraySetOpenCL(GetPointer(caAttentionOut), OpenCL); ArraySetOpenCL(GetPointer(caW0), OpenCL); cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); //--- Score for(uint l = 0; l < iPAMLayers; l++) { int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) return; if(!caScore.Add(temp)) return; } } //+------------------------------------------------------------------+ //| Averages the CNeuronSPyrAttentionOCL weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSPyrAttentionOCL *Source = source; if(iPAMLayers != Source.iPAMLayers) ReturnFalse; CNeuronBaseOCL *s = NULL; CNeuronBaseOCL *c = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 c = caS3.At(l); s = Source.caS3.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Query c = caQuery.At(l); s = Source.caQuery.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Key and Value c = caKV.At(l); s = Source.caKV.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- W0 c = caW0.At(l); s = Source.caW0.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; } //--- FeedForward if(!cFF1.WeightsUpdate(Source.cFF1.AsObject(), tau) || !cFF2.WeightsUpdate(Source.cFF2.AsObject(), tau) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSPyrAttentionOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iWindowIn)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeadsKV)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPAMLayers)) < INT_VALUE) ReturnFalse; //--- Save objects CNeuronBaseOCL *neuron = NULL; for(uint l = 0; l < iPAMLayers; l++) { neuron = caS3.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caQuery.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caKV.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caW0.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; } //--- if(!cFF1.Save(file_handle) || !cFF2.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSPyrAttentionOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Load(const int file_handle) { //--- Clear Arrays if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); caS3.Clear(); caQuery.Clear(); caKV.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iWindowIn = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeadsKV = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iPAMLayers = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; //--- Load objects CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, l, OpenCL, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, s3) || !caS3.Add(s3) ) ReturnFalse; //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caQuery.Add(conv) ) ReturnFalse; //--- Key and Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caKV.Add(conv) ) ReturnFalse; //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caW0.Add(conv) ) ReturnFalse; } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!LoadInsideLayer(file_handle, cFF1.AsObject()) || !LoadInsideLayer(file_handle, cFF2.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMVMHAttentionMLKV layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables; //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits * iVariables; //--- Size of KV tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariables; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //--- Size of weights' matrix of Q tenzor uint kv_weights = (iWindow * iHeadsKV + 1) * iWindowKey; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads * iVariables; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //--- Size of out tensore uint out = iWindow * iUnits * iVariables; //--- Size W0 tensor uint w0 = (iWindowKey * iHeads + 1) * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMVMHAttentionMLKV attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariables)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMVMHAttentionMLKV feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, k, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, v, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMVMHAttentionMLKV attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariables)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMVMHAttentionMLKV layer. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMVMHAttentionMLKV trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMVMHAttentionMLKV weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMVMHAttentionMLKV *Source = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), Source.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMVMHAttentionMLKV state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE) || !FileWriteInteger(file_handle, iVariables, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle)) ReturnFalse; if(!K_Tensors.Save(file_handle) || !K_Weights.Save(file_handle)) ReturnFalse; if(!V_Tensors.Save(file_handle) || !V_Weights.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMVMHAttentionMLKV state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMVMHAttentionMLKV GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMVMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); K_Tensors.SetOpenCL(OpenCL); K_Weights.SetOpenCL(OpenCL); V_Tensors.SetOpenCL(OpenCL); V_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMVCrossAttentionMLKV layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, uint variables_q, uint variables_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables_q, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = window_kv; iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables_q; iVariablesKV = variables_kv; //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits * iVariables; //--- Size of KV tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariablesKV; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //--- Size of weights' matrix of Q tenzor uint kv_weights = (iWindowKV * iHeadsKV + 1) * iWindowKey; //--- Size of Score tensor uint scores = iUnits * iUnitsKV * iHeads * iVariables; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //--- Size of out tensore uint out = iWindow * iUnits * iVariables; //--- Size W0 tensor uint w0 = (iWindowKey * iHeads + 1) * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMVCrossAttentionMLKV attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariablesKV)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMVCrossAttentionMLKV feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, k, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, v, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariablesKV, iWindowKey * iHeadsKV * iVariablesKV, iUnitsKV)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //---out.BufferRead(); //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMVCrossAttentionMLKV attention inside | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariablesKV)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMVCrossAttentionMLKV layer. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); if(!SecondGradient.BufferInit(SecondGradient.Total(), 0) || !SecondGradient.BufferWrite()) ReturnFalse; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- Deactivation7 if(SecondActivation != None && !DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(NeuronOCL.Activation() != None && !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMVCrossAttentionMLKV trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMVCrossAttentionMLKV state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMVMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iVariablesKV), INT_VALUE)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMVCrossAttentionMLKV state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); iVariablesKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Returns the weights of the CNeuronMVCrossAttentionMLKV. | //+------------------------------------------------------------------+ CBufferFloat* CNeuronMVCrossAttentionMLKV::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = FF_Weights.At((iLayers - 1) * (optimization == SGD ? 6 : 9) + 2); return(result); } //+------------------------------------------------------------------+ //| Initializes the CNeuronInjectTST layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); if(!cPatching.Init(0, 0, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; cPatching.SetActivationFunction(None); if(!cCIPosition.Init(0, 1, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCIPosition.SetActivationFunction(None); if(!cCMPosition.Init(0, 2, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCMPosition.SetActivationFunction(None); if(!cChanelIndependentAttention.Init(0, 3, OpenCL, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, variables, optimization, iBatch)) ReturnFalse; cChanelIndependentAttention.SetActivationFunction(None); if(!cChanelMixAttention.Init(0, 4, OpenCL, window * variables, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, optimization, iBatch)) ReturnFalse; cChanelMixAttention.SetActivationFunction(None); if(!cGlobalInjectionAttention.Init(0, 5, OpenCL, window, window_key, heads, window * variables, heads_kv, units_count, units_count, layers, layers_to_one_kv, variables, 1, optimization, iBatch)) ReturnFalse; cGlobalInjectionAttention.SetActivationFunction(None); if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronInjectTST feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cPatching.FeedForward(NeuronOCL)) ReturnFalse; if(!cCIPosition.FeedForward(cPatching.AsObject()) || !cCMPosition.FeedForward(cPatching.AsObject()) ) ReturnFalse; if(!cChanelIndependentAttention.FeedForward(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.FeedForward(cCMPosition.AsObject())) ReturnFalse; if(!cGlobalInjectionAttention.FeedForward(cCIPosition.AsObject(), cCMPosition.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronInjectTST | //| layer. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cChanelIndependentAttention.CalcHiddenGradients(cGlobalInjectionAttention.AsObject(), cChanelMixAttention.getOutput(), cChanelMixAttention.getGradient(), (ENUM_ACTIVATION)cChanelMixAttention.Activation()) ) ReturnFalse; //--- if(!cCMPosition.CalcHiddenGradients(cChanelMixAttention.AsObject())) ReturnFalse; if(!cCIPosition.CalcHiddenGradients(cChanelIndependentAttention.AsObject())) ReturnFalse; //--- if(!cPatching.CalcHiddenGradients(cCIPosition.AsObject())) ReturnFalse; if(!SumAndNormalize(cPatching.getGradient(), cPatching.getGradient(), GetPointer(cTemp), 1, false)) ReturnFalse; if(!cPatching.CalcHiddenGradients(cCMPosition.AsObject())) ReturnFalse; if(!SumAndNormalize(cPatching.getGradient(), GetPointer(cTemp), cPatching.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronInjectTST trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cPatching.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cCIPosition.UpdateInputWeights(cPatching.AsObject()) || !cCMPosition.UpdateInputWeights(cPatching.AsObject()) ) ReturnFalse; //--- if(!cChanelIndependentAttention.UpdateInputWeights(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.UpdateInputWeights(cCMPosition.AsObject())) ReturnFalse; //--- if(!cGlobalInjectionAttention.UpdateInputWeights(cChanelIndependentAttention.AsObject(), cChanelMixAttention.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronInjectTST state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!cCIPosition.Save(file_handle)) ReturnFalse; if(!cCMPosition.Save(file_handle)) ReturnFalse; if(!cChanelIndependentAttention.Save(file_handle)) ReturnFalse; if(!cChanelMixAttention.Save(file_handle)) ReturnFalse; if(!cGlobalInjectionAttention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronInjectTST state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Load(const int file_handle) { cTemp.BufferFree(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCIPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCMPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelIndependentAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelMixAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGlobalInjectionAttention.AsObject())) ReturnFalse; //--- if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronInjectTST GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronInjectTST::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cPatching.SetOpenCL(OpenCL); cCIPosition.SetOpenCL(OpenCL); cCMPosition.SetOpenCL(OpenCL); cChanelIndependentAttention.SetOpenCL(OpenCL); cChanelMixAttention.SetOpenCL(OpenCL); cGlobalInjectionAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Returns the weights of the CNeuronInjectTST. | //+------------------------------------------------------------------+ CBufferFloat* CNeuronInjectTST::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = cGlobalInjectionAttention.getWeights(); //--- return(result); } //+------------------------------------------------------------------+ //| Reports checked-transfer support for CNeuronTransposeRCDOCL. | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::SupportsOpenCLChecked(void) const { return(Type() == defNeuronTransposeRCDOCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTransposeRCDOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * dimension, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTransposeRCDOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iWindow, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, Output.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTransposeRCDOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindow, iCount, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, NeuronOCL.getGradientIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, Gradient.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSceneSpecific layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, 8, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(8 * units_count_kv, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cSceneSpecificKnowledge.Init(0, 1, OpenCL, 8 * units_count_kv, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSceneSpecific feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::feedForward(CNeuronBaseOCL* NeuronOCL) { if(bTrain && !cSceneSpecificKnowledge.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSceneSpecific layer. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(NeuronOCL, cSceneSpecificKnowledge.getOutput(), cSceneSpecificKnowledge.getGradient(), (ENUM_ACTIVATION)cSceneSpecificKnowledge.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSceneSpecific trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; if(!cSceneSpecificKnowledge.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSceneSpecific state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cSceneSpecificKnowledge.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSceneSpecific state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSceneSpecificKnowledge.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSceneSpecific GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSceneSpecific::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cSceneSpecificKnowledge.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMLMHSceneConditionAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- //--- Size of Q tensor uint num_q = iWindowKey * iHeads * iUnits; //--- Size of KV tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //--- Size of weights' matrix of Q tenzor uint q_weights = (iWindow * iHeads) * iWindowKey; //--- Size of weights' matrix of KV tenzor uint kv_weights = 2 * (iWindow * iHeadsKV) * iWindowKey; //--- Size of Score tensor uint scores = iUnits * iUnits * iHeads; //--- Size of multi-heads self-attention uint mh_out = iWindowKey * iHeads * iUnits; //--- Size of out tensore uint out = iWindow * iUnits; //--- Size W0 tensor uint w0 = (iWindowKey * iHeads + 1) * iWindow; //--- Size of weights' matrix 1-st feed forward layer uint ff_1 = 4 * (iWindow + 1) * iWindow; //--- Size of weights' matrix 2-nd feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //--- CNeuronBaseOCL *base = NULL; CNeuronSceneSpecific *ss = NULL; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } if(i % iLayersToOneKV == 0) { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init((q_weights + kv_weights), cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init((q_weights + kv_weights), cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } else { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init(q_weights, cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(q_weights, cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax((num_q + num_kv)*iWindow, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMLMHSceneConditionAttention feed-forward pass | //| and writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.FeedForward(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.FeedForward(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(bTrain && !sa.FeedForward(cSceneAgnostic[i * 2])) ReturnFalse; CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); //--- weights q_weights = QKV_Weights[i * 2]; q = QKV_Tensors[i * 2]; if(i % iLayersToOneKV == 0) { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), GetPointer(Temp))) ReturnFalse; kv_weights = KV_Weights[(i / iLayersToOneKV) * 2]; kv = KV_Tensors[(i / iLayersToOneKV) * 2]; if(IsStopped() || !DeConcat(q_weights, kv_weights, GetPointer(Temp), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(IsStopped() || !MatMul(inputs, kv_weights, kv, iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; } else { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), q_weights)) ReturnFalse; } if(IsStopped() || !MatMul(inputs, q_weights, q, iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors[i * 2]; CBufferFloat *out = AO_Tensors[i * 2]; if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors[i * 6]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9)], out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors[i * 6 + 1]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors[i * 6 + 2]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- result return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMLMHSceneConditionAttention layer. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors[KV_Tensors.Total() - 1]; CNeuronBaseOCL *ss = NULL, *sa = NULL; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors[(i / iLayersToOneKV) * 2 + 1]; //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], out_grad, FF_Tensors[i * 6 + 1], FF_Tensors[i * 6 + 4], 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors[i * 6 + 3]; if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], FF_Tensors[i * 6 + 4], FF_Tensors[i * 6], temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9)], out_grad, AO_Tensors[i * 2], AO_Tensors[i * 2 + 1], iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value sa = cSceneAgnostic[i * 2 + 1]; ss = cSceneSpecific[i * 2 + 1]; if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[(i / iLayersToOneKV) * 2], kv_g, S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2], GetPointer(Temp), S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !MatMulGrad(inp, temp, QKV_Weights[i * 2], QKV_Weights[i * 2 + 1], QKV_Tensors[i * 2 + 1], iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !MatMulGrad(inp, GetPointer(Temp), KV_Weights[i / iLayersToOneKV * 2], KV_Weights[i / iLayersToOneKV * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2 + 1], iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(QKV_Weights[i * 2 + 1], KV_Weights[i / iLayersToOneKV * 2 + 1], ss.getGradient(), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), ss.getGradient(), ss.Activation(), sa.Activation())) ReturnFalse; } else { if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), QKV_Weights[i * 2 + 1], ss.Activation(), sa.Activation())) ReturnFalse; } if(i > 0) out_grad = temp; } //--- CBufferFloat *inp_grad = prevLayer.getGradient(); if(!prevLayer.SetGradient(GetPointer(Temp), false)) ReturnFalse; for(int i = int(iLayers - 2); (i >= 0 && !IsStopped()); i -= 2) { ss = cSceneSpecific[i]; if(IsStopped() || !ss.CalcHiddenGradients(cSceneSpecific[i + 1])) ReturnFalse; if(IsStopped() || !prevLayer.CalcHiddenGradients(ss, NULL)) ReturnFalse; if(IsStopped() || !SumAndNormalize(prevLayer.getGradient(), inp_grad, inp_grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!prevLayer.SetGradient(inp_grad, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMLMHSceneConditionAttention trainable weights | //| from the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.UpdateInputWeights(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.UpdateInputWeights(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(!sa.UpdateInputWeights(cSceneAgnostic[i * 2])) ReturnFalse; //--- Attention out if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), FF_Tensors.At(i * 6 + 3), AO_Tensors.At(i * 2), (optimization == SGD ? FF_Weights.At(i * 6 + 3) : FF_Weights.At(i * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- Feed Forward if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), (optimization == SGD ? FF_Weights.At(i * 6 + 4) : FF_Weights.At(i * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(i * 6 + 5), FF_Tensors.At(i * 6 + 1), (optimization == SGD ? FF_Weights.At(i * 6 + 5) : FF_Weights.At(i * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; } //--- result return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMLMHSceneConditionAttention state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cSceneSpecific.Save(file_handle)) ReturnFalse; if(!cSceneAgnostic.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMLMHSceneConditionAttention state from the | //| file handle. | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Load(file_handle)) ReturnFalse; if(!cSceneSpecific.Load(file_handle)) ReturnFalse; if(!cSceneAgnostic.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMLMHSceneConditionAttention GPU buffers to | //| the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMLMHSceneConditionAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionMLKV::SetOpenCL(obj); cSceneSpecific.SetOpenCL(OpenCL); cSceneAgnostic.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSPFormer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; //--- Init SuperPoints for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } } //--- CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 6, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 7, OpenCL, iSPWindow, iSPWindow, 2 * iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKeyValue.Add(conv)) ReturnFalse; } //--- Mask conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 8, OpenCL, iSPWindow, iSPWindow, iUnits * iHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cMask.Add(conv)) ReturnFalse; transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, l * 14 + 9, OpenCL, iSPUnits, iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMask.Add(transp)) ReturnFalse; //--- MH Cross Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 10, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; //--- Cross Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 11, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 12, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 13, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 14, OpenCL, iWindow, iWindow, 2 * iWindowKey * iSPHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKeyValue.Add(conv)) ReturnFalse; } //--- MH Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 15, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; //--- Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 16, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 17, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- FeedForward conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 18, OpenCL, iWindow, iWindow, iWindow * 4, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 19, OpenCL, iWindow * 4, iWindow * 4, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 20, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(l == (iLayers - 1)) { if(!SetGradient(conv.getGradient())) ReturnFalse; if(!SetOutput(base.getOutput())) ReturnFalse; } } //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPFormer attention out operation. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* kv, const int scores, CNeuronBaseOCL* out, CNeuronBaseOCL* mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f ) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHMaskAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_mask_at_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_at_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_at_score, scores) setBuffer(kernel, def_k_mask_at_out, out.getOutputIndex()) setBuffer(kernel, def_k_mask_at_mask, (!mask ? scores : mask.getOutputIndex())) setArgument(kernel, def_k_mask_at_dimension, dimension) setArgument(kernel, def_k_mask_at_heads_kv, heads_kv) setArgument(kernel, def_k_mask_at_mask_level, (!mask ? 0 : mask_level)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPFormer attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* kv, const int scores, CNeuronBaseOCL* out, CNeuronBaseOCL* mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHMaskAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_mask_atg_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_kv_g, kv.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_scores, scores) setBuffer(kernel, def_k_mask_atg_gradient, out.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_mask, (!mask ? scores : mask.getOutputIndex())) setBuffer(kernel, def_k_mask_atg_mask_g, (!mask ? scores : mask.getGradientIndex())) setArgument(kernel, def_k_mask_atg_kunits, units_kv) setArgument(kernel, def_k_mask_atg_heads_kv, heads_kv) setArgument(kernel, def_k_mask_atg_mask_level, (!mask ? 0 : mask_level)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!kv.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSPFormer feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.FeedForward(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.FeedForward(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; neuron = cMask[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cMask[l * 2])) ReturnFalse; if(!AttentionOut(q, kv_cross, cScores[l * 2], cMHCrossAttentionOut[l], neuron, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.FeedForward(inputs)) ReturnFalse; } if(!AttentionOut(q, kv_self, cScores[l * 2 + 1], cMHSelfAttentionOut[l], NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 1]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.FeedForward(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cFeedForward[l * 2])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 2]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronSPFormer | //| layer. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *superpoints = cSuperPoints[cSuperPoints.Total() - 1]; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = cSPKeyValue[cSPKeyValue.Total() - 1], *kv_self = cQKeyValue[cQKeyValue.Total() - 1]; //--- if(!cTempSP.Fill(0) || !cTempSelfKV.Fill(0) || !cTempCrossKV.Fill(0)) ReturnFalse; for(int l = int(iLayers - 1); l >= 0; l--) { //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; neuron = cResidual[l * 3 + 1]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; if(!SumAndNormalize(((CNeuronBaseOCL*)cResidual[l * 3 + 2]).getGradient(), neuron.getGradient(), ((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention neuron = cMHSelfAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cSelfAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 3]; if(((l + 1) % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !cTempSelfKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_self, cScores[l * 2 + 1], neuron, NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(kv_self.getGradient(), GetPointer(cTempSelfKV), kv_self.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(kv_self.getGradient(), GetPointer(cTempSelfKV), GetPointer(cTempSelfKV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } inputs = cResidual[l * 3]; if(!inputs || !inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if((l % iLayersSP) == 0) { CBufferFloat *temp = inputs.getGradient(); if(!inputs.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!inputs.CalcHiddenGradients(kv_self, NULL)) ReturnFalse; if(!SumAndNormalize(temp, GetPointer(cTempQ), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!inputs.SetGradient(temp, false)) ReturnFalse; } if(!SumAndNormalize(((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross Attention neuron = cMHCrossAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cCrossAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 2]; if(((l + 1) % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !cTempCrossKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_cross, cScores[l * 2], neuron, cMask[l * 2 + 1], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; inputs = (l == 0 ? cQuery[1] : cResidual[l * 3 - 1]); if(!inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if(!SumAndNormalize(inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), inputs.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((l % iLayersSP) == 0) { if(!superpoints.CalcHiddenGradients(kv_cross, NULL)) ReturnFalse; if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cMask[l * 2 + 1]) || !DeActivation(neuron.getOutput(), neuron.getGradient(), neuron.getGradient(), neuron.Activation())) ReturnFalse; if(!superpoints.CalcHiddenGradients(neuron, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), superpoints.getGradient(), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- for(int l = cSuperPoints.Total() - 2; l >= 0; l--) { superpoints = cSuperPoints[l]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[l + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(superpoints, NULL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSPFormer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.UpdateInputWeights(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.UpdateInputWeights(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.UpdateInputWeights(inputs)) ReturnFalse; } neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.UpdateInputWeights(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSPFormer GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSPFormer::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL);; cQuery.SetOpenCL(OpenCL); cSPKeyValue.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cQKeyValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| Handles the CNeuronSPFormer create buffers operation. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::CreateBuffers(void) { for(uint l = 0; l < iLayers; l++) { //--- Cross Attention int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); vector temp = vector::Zeros(s); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; if(!OpenCL.BufferFromVector(s, temp, CL_MEM_READ_WRITE)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iSPWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempSelfKV.BufferInit(2 * iWindowKey * iUnits * iHeads, 0) || !cTempSelfKV.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossKV.BufferInit(2 * iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossKV.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSPFormer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cSPKeyValue.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cQKeyValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSPFormer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cSPKeyValue.Load(file_handle)) ReturnFalse; if(!cMask.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cQKeyValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(uint i = 0; i < iLayers; i++) { neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; } //--- if(Output != neuron.getOutput()) if(!SetOutput(neuron.getOutput())) ReturnFalse; //--- if(!CreateBuffers()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMAFT layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMAFT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; buf = base.getWeights(); if(!buf || !buf.BufferInit(buf.Total(), 0) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; CNeuronLearnabledPE *pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 2, OpenCL, base.Neurons(), optimization, iBatch)) ReturnFalse; if(!cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 3, OpenCL, pe.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetOutput(pe.GetPE())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 4; for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } layer_id++; } CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMAFT GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMAFT::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| Handles the CNeuronMAFT create buffers operation. | //+------------------------------------------------------------------+ bool CNeuronMAFT::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Score int s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross Attention s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMAFT attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int pos_bias, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const bool use_pos_bias) { if(!OpenCL || !q || !k || !v || !out || scores < 0 || (use_pos_bias && pos_bias < 0)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, (use_pos_bias ? pos_bias : scores)) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, int(use_pos_bias)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMAFT attention inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!v.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMAFT calc position bias operation. | //+------------------------------------------------------------------+ bool CNeuronMAFT::CalcPositionBias(CBufferFloat* pos_q, CBufferFloat* pos_k, const int pos_bias, const int units, const int units_kv, const int dimension) { if(!OpenCL || !pos_q || !pos_k || pos_bias < 0) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {units/*Q units*/, units_kv}; int kernel = def_k_CalcPositionBias; //--- ResetLastError(); setBuffer(kernel, def_k_cpb_data1, pos_q.GetIndex()) setBuffer(kernel, def_k_cpb_data2, pos_k.GetIndex()) setBuffer(kernel, def_k_cpb_result, pos_bias) setArgument(kernel, def_k_cpb_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG vector temp; if(!OpenCL.BufferToVector(pos_bias, temp)) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMAFT feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMAFT::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.FeedForward(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 2], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; value = cQPosition[l * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), inputs.getOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Calc Position bias if(!CalcPositionBias(value.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 2 + 1], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormalize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMAFT | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMAFT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cCrossAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 4]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2 + 1], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 3]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(l == 0) base = cQuery[2]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), GetPointer(cTempQ), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), GetPointer(cTempQ), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormalize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- Qeury query = cQuery[1]; if(!query || !query.CalcHiddenGradients(cQuery[2])) ReturnFalse; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[2]).AddPEGradient(base.getGradient()))) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } //--- From Value superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMAFT trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMAFT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.UpdateInputWeights(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMAFT state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMAFT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cQPosition.Save(file_handle)) ReturnFalse; if(!cQKey.Save(file_handle)) ReturnFalse; if(!cQValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSPKey.Save(file_handle)) ReturnFalse; if(!cSPValue.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMAFT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMAFT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cQPosition.Load(file_handle)) ReturnFalse; if(!cQKey.Load(file_handle)) ReturnFalse; if(!cQValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSPKey.Load(file_handle)) ReturnFalse; if(!cSPValue.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; for(uint i = 0; i < iLayers; i++) { CNeuronBaseOCL *neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; neuron = cQPosition[i * 2 + 1]; if(!neuron || !(((CNeuronBaseOCL*)cQPosition[(i + 1) * 2]).SetGradient(neuron.getGradient())) ) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[cQPosition.Total() - 1]).SetGradientIndex(getGradientIndex()))) ReturnFalse; if(Type() == defNeuronMAFT) { if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[2]).GetPE()))) ReturnFalse; } //--- if(!CreateBuffers()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGRES layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGRES::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint ref_size, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- CNeuronBaseOCL *base = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronConvOCL *conv = NULL; CNeuronLearnabledPE *pe = NULL; //--- Init Querys cQuery.Clear(); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 0, OpenCL, iSPUnits, iSPWindow, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iSPUnits, iSPUnits, iUnits, 1, iSPWindow, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 2, OpenCL, iSPWindow, iUnits, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 3, OpenCL, iSPWindow, iSPWindow, iWindow, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch) || !cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 5, OpenCL, pe.Neurons(), optimization, iBatch) || !base.SetOutput(pe.GetPE()) || !cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 6; cSuperPoints.Clear(); for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual || !residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch) || !cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); } layer_id++; } conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch) || !cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Reference cReference.Clear(); base = new CNeuronBaseOCL(); if(!base || !base.Init(iWindow * iUnits, layer_id, OpenCL, ref_size, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; base.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, base.Neurons(), optimization, iBatch) || !cReference.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers cQKey.Clear(); cQValue.Clear(); cSPKey.Clear(); cSPValue.Clear(); cSelfAttentionOut.Clear(); cCrossAttentionOut.Clear(); cMHCrossAttentionOut.Clear(); cMHSelfAttentionOut.Clear(); cMHRefAttentionOut.Clear(); cRefAttentionOut.Clear(); cRefKey.Clear(); cRefValue.Clear(); cResidual.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cResidual.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Reference Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHRefAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cRefAttentionOut.Add(conv)) ReturnFalse; layer_id++; if(!conv.SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[cSelfAttentionOut.Total() - 1]).getGradient(), true)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronGRES create buffers operation. | //+------------------------------------------------------------------+ bool CNeuronGRES::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Score int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Reference Cross-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGRES feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronGRES::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).FeedForward(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; if(!SecondInput) ReturnFalse; if(reference.getOutput() != SecondInput) if(!reference.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).FeedForward(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Calc Position bias cross = cQPosition[l * 2]; if(!cross || !CalcPositionBias(cross.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 3], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormalize(cross.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 1], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.FeedForward(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.FeedForward(reference)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 2], cMHRefAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.FeedForward(cMHRefAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormalize(cross.getOutput(), self.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormalize(inputs.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormalize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGRES GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronGRES::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } //--- cScores.Clear(); cPositionBias.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cReference.SetOpenCL(OpenCL); cRefKey.SetOpenCL(OpenCL); cRefValue.SetOpenCL(OpenCL); cMHRefAttentionOut.SetOpenCL(OpenCL); cRefAttentionOut.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronGRES | //| layer. | //+------------------------------------------------------------------+ bool CNeuronGRES::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Reference Cross-Attention base = cMHRefAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cRefAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 7]; key = cRefKey[l]; value = cRefValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 3 + 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; value = cCrossAttentionOut[l]; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cSelfAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 6]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2 + 1], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormalize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 3 + 5]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } if(l == 0) base = cQuery[4]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual if(!SumAndNormalize(base.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = base; } //--- Qeury query = cQuery[3]; if(!query || !query.CalcHiddenGradients(cQuery[4])) ReturnFalse; base = cQPosition[0]; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[4]).AddPEGradient(base.getGradient()))) ReturnFalse; if(!DiversityLoss(query, iUnits, iWindow, true)) ReturnFalse; for(int i = 2; i >= 0; i--) { query = cQuery[i]; if(!query || !query.CalcHiddenGradients(cQuery[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(query, NULL)) ReturnFalse; CBufferFloat *inputs_gr = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(query.getGradient(), false)) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; //--- From Value CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; if(!DiversityLoss(superpoints, iSPUnits, iSPWindow, true)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(inputs_gr, false)) ReturnFalse; //--- Reference base = cReference[0]; if(base.getGradient() != SecondGradient) { if(!base.SetGradient(SecondGradient)) ReturnFalse; base.SetActivationFunction(SecondActivation); } base = cReference[2]; if(!base || !base.CalcHiddenGradients(cRefKey[0])) ReturnFalse; inputs_gr = base.getGradient(); if(!base.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[0])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; for(uint i = 1; i < iLayers; i++) { if(!base.CalcHiddenGradients(cRefKey[i])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[i])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!base.SetGradient(inputs_gr, false)) ReturnFalse; base = cReference[1]; if(!base.CalcHiddenGradients(cReference[2])) ReturnFalse; if(!DiversityLoss(base, iUnits, iWindow, true)) ReturnFalse; base = cReference[0]; if(!base.CalcHiddenGradients(cReference[1])) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGRES trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronGRES::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).UpdateInputWeights(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).UpdateInputWeights(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.UpdateInputWeights(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.UpdateInputWeights(reference)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.UpdateInputWeights(cMHRefAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronGRES state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGRES::Save(const int file_handle) { if(!CNeuronMAFT::Save(file_handle)) ReturnFalse; //--- if(!cReference.Save(file_handle)) ReturnFalse; if(!cRefKey.Save(file_handle)) ReturnFalse; if(!cRefValue.Save(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Save(file_handle)) ReturnFalse; if(!cRefAttentionOut.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGRES state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronGRES::Load(const int file_handle) { if(!CNeuronMAFT::Load(file_handle)) ReturnFalse; //--- if(!cReference.Load(file_handle)) ReturnFalse; if(!cRefKey.Load(file_handle)) ReturnFalse; if(!cRefValue.Load(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Load(file_handle)) ReturnFalse; if(!cRefAttentionOut.Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < iLayers; i++) { if(!(((CNeuronBaseOCL*)cRefAttentionOut[i]).SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[i]).getGradient(), true))) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[4]).GetPE()))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGEGWA layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_kv, uint heads_kv, uint units_count_kv, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronGEGWA *temp = new CNeuronGEGWA(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, (units_count + 1) / 2, window_kv, heads_kv, units_count_kv, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronMLCrossAttentionMLKV *temp = new CNeuronMLCrossAttentionMLKV(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, window_kv, heads_kv, (units_count + 1) / 2, units_count_kv, layers, 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(!cResidual.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cCrossAttention.Init(0, 6, OpenCL, window, window_key, heads, window_kv, heads_kv, units_count, units_count_kv, layers, 1, optimization, iBatch)) ReturnFalse; if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGEGWA feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, false)) ReturnFalse; if(!cCrossAttention.FeedForward(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronGEGWA | //| layer. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::calcInputGradients(CNeuronBaseOCL* prevLayer, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; if(!cResidual.CalcHiddenGradients(cCrossAttention.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(bAddNeckGradient) { CBufferFloat *temp = cNeck.getGradient(); if(!cNeck.SetGradient(cMergeSplit[0].getGradient(), false)) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(cNeck.getGradient(), temp, temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cNeck.SetGradient(temp, false)) ReturnFalse; } else if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject(), SecondInput, GetPointer(cTemp), SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, GetPointer(cTemp), SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), GetPointer(cTemp), cMergeSplit[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGEGWA trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; if(!cCrossAttention.UpdateInputWeights(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronGEGWA state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Save(const int file_handle) { if(!CNeuronUShapeAttention::Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGEGWA state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronGEGWA: cNeck = new CNeuronGEGWA(); if(!cNeck) ReturnFalse; break; case defNeuronMLCrossAttentionMLKV: cNeck = new CNeuronMLCrossAttentionMLKV(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cResidual.Init(0, 0, OpenCL, cCrossAttention.Neurons(), optimization, iBatch)) ReturnFalse; if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGEGWA GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronGEGWA::SetOpenCL(COpenCLMy * obj) { CNeuronUShapeAttention::SetOpenCL(obj); cResidual.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Returns the inside layer of the CNeuronGEGWA. | //+------------------------------------------------------------------+ CNeuronBaseOCL* CNeuronGEGWA::GetInsideLayer(const int layer) const { if(layer < 0) return(NULL); //--- if(layer == 0) return(cNeck); //--- if(!cNeck || cNeck.Type() != Type()) return(NULL); //--- CNeuronGEGWA* temp = cNeck; return(temp.GetInsideLayer(layer - 1)); } //+------------------------------------------------------------------+ //| Initializes the CNeuronLPC layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronLPC::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, window, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(window * units_count, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cPrimitives.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronLPC feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronLPC::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(bTrain && !cPrimitives.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronLPC | //| layer. | //+------------------------------------------------------------------+ bool CNeuronLPC::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(cPrimitives.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!DiversityLoss(cPrimitives.AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronLPC trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronLPC::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!cPrimitives.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronLPC state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronLPC::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cPrimitives.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronLPC state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronLPC::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrimitives.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronLPC GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronLPC::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cPrimitives.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Handles the CNeuronOCM attention out operation. | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, scores) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronOCM attention inside gradients operation. | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!v.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronOCM layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronOCM::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint prim_window, uint window_key, uint prim_units, uint prim_heads, uint cont_window, uint cont_units, uint cont_heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, cont_window * cont_units, optimization_type, batch)) ReturnFalse; //--- iPrimWindow = prim_window; iPrimUnits = prim_units; iPrimHeads = prim_heads; iContWindow = cont_window; iContUnits = cont_units; iContHeads = cont_heads; iWindowKey = window_key; //--- cQuery.Clear(); cKey.Clear(); cValue.Clear(); cMHAttentionOut.Clear(); cAttentionOut.Clear(); cResidual.Clear(); cFeedForward.Clear(); //--- CNeuronBaseOCL *neuron = NULL; CNeuronConvOCL *conv = NULL; //--- Primitives Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 0, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 2, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 4, OpenCL, iPrimHeads * iWindowKey, iPrimHeads * iWindowKey, iPrimWindow, iPrimUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 6, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 7, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 8, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 10, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 12, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 13, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 14, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 16, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 18, OpenCL, iContWindow, iContWindow, 4 * iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 19, OpenCL, 4 * iContWindow, 4 * iContWindow, iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; //--- if(!SetGradient(conv.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronOCM feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronOCM::feedForward(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Primitives) || !k.FeedForward(Primitives) || !v.FeedForward(Primitives) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[0]) ) ReturnFalse; v = cResidual[0]; if(!v || !SumAndNormalize(Primitives.getOutput(), neuron.getOutput(), v.getOutput(), iPrimWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Context) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[1]) ) ReturnFalse; v = cResidual[1]; if(!v || !SumAndNormalize(Context.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(neuron) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.FeedForward(cMHAttentionOut[2]) ) ReturnFalse; v = cResidual[2]; if(!v || !SumAndNormalize(q.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.FeedForward(neuron) || !k.FeedForward(q) || !SumAndNormalize(neuron.getOutput(), k.getOutput(), Output, iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronOCM | //| layer. | //+------------------------------------------------------------------+ bool CNeuronOCM::calcInputGradients(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { if(!Primitives || !Context) ReturnFalse; //--- CNeuronBaseOCL *neuron = cResidual[2], *q = cFeedForward[1], *k = cAttentionOut[2], *v = NULL, *residual = GetPointer(this); //--- Feed Forward if(!neuron || !q || !k || !q.CalcHiddenGradients(cFeedForward[1]) || !neuron.CalcHiddenGradients(q, NULL) || !SumAndNormalize(neuron.getGradient(), residual.getGradient(), k.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = k; //--- Context Self-Attention neuron = cMHAttentionOut[2]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iContUnits, iContHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[1]; if(!neuron || !neuron.CalcHiddenGradients(q, NULL) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), q.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = q; if(!neuron.CalcHiddenGradients(k, NULL) || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(v, NULL) || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- Cross-Attention neuron = cMHAttentionOut[1]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[0]; if(!Context.CalcHiddenGradients(q, NULL) || !DeActivation(Context.getOutput(), residual.getGradient(), residual.getGradient(), Context.Activation()) || !SumAndNormalize(Context.getGradient(), residual.getGradient(), Context.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = cAttentionOut[0]; if(!neuron.CalcHiddenGradients(k, NULL)) ReturnFalse; CBufferFloat *temp = neuron.getGradient(); if(!neuron.SetGradient(residual.getGradient(), false)) ReturnFalse; if(!neuron.CalcHiddenGradients(v, NULL) || !SumAndNormalize(temp, neuron.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; //--- Primitives Self-Attention neuron = cMHAttentionOut[0]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[0]; k = cKey[0]; v = cValue[0]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; if(!Primitives.CalcHiddenGradients(q, NULL) || !DeActivation(Primitives.getOutput(), residual.getGradient(), residual.getGradient(), Primitives.Activation()) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(k, NULL) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(v, NULL) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), Primitives.getGradient(), iPrimWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronOCM trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronOCM::updateInputWeights(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Primitives) || !k.UpdateInputWeights(Primitives) || !v.UpdateInputWeights(Primitives) ) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[0]) ) ReturnFalse; neuron = cResidual[0]; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Context) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[1]) ) ReturnFalse; neuron = cResidual[1]; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.UpdateInputWeights(cMHAttentionOut[2]) ) ReturnFalse; neuron = cResidual[2]; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(q) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronOCM create buffers operation. | //+------------------------------------------------------------------+ bool CNeuronOCM::CreateBuffers(void) { //--- Primitives Self-Attention int s = int(iPrimUnits * iPrimUnits * iPrimHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross-Attention s = int(iContUnits * iPrimUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Content Self-Attention s = int(iContUnits * iContUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronOCM GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronOCM::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } cScores.Clear(); CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); CreateBuffers(); } //+------------------------------------------------------------------+ //| Saves the CNeuronOCM state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronOCM::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save objects if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iPrimWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronOCM state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronOCM::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load objects if(!cQuery.Load(file_handle)) ReturnFalse; if(!cKey.Load(file_handle)) ReturnFalse; if(!cValue.Load(file_handle)) ReturnFalse; if(!cAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- Load constants iPrimWindow = (uint)FileReadInteger(file_handle); iPrimUnits = (uint)FileReadInteger(file_handle); iPrimHeads = (uint)FileReadInteger(file_handle); iContWindow = (uint)FileReadInteger(file_handle); iContUnits = (uint)FileReadInteger(file_handle); iContHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- CNeuronBaseOCL *neuron = NULL; cMHAttentionOut.Clear(); cResidual.Clear(); //--- Primitives Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, iPrimWindow * iPrimUnits, optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- neuron = cFeedForward[1]; if(!SetGradient(neuron.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRefMask layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronRefMask::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint content_size, uint content_units, uint primitive_units, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * content_units, optimization_type, batch)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.Init(0, 0, OpenCL, window, window_key, heads, units_count, window, heads, (content_units + 3), 2, layers, optimization, iBatch)) ReturnFalse; cGEGWA.AddNeckGradient(true); //--- Content Encoder cContentEncoder.Clear(); cContentEncoder.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * content_units, 1, OpenCL, content_size, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 2, OpenCL, window * content_units, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, window * (content_units + 3), optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; //--- Background cBackGround.Clear(); cBackGround.SetOpenCL(OpenCL); neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * 3, 4, OpenCL, content_size, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, window * 3, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.Init(0, 6, OpenCL, window, window_key, heads, heads, primitive_units, content_units, 2, 1, optimization, iBatch)) ReturnFalse; //--- Decoder cDecoder.Clear(); cDecoder.SetOpenCL(OpenCL); CNeuronOCM *ocm = new CNeuronOCM(); if(!ocm || !ocm.Init(0, 7, OpenCL, window, window_key, units_count, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; for(uint i = 0; i < layers; i++) { neuron = cGEGWA.GetInsideLayer(i); ocm = new CNeuronOCM(); if(!ocm || !neuron || !ocm.Init(0, i + 8, OpenCL, window, window_key, neuron.Neurons() / window, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; } //--- Object Cluster Module if(!cOCM.Init(0, layers + 8, OpenCL, window, window_key, primitive_units, heads, window, content_units, heads, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput(), true) || !SetGradient(cOCM.getGradient(), true) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronRefMask feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronRefMask::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!SecondInput) ReturnFalse; //--- Context Encoder CNeuronBaseOCL *context = cContentEncoder[0]; if(context.getOutput() != SecondInput) { if(!context.SetOutput(SecondInput, true)) ReturnFalse; } int content_total = cContentEncoder.Total(); for(int i = 1; i < content_total - 1; i++) { context = cContentEncoder[i]; if(!context || !context.FeedForward(cContentEncoder[i - 1]) ) ReturnFalse; } //--- Background Encoder CNeuronBaseOCL *background = NULL; if(bTrain) { for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.FeedForward(cBackGround[i - 1]) ) ReturnFalse; } } else { background = cBackGround[cBackGround.Total() - 1]; if(!background) ReturnFalse; } CNeuronBaseOCL *neuron = cContentEncoder[content_total - 1]; if(!neuron || !Concat(context.getOutput(), background.getOutput(), neuron.getOutput(), context.Neurons(), background.Neurons(), 1)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.FeedForward(NeuronOCL, neuron.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.FeedForward(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.feedForward(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.feedForward(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.feedForward(decoder, context)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronRefMask | //| layer. | //+------------------------------------------------------------------+ bool CNeuronRefMask::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *neuron = cContentEncoder[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) { if(!neuron.SetGradient(SecondGradient)) ReturnFalse; neuron.SetActivationFunction(SecondActivation); } //--- Object Cluster Module CNeuronBaseOCL *context = cContentEncoder[cContentEncoder.Total() - 2]; if(!cOCM.calcInputGradients(cDecoder[cDecoder.Total() - 1], context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = NULL; for(int i = cDecoder.Total() - 1; i > 0; i--) { decoder = cDecoder[i]; if(!decoder.calcInputGradients(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } decoder = cDecoder[0]; if(!decoder.calcInputGradients(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; //--- Linguistic Primitive Construction CBufferFloat *context_grad = context.getGradient(); if(!context.SetGradient(PrevOutput, false)) ReturnFalse; if(!context.CalcHiddenGradients(cLPC.AsObject()) || !SumAndNormalize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) ) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention neuron = cContentEncoder[cContentEncoder.Total() - 1]; if(!neuron || !NeuronOCL.CalcHiddenGradients((CObject*)GetPointer(cGEGWA), neuron.getOutput(), neuron.getGradient(), (ENUM_ACTIVATION)neuron.Activation())) ReturnFalse; if(!DiversityLoss(neuron, cOCM.GetContextWindow(), neuron.Neurons() / cOCM.GetContextWindow(), true)) ReturnFalse; CNeuronBaseOCL *background = cBackGround[cBackGround.Total() - 1]; if(!background || !DeConcat(context.getGradient(), background.getGradient(), neuron.getGradient(), context.Neurons(), background.Neurons(), 1) || !DeActivation(context.getOutput(), context.getGradient(), context.getGradient(), context.Activation()) || !SumAndNormalize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) || !context.SetGradient(context_grad, false) ) ReturnFalse; //--- Context Encoder for(int i = cContentEncoder.Total() - 3; i >= 0; i--) { context = cContentEncoder[i]; if(!context || !context.CalcHiddenGradients(cContentEncoder[i + 1]) ) ReturnFalse; } //--- Background if(!DeActivation(background.getOutput(), background.getGradient(), background.getGradient(), background.Activation())) ReturnFalse; for(int i = cBackGround.Total() - 2; i > 0; i--) { background = cBackGround[i]; if(!background || !background.CalcHiddenGradients(cBackGround[i + 1]) ) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronRefMask trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronRefMask::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Context Encoder CNeuronBaseOCL *context = NULL; for(int i = 1; i < cContentEncoder.Total() - 1; i++) { context = cContentEncoder[i]; if(!context || !context.UpdateInputWeights(cContentEncoder[i - 1], NULL) ) ReturnFalse; } //--- Background CNeuronBaseOCL *background = NULL; for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.UpdateInputWeights(cBackGround[i - 1]) ) ReturnFalse; } //--- Geometry-Enhaced Group-Word Attention background = cContentEncoder[cContentEncoder.Total() - 1]; if(!cGEGWA.UpdateInputWeights(NeuronOCL, background.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.UpdateInputWeights(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.updateInputWeights(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.updateInputWeights(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.updateInputWeights(decoder, context)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronRefMask GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronRefMask::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cGEGWA.SetOpenCL(OpenCL); cContentEncoder.SetOpenCL(OpenCL); cBackGround.SetOpenCL(OpenCL); cLPC.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); cOCM.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronRefMask state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronRefMask::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cGEGWA.Save(file_handle)) ReturnFalse; if(!cContentEncoder.Save(file_handle)) ReturnFalse; if(!cBackGround.Save(file_handle)) ReturnFalse; if(!cLPC.Save(file_handle)) ReturnFalse; if(!cDecoder.Save(file_handle)) ReturnFalse; if(!cOCM.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRefMask state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronRefMask::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGEGWA.AsObject())) ReturnFalse; if(!cContentEncoder.Load(file_handle)) ReturnFalse; if(!cBackGround.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLPC.AsObject())) ReturnFalse; if(!cDecoder.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOCM.AsObject())) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput()) || !SetGradient(cOCM.getGradient()) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRelativeSelfAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(GELU); idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnits, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iUnits, optimization, iBatch) || !cScale.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 2 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 2 * iWindow, 2 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronRelativeSelfAttention attention out | //| operation. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MHRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronRelativeSelfAttention attention gradient | //| operation. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionGradient(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; int kernel = def_k_MHRelativeAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_ratg_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_ratg_q_g, cQuery.getGradientIndex()) setBuffer(kernel, def_k_ratg_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_ratg_k_g, cKey.getGradientIndex()) setBuffer(kernel, def_k_ratg_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_ratg_v_g, cValue.getGradientIndex()) setBuffer(kernel, def_k_ratg_scores, iScore) setBuffer(kernel, def_k_ratg_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bk_g, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bv_g, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gc_g, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gp_g, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gradient, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradientIndex()) setArgument(kernel, def_k_ratg_kunits, int(cKey.Neurons() / (iHeads * iWindowKey))) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronRelativeSelfAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(NeuronOCL) || !cValue.FeedForward(NeuronOCL) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; if(!MatMul(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cScale[0]).getOutput(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronRelativeSelfAttention layer. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = cScale.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).CalcHiddenGradients(cScale[i + 1])) ReturnFalse; if(!MatMulGrad(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getGradient(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradient(), ((CNeuronBaseOCL*)cScale[0]).getGradient(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).CalcHiddenGradients(cMHAttentionPooling[i + 1])) ReturnFalse; if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronRelativeSelfAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(NeuronOCL) || !cValue.UpdateInputWeights(NeuronOCL) ) ReturnFalse; for(int i = cBKey.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = cBValue.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = cGlobalContentBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; for(int i = cScale.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronRelativeSelfAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeSelfAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau) || !cKey.WeightsUpdate(Source.cKey.AsObject(), tau) || !cValue.WeightsUpdate(Source.cValue.AsObject(), tau) ) ReturnFalse; if(!cBKey.WeightsUpdate(Source.cBKey.AsObject(), tau)) ReturnFalse; if(!cBValue.WeightsUpdate(Source.cBValue.AsObject(), tau)) ReturnFalse; if(!cGlobalContentBias.WeightsUpdate(Source.cGlobalContentBias.AsObject(), tau)) ReturnFalse; if(!cGlobalPositionalBias.WeightsUpdate(Source.cGlobalPositionalBias.AsObject(), tau)) ReturnFalse; if(!cMHAttentionPooling.WeightsUpdate(Source.cMHAttentionPooling.AsObject(), tau)) ReturnFalse; if(!cScale.WeightsUpdate(Source.cScale.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronRelativeSelfAttention GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronRelativeSelfAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnits * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(iUnits * iUnits, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronRelativeSelfAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cBKey.Save(file_handle)) ReturnFalse; if(!cBValue.Save(file_handle)) ReturnFalse; if(!cGlobalContentBias.Save(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Save(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRelativeSelfAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; SetOpenCL(OpenCL); //--- if(!cBKey.Load(file_handle)) ReturnFalse; if(!cBValue.Load(file_handle)) ReturnFalse; if(!cGlobalContentBias.Load(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Load(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Load(file_handle)) ReturnFalse; if(!cScale.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cScale[cScale.Total() - 1]; if(!SetGradient(neuron.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRMAT layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronRMAT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CResidualConv(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, window, units_count, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronRMAT feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronRMAT::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *neuron = cLayers[0]; if(!neuron || !neuron.FeedForward(NeuronOCL)) ReturnFalse; for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron || !neuron.FeedForward(cLayers[i - 1])) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronRMAT | //| layer. | //+------------------------------------------------------------------+ bool CNeuronRMAT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(neuron.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronRMAT trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronRMAT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1])) ReturnFalse; } neuron = cLayers[0]; if(!neuron.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronRMAT weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronRMAT::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRMAT *Source = source; if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronRMAT state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronRMAT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle) || !cLayers.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRMAT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronRMAT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle) || !cLayers.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cLayers[-1]; SetOutput(neuron.getOutput(), true); SetGradient(neuron.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronRMAT GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronRMAT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cLayers.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHAttentionPooling layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindow = window; iUnits = units_count; iHeads = heads; //--- cNeurons.Clear(); cNeurons.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow * iHeads, iWindow * iHeads, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cNeurons.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHAttentionPooling feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.FeedForward(prev) ) ReturnFalse; prev = current; } //--- if(!MatMul(current.getOutput(), NeuronOCL.getOutput(), Output, 1, iHeads, iWindow, iUnits)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHAttentionPooling layer. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *current = cNeurons[cNeurons.Total() - 1]; if(!MatMulGrad(current.getOutput(), current.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, 1, iHeads, iWindow, iUnits) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation()) ) ReturnFalse; for(int i = cNeurons.Total() - 2; i > 0; i--) { current = cNeurons[i]; if(!current.CalcHiddenGradients(cNeurons[i + 1])) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHAttentionPooling trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current;; } //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHAttentionPooling weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHAttentionPooling* Source = source; //--- if(!cNeurons.WeightsUpdate(Source.cNeurons.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHAttentionPooling state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cNeurons.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHAttentionPooling state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); cNeurons.SetOpenCL(OpenCL); //--- if(!cNeurons.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHAttentionPooling GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMHAttentionPooling::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cNeurons.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the activation function of the CNeuronMotifs. | //+------------------------------------------------------------------+ void CNeuronMotifs::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL::SetActivationFunction(value); cMotifs.SetActivationFunction(activation); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMotifs layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMotifs::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { uint inputs = (units_count * step + (window - step)) * dimension; uint motifs = units_count * dimension; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs + motifs, optimization_type, batch)) ReturnFalse; if(!cMotifs.Init(0, 0, OpenCL, dimension * window, dimension * step, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMotifs feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMotifs::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Activation() != activation) SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); if(!cMotifs.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cMotifs.getOutput(), Output, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMotifs | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMotifs::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cMotifs.getGradient(), Gradient, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMotifs trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMotifs::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cMotifs.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMotifs state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMotifs::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cMotifs.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMotifs state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMotifs::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMotifs.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMotifs GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMotifs::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cMotifs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiScaleAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; //--- uint units1 = (iUnits + 1) / 2; uint units2 = (iUnits + 2) / 3; uint units3 = (iUnits + 3) / 4; uint wide = MathMax(MathMax(iUnits, units1 * 2), MathMax(units2 * 3, units3 * 4)); //--- int idx = 0; if(!cWideInputs.Init(0, idx, OpenCL, wide * iWindow, optimization, iBatch)) ReturnFalse; CBufferFloat *temp = cWideInputs.getOutput(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- idx++; if(!cAttentions[0].Init(0, idx, OpenCL, iWindow, window_key, iUnits, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[1].Init(0, idx, OpenCL, 2 * iWindow, window_key, units1, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[2].Init(0, idx, OpenCL, 3 * iWindow, window_key, units2, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[3].Init(0, idx, OpenCL, 4 * iWindow, window_key, units3, heads, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cConcatAttentions.Init(0, idx, OpenCL, 4 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cPooling.Init(0, idx, OpenCL, iWindow, iUnits, 4, optimization, iBatch)) ReturnFalse; //--- SetActivationFunction(None); if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiScaleAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Attention if(!cAttentions[0].FeedForward(NeuronOCL)) ReturnFalse; //--- Copy inputs if(!Concat(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cWideInputs.getOutput(), iWindow, 0, iUnits)) ReturnFalse; if(cWideInputs.Activation() != NeuronOCL.Activation()) cWideInputs.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].FeedForward(cWideInputs.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!Concat(cAttentions[0].getOutput(), cAttentions[1].getOutput(), cAttentions[2].getOutput(), cAttentions[3].getOutput(), cConcatAttentions.getOutput(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention pooling if(!cPooling.FeedForward(cConcatAttentions.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiScaleAttention layer. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Attention pooling if(!cConcatAttentions.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!DeConcat(cAttentions[0].getGradient(), cAttentions[1].getGradient(), cAttentions[2].getGradient(), cAttentions[3].getGradient(), cConcatAttentions.getGradient(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention if(!NeuronOCL.CalcHiddenGradients(cAttentions[0].AsObject())) ReturnFalse; //--- Multi scale attentions for(uint i = 1; i < cAttentions.Size(); i++) { if(!cWideInputs.CalcHiddenGradients(cAttentions[i].AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cWideInputs.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiScaleAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- Attention if(!cAttentions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].UpdateInputWeights(cWideInputs.AsObject())) ReturnFalse; //--- Attention pooling if(!cPooling.UpdateInputWeights(cConcatAttentions.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultiScaleAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultiScaleAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cWideInputs.SetOpenCL(OpenCL); for(int i = 0; i < 4; i++) cAttentions[i].SetOpenCL(OpenCL); cConcatAttentions.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiScaleAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cWideInputs.Save(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!cAttentions[i].Save(file_handle)) ReturnFalse; if(!cConcatAttentions.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiScaleAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cWideInputs.AsObject())) ReturnFalse; for(int i = 0; i < 4; i++) if(!LoadInsideLayer(file_handle, cAttentions[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatAttentions.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMolformer layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMolformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint motif_window, uint motif_step, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronMotifs *motif = new CNeuronMotifs(); uint motif_units = units_count - MathMax(motif_window - motif_step, 0); motif_units = (motif_units + motif_step - 1) / motif_step; if(!motif || !motif.Init(0, idx, OpenCL, window, motif_window, motif_step, motif_units, optimization, iBatch) || !cLayers.Add(motif) ) ReturnFalse; //--- idx++; CNeuronMultiScaleAttention *msat = NULL; CResidualConv *ff = NULL; uint units_total = units_count + motif_units; for(uint i = 0; i < layers; i++) { //--- Attention msat = new CNeuronMultiScaleAttention(); if(!msat || !msat.Init(0, idx, OpenCL, window, window_key, units_total, heads, optimization, iBatch) || !cLayers.Add(msat) ) ReturnFalse; idx++; //--- FeedForward ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units_total, optimization, iBatch) || !cLayers.Add(ff) ) ReturnFalse; idx++; } //--- Out CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, units_total, window, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, units_total, units_total, units_count, window, 1, optimization, iBatch) || !cLayers.Add(conv) ) ReturnFalse; idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; //--- if(!SetOutput(transp.getOutput()) || !SetGradient(transp.getGradient())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRelativeCrossAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iUnitsKV = units_kv; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnitsKV, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling(); if(!pooling || !pooling.Init(0, idx, OpenCL, iWindowKey, iUnits, iHeads, optimization, iBatch) || !cMHAttentionPooling.Add(pooling) ) ReturnFalse; //--- idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(GELU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- Key-Value inputs projection cKVProjection.Clear(); cKVProjection.SetOpenCL(OpenCL); idx++; neuron = new CNeuronBaseOCL; if(!neuron || !neuron.Init(0, idx, OpenCL, window_kv * iUnitsKV, optimization, iBatch) || !cKVProjection.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, window_kv, window_kv, iWindow, iUnitsKV, 1, optimization, iBatch) || !cKVProjection.Add(conv) ) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronRelativeCrossAttention GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronRelativeCrossAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cKVProjection.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnitsKV * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(MathMax(iUnits, iWindow) * iUnitsKV, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Runs the CNeuronRelativeCrossAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron || !SecondInput) ReturnFalse; if(neuron.getOutput() != SecondInput) if(!neuron.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.FeedForward(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(neuron) || !cValue.FeedForward(neuron) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(neuron.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cScale[0]).FeedForward(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronRelativeCrossAttention layer. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL || !SecondGradient) ReturnFalse; CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) if(!neuron.SetGradient(SecondGradient)) ReturnFalse; if(neuron.Activation() != SecondActivation) neuron.SetActivationFunction(SecondActivation); //--- if(!DiversityLoss(AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- CNeuronBaseOCL* next = cScale[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cScale.Total() - 2; i >= 0; i--) { curr = cScale[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } for(int i = cMHAttentionPooling.Total() - 1; i >= 0; i--) { curr = cMHAttentionPooling[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; neuron = cKVProjection[cKVProjection.Total() - 1]; if(!neuron || !MatMulGrad(neuron.getOutput(), neuron.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) || !DiversityLoss(NeuronOCL, iUnits, iWindow, true) ) ReturnFalse; temp = neuron.getGradient(); if(!neuron.SetGradient(GetPointer(cTemp), false) || !neuron.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; for(int i = cKVProjection.Total() - 2; i >= 0; i--) { neuron = cKVProjection[i]; if(!neuron || !neuron.CalcHiddenGradients(cKVProjection[i + 1])) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronRelativeCrossAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *neuron = NULL; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.UpdateInputWeights(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(neuron) || !cValue.UpdateInputWeights(neuron) ) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cScale[0]).UpdateInputWeights(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronRelativeCrossAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronRelativeSelfAttention::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeCrossAttention *Source = source; if(!cKVProjection.WeightsUpdate(GetPointer(Source.cKVProjection), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronRelativeCrossAttention state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Save(const int file_handle) { if(!CNeuronRelativeSelfAttention::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnitsKV) < INT_VALUE) ReturnFalse; if(!cKVProjection.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRelativeCrossAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Load(const int file_handle) { if(!CNeuronRelativeSelfAttention::Load(file_handle)) ReturnFalse; iUnitsKV = (uint)FileReadInteger(file_handle); SetOpenCL(OpenCL); if(!cKVProjection.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMotifEncoder layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMotifEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units_count < 3) ReturnFalse; //--- cLayers.Clear(); //--- int bars_to_paattern = (units_count > 10 ? 3 : 2); CNeuronConvOCL *conv = new CNeuronConvOCL(); int idx = 0; int units = (int)units_count - bars_to_paattern + 1; if(!conv || !conv.Init(0, idx, open_cl, bars_to_paattern * window, window, window, units, 1, optimization_type, batch) || !cLayers.Add(conv) ) ReturnFalse; conv.SetActivationFunction(SIGMOID); idx++; units = units - bars_to_paattern + 1; CNeuronMotifs *motifs = new CNeuronMotifs(); if(!motifs || !motifs.Init(0, idx, open_cl, window, bars_to_paattern, 1, units, optimization_type, batch) || !cLayers.Add(motifs) ) ReturnFalse; motifs.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, motifs.Neurons(), optimization_type, batch)) ReturnFalse; cLayers.SetOpenCL(OpenCL); //--- CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *ff = NULL; units = int(motifs.Neurons() / window); for(uint i = 0; i < layers; i++) { idx++; attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, idx, OpenCL, window, window_key, units, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPropertyAwareAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * properties, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = NULL; CNeuronRelativeSelfAttention *self_attention = NULL; CNeuronRelativeCrossAttention *cross_attention = NULL; CResidualConv *ff = NULL; //--- int idx = 0; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * properties, idx, OpenCL, 1, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; CBufferFloat *temp = neuron.getOutput(); if(!temp.Fill(1)) ReturnFalse; idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, window * properties, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; //--- for(uint i = 0; i < layers; i++) { idx++; self_attention = new CNeuronRelativeSelfAttention(); if(!self_attention || !self_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, optimization, iBatch) || !cLayers.Add(self_attention) ) { DeleteObj(self_attention); ReturnFalse; } idx++; cross_attention = new CNeuronRelativeCrossAttention(); if(!cross_attention || !cross_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, window, units_count, optimization, iBatch) || !cLayers.Add(cross_attention) ) { DeleteObj(cross_attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, properties, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPropertyAwareAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- if(bTrain) { neuron = cLayers[1]; if(!neuron || !neuron.FeedForward(cLayers[0])) ReturnFalse; } for(int i = 2; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.FeedForward(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPropertyAwareAttention layer. | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(cTemp.GetIndex() < 0 || cTemp.Total() < NeuronOCL.Neurons()) { cTemp.BufferFree(); if(!cTemp.BufferInit(NeuronOCL.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; } if(!NeuronOCL.getGradient() || !NeuronOCL.getGradient().Fill(0)) ReturnFalse;; //--- CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i > 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1], NeuronOCL.getOutput(), GetPointer(cTemp), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(neuron.Type() == defNeuronRelativeCrossAttention) if(!SumAndNormalize(NeuronOCL.getGradient(), GetPointer(cTemp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPropertyAwareAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAMCT layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronAMCT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- int idx = 0; if(!cAtomEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; idx++; if(!cMotifEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; //--- cMotifProjection.Clear(); cMotifProjection.SetOpenCL(OpenCL); int motifs = int(cMotifEncoder.Neurons() / window); idx++; CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, motifs, window, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, motifs, motifs, units_count, 1, window, optimization, iBatch) || !cMotifProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cPropertyDecoder.Init(0, idx, OpenCL, window, window_key, properties, motifs, heads, layers, optimization, iBatch)) ReturnFalse; //--- cPropertyProjection.Clear(); cPropertyProjection.SetOpenCL(OpenCL); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, properties, window, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, properties, properties, units_count, 1, window, optimization, iBatch) || !cPropertyProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cConcatenate.Init(0, idx, OpenCL, 3 * window * units_count, optimization, iBatch)) ReturnFalse; idx++; if(!cPooling.Init(0, idx, OpenCL, window, units_count, 3, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput(), true) || !SetGradient(cPooling.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAMCT feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronAMCT::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cAtomEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.FeedForward(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Concatenate uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); prev = cMotifProjection[cMotifProjection.Total() - 1]; if(!Concat(cAtomEncoder.getOutput(), prev.getOutput(), current.getOutput(), cConcatenate.getOutput(), window, window, window, units)) ReturnFalse; //--- Out if(!cPooling.FeedForward(cConcatenate.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronAMCT | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAMCT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); CNeuronBaseOCL *motifs = cMotifProjection[cMotifProjection.Total() - 1]; CNeuronBaseOCL *prop = cPropertyProjection[cPropertyProjection.Total() - 1]; if(!motifs || !prop || !DeConcat(cAtomEncoder.getGradient(), motifs.getGradient(), prop.getGradient(), cConcatenate.getGradient(), window, window, window, units)) ReturnFalse; //--- if(cAtomEncoder.Activation() != None) if(!DeActivation(cAtomEncoder.getOutput(), cAtomEncoder.getGradient(), cAtomEncoder.getGradient(), cAtomEncoder.Activation())) ReturnFalse; if(motifs.Activation() != None) if(!DeActivation(motifs.getOutput(), motifs.getGradient(), motifs.getGradient(), motifs.Activation())) ReturnFalse; if(prop.Activation() != None) if(!DeActivation(prop.getOutput(), prop.getGradient(), prop.getGradient(), prop.Activation())) ReturnFalse; //--- if(!motifs.calcAlignmentGradient(cAtomEncoder.AsObject(), true)) ReturnFalse; for(int i = cMotifProjection.Total() - 2; i >= 0; i--) { motifs = cMotifProjection[i]; if(!motifs || !motifs.CalcHiddenGradients(cMotifProjection[i + 1])) ReturnFalse; } //--- for(int i = cPropertyProjection.Total() - 2; i >= 0; i--) { prop = cPropertyProjection[i]; if(!prop || !prop.CalcHiddenGradients(cPropertyProjection[i + 1])) ReturnFalse; } //--- if(!cPropertyDecoder.CalcHiddenGradients(cPropertyProjection[0]) || !cMotifEncoder.CalcHiddenGradients(cPropertyDecoder.AsObject())) ReturnFalse; CBufferFloat *temp = cMotifEncoder.getGradient(); if(!cMotifEncoder.SetGradient(cConcatenate.getGradient(), false) || !cMotifEncoder.CalcHiddenGradients(cMotifProjection[0]) || !SumAndNormalize(temp, cMotifEncoder.getGradient(), temp, window, false, 0, 0, 0, 1) || !cMotifEncoder.SetGradient(temp, false) || !NeuronOCL.CalcHiddenGradients(cMotifEncoder.AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cConcatenate.getGradient(), false) || !NeuronOCL.CalcHiddenGradients(cAtomEncoder.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, window, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAMCT trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronAMCT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cAtomEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.UpdateInputWeights(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Out if(!cPooling.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronAMCT GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronAMCT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cAtomEncoder.SetOpenCL(OpenCL); cMotifEncoder.SetOpenCL(OpenCL); cMotifProjection.SetOpenCL(OpenCL); cPropertyDecoder.SetOpenCL(OpenCL); cPropertyProjection.SetOpenCL(OpenCL); cConcatenate.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronAMCT state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAMCT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAtomEncoder.Save(file_handle)) ReturnFalse; if(!cMotifEncoder.Save(file_handle)) ReturnFalse; if(!cMotifProjection.Save(file_handle)) ReturnFalse; if(!cPropertyDecoder.Save(file_handle)) ReturnFalse; if(!cPropertyProjection.Save(file_handle)) ReturnFalse; if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAMCT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAMCT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cAtomEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMotifEncoder.AsObject())) ReturnFalse; if(!cMotifProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPropertyDecoder.AsObject())) ReturnFalse; if(!cPropertyProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNAFS feature smoothing operation. | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothing(const CNeuronBaseOCL* neuron, const CNeuronBaseOCL* smoothing) { if(!OpenCL || !neuron || !neuron.getOutput() || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothing; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getOutputIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getOutputIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!smoothing.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronNAFS feature smoothing gradient operation. | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothingGradient(const CNeuronBaseOCL* neuron, const CNeuronBaseOCL* smoothing) { if(!OpenCL || !neuron || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothingGradient; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getGradientIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getGradientIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!neuron.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronNAFS layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronNAFS::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint smoothing, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units_count, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iSmoothing = smoothing; iUnits = units_count; //--- if(!cFeatureSmoothing.Init(0, 0, OpenCL, (iSmoothing + 1)*iUnits * iDimension, optimization, iBatch)) ReturnFalse; cFeatureSmoothing.SetActivationFunction(None); if(!cTranspose.Init(0, 1, OpenCL, (iSmoothing + 1)*iUnits, iDimension, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); if(!cDistance.Init(0, 2, OpenCL, (iSmoothing + 1)*iUnits, optimization, iBatch)) ReturnFalse; cDistance.SetActivationFunction(None); if(!cAdaptation.Init(0, 3, OpenCL, cDistance.Neurons(), optimization, iBatch)) ReturnFalse; cAdaptation.SetActivationFunction(None); cAdaptation.SetHeads(iUnits); //--- SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronNAFS feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronNAFS::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!FeatureSmoothing(NeuronOCL, cFeatureSmoothing.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cFeatureSmoothing.AsObject())) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cAdaptation.FeedForward(cDistance.AsObject())) ReturnFalse; if(!MatMul(cAdaptation.getOutput(), cFeatureSmoothing.getOutput(), Output, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronNAFS | //| layer. | //+------------------------------------------------------------------+ bool CNeuronNAFS::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cAdaptation.getOutput(), cAdaptation.getGradient(), cFeatureSmoothing.getOutput(), cFeatureSmoothing.getPrevOutput(), Gradient, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; if(!cDistance.CalcHiddenGradients(cAdaptation.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getGradient(), cDistance.getGradient(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cFeatureSmoothing.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormalize(cFeatureSmoothing.getGradient(), cFeatureSmoothing.getPrevOutput(), cFeatureSmoothing.getGradient(), iDimension, false, 0, 0, 0, 1) ) ReturnFalse; if(!FeatureSmoothingGradient(NeuronOCL, cFeatureSmoothing.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cFeatureSmoothing.getPrevOutput(), NeuronOCL.getGradient(), iDimension, false, 0, 0, 0, 1) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation()) ) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronNAFS GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronNAFS::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cFeatureSmoothing.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cAdaptation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Saves the CNeuronNAFS state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronNAFS::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSmoothing)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; //--- Save objects if(!cFeatureSmoothing.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cAdaptation.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronNAFS state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronNAFS::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSmoothing = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); //--- Load objects if(!LoadInsideLayer(file_handle, cFeatureSmoothing.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAdaptation.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Sets the activation function of the CNeuronRMAT. | //+------------------------------------------------------------------+ void CNeuronRMAT::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL *neuron = cLayers[-1]; if(!neuron) return; neuron.SetActivationFunction(value); CNeuronBaseOCL::SetActivationFunction((ENUM_ACTIVATION)neuron.Activation()); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTransposeVRCOCL layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, count * window, variables, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, variables, count * window, optimization, iBatch)) ReturnFalse; if(!cTransposeRCD.Init(0, 1, OpenCL, count, window, variables, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTransposeVRCOCL feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeRCD.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- return(CNeuronTransposeOCL::feedForward(cTransposeRCD.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronTransposeVRCOCL layer. | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronTransposeOCL::calcInputGradients(cTransposeRCD.AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cTransposeRCD.AsObject())) ReturnFalse; //--- return(prevLayer.CalcHiddenGradients(cTranspose.AsObject())); } //+------------------------------------------------------------------+ //| Saves the CNeuronTransposeVRCOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cTransposeRCD.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTransposeVRCOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeRCD.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTransposeVRCOCL GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronTransposeVRCOCL::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cTransposeRCD.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHConvOCL layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronProofOCL::Init(numOutputs, myIndex, open_cl, window, step, units_count * window_out * variables, ADAM, batch)) ReturnFalse; //--- iWindowOut = window_out; iVariables = variables; iHeads = MathMax(MathMin(heads, window), 1); //--- const int window_h = int((iWindow + heads - 1) / heads); const int count = (int)((window_h + 1) * iWindowOut * iVariables); if(!WeightsConv) { WeightsConv = new CBufferFloat(); if(!WeightsConv) ReturnFalse; } if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(window_h + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!FirstMomentumConv) { FirstMomentumConv = new CBufferFloat(); if(!FirstMomentumConv) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SecondMomentumConv) { SecondMomentumConv = new CBufferFloat(); if(!SecondMomentumConv) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(DeltaWeightsConv); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHConvOCL feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3]; global_work_size[0] = Output.Total() / (iWindowOut * iVariables); global_work_size[1] = iHeads; global_work_size[2] = iVariables; //--- ResetLastError(); const int kernel = def_k_FeedForwardMHConv; setBuffer(kernel, def_k_ffc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffс_window_out, (int)iWindowOut) setArgument(kernel, def_k_ffc_activation, (int)activation) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMHConvOCL | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iVariables; global_work_size[1] = iVariables; ResetLastError(); const int kernel = def_k_CalcHiddenGradientMHConv; setBuffer(kernel, def_k_chgc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_g, Gradient.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_chgc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgc_step, (int)iStep) setArgument(kernel, def_k_chgc_window_in, (int)iWindow) setArgument(kernel, def_k_chgc_window_out, (int)iWindowOut) setArgument(kernel, def_k_chgc_activation, (int)NeuronOCL.Activation()) setArgument(kernel, def_k_chgc_shift_out, (int)0) setArgument(kernel, def_k_chgc_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHConvOCL trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { WeightsConv.Total() }; float lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); ResetLastError(); const int kernel = def_k_UpdateWeightsMHConvAdam; setBuffer(kernel, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwca_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwca_l, lt) setArgument(kernel, def_k_uwca_b1, b1) setArgument(kernel, def_k_uwca_b2, b2) setArgument(kernel, def_k_uwca_window_in, (int)iWindow) setArgument(kernel, def_k_uwca_window_out, (int)iWindowOut) setArgument(kernel, def_k_uwca_step, (int)iStep) setArgument(kernel, def_k_uwca_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif t++; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHConvOCL state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!FileWriteInteger(file_handle, (int)iHeads)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHConvOCL state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHFeedForward layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- if(!acConvolutions[0].Init(0, 0, OpenCL, window, window, window_out, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[0].SetActivationFunction(GELU); //--- if(!acConvolutions[1].Init(0, 1, OpenCL, window_out, window_out, window, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[1].SetActivationFunction(None); //--- if(!SetGradient(acConvolutions[1].getGradient(), true)) ReturnFalse; SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHFeedForward feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::feedForward(CNeuronBaseOCL* NeuronOCL) { CObject *prev = NeuronOCL; for(uint i = 0; i < acConvolutions.Size(); i++) { if(!acConvolutions[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(acConvolutions[i]); } //--- if(!SumAndNormalize(NeuronOCL.getOutput(), acConvolutions[acConvolutions.Size() - 1].getOutput(), Output, acConvolutions[0].GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHFeedForward layer. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = (int)acConvolutions.Size() - 2; i >= 0; i--) { if(!acConvolutions[i].CalcHiddenGradients(acConvolutions[i + 1].AsObject())) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(acConvolutions[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), Gradient, NeuronOCL.Activation()) || !SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHFeedForward trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { for(int i = (int)acConvolutions.Size() - 1; i > 0; i--) { if(!acConvolutions[i].UpdateInputWeights(acConvolutions[i - 1].AsObject())) ReturnFalse; } if(!acConvolutions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHFeedForward weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMHFeedForward* Source = source; if(acConvolutions.Size() != Source.acConvolutions.Size()) ReturnFalse; for(int i = 0; i < (int)acConvolutions.Size(); i++) if(!acConvolutions[i].WeightsUpdate(Source.acConvolutions[i].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHFeedForward state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!acConvolutions[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHFeedForward state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!LoadInsideLayer(file_handle, acConvolutions[i].AsObject())) ReturnFalse; //--- if(!SetGradient(acConvolutions[acConvolutions.Size() - 1].getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHFeedForward GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMHFeedForward::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < acConvolutions.Size(); i++) acConvolutions[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDMHAttention layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronMHFeedForward *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronCrossDMHAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronRelativeCrossAttention *cross = NULL; CNeuronMHFeedForward *conv = NULL; bool use_self = units_count > 0; int layer = 0; for(uint i = 0; i < layers; i++) { if(use_self) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, layer, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } layer++; } cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, layer, OpenCL, window, window_key, units_count, heads, window_cross, units_cross, optimization, iBatch) || !cLayers.Add(cross) ) { DeleteObj(cross); ReturnFalse; } layer++; conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, layer, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } layer++; } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronCrossDMHAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CObject *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cLayers.Total(); i++) { current = cLayers[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronCrossDMHAttention layer. | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None ) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(PrevOutput.Total() != SecondGradient.Total()) { PrevOutput.BufferFree(); if(!PrevOutput.BufferInit(SecondGradient.Total(), 0) || !PrevOutput.BufferCreate(OpenCL)) ReturnFalse; } if(!SecondGradient.Fill(0) || !PrevOutput.Fill(0)) ReturnFalse; //--- CNeuronBaseOCL *next = cLayers[-1]; CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { current = cLayers[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, PrevOutput, SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; next = current; } //--- if(!NeuronOCL.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, PrevOutput, SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronCrossDMHAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { current = cLayers[i]; if(!current || !current.UpdateInputWeights(cLayers[i - 1], SecondInput)) ReturnFalse; } //--- if(!((CNeuronBaseOCL*)cLayers[0]).UpdateInputWeights(NeuronOCL, SecondInput)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDilatedCasualConv layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronDilatedCasualConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint dimension, uint units_count, uint variables, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 1, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); uint units = units_count; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint i = 0; i < layers; i++) { s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, i * 2, OpenCL, dimension, dimension * units * variables, optimization, iBatch) || !cLayers.Add(s3)) { DeleteObj(s3); ReturnFalse; } s3.SetActivationFunction(None); //--- conv = new CNeuronConvOCL(); units = MathMax((units - window + step) / step, 1); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window * dimension, step * dimension, dimension, units, variables, optimization, iBatch) || !cLayers.Add(conv)) { if(!!conv) DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(GELU); } //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, OpenCL, conv.Neurons(), optimization_type, batch)) ReturnFalse; //--- if(!SetGradient(conv.getGradient(), true) || !SetOutput(conv.getOutput(), true)) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMultiScaleRelativeSelfAttention attention out | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMultiScaleRelativeCrossAttention attention | //| out operation. | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeCrossAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRecursiveAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint history_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMultiScaleRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, window_key, history_size, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cSelfAttention.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, optimization, iBatch)) ReturnFalse; index++; if(!cTransposeSA.Init(0, index, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cConvolution.Init(0, index, OpenCL, iUnits, iUnits, iWindowKey, 1, iWindow, optimization, iBatch)) ReturnFalse; index++; uint windows[] = { iWindowKey * iWindow }; if(!cHistory.Init(0, index, OpenCL, iUnitsKV, iWindowKey, windows)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronRecursiveAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeSA.FeedForward(cSelfAttention.AsObject())) ReturnFalse; if(!cConvolution.FeedForward(cTransposeSA.AsObject())) ReturnFalse; if(!cHistory.FeedForward(cConvolution.AsObject())) ReturnFalse; //--- return(CNeuronMultiScaleRelativeCrossAttention::feedForward(cSelfAttention.AsObject(), cHistory.getOutput())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronRecursiveAttention layer. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMultiScaleRelativeCrossAttention::calcInputGradients(cSelfAttention.AsObject(), cHistory.getOutput(), cHistory.getGradient(), (ENUM_ACTIVATION)cHistory.Activation())) ReturnFalse; if(!cConvolution.CalcHiddenGradients(cHistory.AsObject())) ReturnFalse; if(!cTransposeSA.CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; CBufferFloat *temp = cSelfAttention.getGradient(); if(!cSelfAttention.SetGradient(cTransposeSA.getPrevOutput(), false) || !cSelfAttention.CalcHiddenGradients(cTransposeSA.AsObject()) || !SumAndNormalize(temp, cSelfAttention.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cSelfAttention.SetGradient(temp, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSelfAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronRecursiveAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMultiScaleRelativeCrossAttention::updateInputWeights(cSelfAttention.AsObject(), cHistory.getOutput())) ReturnFalse; if(!cHistory.UpdateInputWeights(cConvolution.AsObject())) ReturnFalse; if(!cConvolution.UpdateInputWeights(cTransposeSA.AsObject())) ReturnFalse; if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronRecursiveAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Save(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; if(!cTransposeSA.Save(file_handle)) ReturnFalse; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cHistory.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRecursiveAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Load(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeSA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistory.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronRecursiveAttention execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Clear(void) { if(!CNeuronMultiScaleRelativeCrossAttention::Clear()) ReturnFalse; if(!cSelfAttention.Clear()) ReturnFalse; if(!cTransposeSA.Clear()) ReturnFalse; if(!cConvolution.Clear()) ReturnFalse; if(!cHistory.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronRecursiveAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronRecursiveAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMultiScaleRelativeCrossAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); cTransposeSA.SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cHistory.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronLinerAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iVariables = variables; //--- int index = 0; if(!cQuery.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(SIGMOID); index++; if(!cKey.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(TANH); index++; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction(TANH); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronLinerAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL)) ReturnFalse; if(!cKey.FeedForward(NeuronOCL) || !cKeyT.FeedForward(cKey.AsObject())) ReturnFalse; if(!MatMul(cKeyT.getOutput(), NeuronOCL.getOutput(), cKeyValue.getOutput(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!MatMul(cQuery.getOutput(), cKeyValue.getOutput(), cAttentionOut.getOutput(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttentionOut.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronLinerAttention layer. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cQuery.getOutput(), cQuery.getGradient(), cKeyValue.getOutput(), cKeyValue.getGradient(), cAttentionOut.getGradient(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!MatMulGrad(cKeyT.getOutput(), cKeyT.getGradient(), NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cKeyValue.getGradient(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!SumAndNormalize(Gradient, cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(cKeyT.Activation() != None) if(!DeActivation(cKeyT.getOutput(), cKeyT.getGradient(), cKeyT.getGradient(), cKeyT.Activation())) ReturnFalse; if(cQuery.Activation() != None) if(!DeActivation(cQuery.getOutput(), cQuery.getGradient(), cQuery.getGradient(), cQuery.Activation())) ReturnFalse; //--- if(!cKey.CalcHiddenGradients(cKeyT.AsObject()) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronLinerAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKey.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronLinerAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronLinerAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau)) ReturnFalse; if(!cKey.WeightsUpdate(Source.cKey.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronLinerAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronLinerAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (int)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; //--- int index = 2; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction((ENUM_ACTIVATION)cKey.Activation()); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronLinerAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronLinerAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cKeyT.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronComplexMVMHMaskAttention layer and | //| allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(window_key, 1); iUnits = units_count; iHeads = MathMax(heads, 1); iVariables = variables; //--- uint index = 0; if(!cMask.Init(0, index, OpenCL, 2 * iWindow, 2 * iWindow, iVariables * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cMask.SetActivationFunction(SIGMOID); CBufferFloat *temp = cMask.GetWeightsConv(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- index++; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(None); index++; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cPooling.Init(0, index, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(LReLU); index++; if(!cFeedForward[1].Init(0, index, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMVMHMaskAttention attention out | //| operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables/*Q units*/, iVariables/*K units*/, iHeads * iUnits/*Heads*/}; uint local_work_size[3] = {1, iVariables, 1}; ResetLastError(); int kernel = def_k_MaskAttentionComplex; setBuffer(kernel, def_k_maskattcom_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcom_out, cMHAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_masks, cMask.getOutputIndex()) setArgument(kernel, def_k_maskattcom_dimension, (int)iWindowKey) setArgument(kernel, def_k_maskattcom_heads_kv, (int)(iHeads * iUnits)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMVMHMaskAttention attention inside | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables, iWindowKey, iHeads * iUnits}; ResetLastError(); int kernel = def_k_MaskAttentionGradientsComplex; setBuffer(kernel, def_k_maskattcomgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcomgr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_mask_g, cMask.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_maskattcomgr_kunits, (int)iVariables) setArgument(kernel, def_k_maskattcomgr_heads_kv, (int)(iHeads * iUnits)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQ.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronComplexMVMHMaskAttention sum and normalize | //| operation. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::SumAndNormalize(CBufferFloat* tensor1, CBufferFloat* tensor2, CBufferFloat* out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f ) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; int kernel = def_k_MatrixSum; setBuffer(kernel, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_sum_matrix_out, out.GetIndex()) setArgument(kernel, def_k_sum_dimension, 2 * dimension) setArgument(kernel, def_k_sum_shift_in1, 2 * shift_in1) setArgument(kernel, def_k_sum_shift_in2, 2 * shift_in2) setArgument(kernel, def_k_sum_shift_out, 2 * shift_out) setArgument(kernel, def_k_sum_multiplyer, multiplyer) setBuffer(kernel, def_k_sum_numeric_flags, out.GetIndex()) setArgument(kernel, def_k_sum_flags_enabled, 0) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- if(!normilize) return(true); //--- kernel = def_k_ComplexNormalize; setBuffer(kernel, def_k_cn_inputs, out.GetIndex()) setBuffer(kernel, def_k_cn_outputs, out.GetIndex()) setBuffer(kernel, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(kernel, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(kernel, def_k_cn_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronComplexMVMHMaskAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; //---cMask.getOutput().BufferRead(); if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //---cQKV.getOutput().BufferRead(); if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; //---cPooling.getOutput().BufferRead(); if(!SumAndNormalize(NeuronOCL.getOutput(), cPooling.getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; //---cFeedForward[0].getOutput().BufferRead(); if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; //---cFeedForward[1].getOutput().BufferRead(); if(!SumAndNormalize(cResidual.getOutput(), cFeedForward[1].getOutput(), getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronComplexMVMHMaskAttention layer. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; //--- if(!DeActivation(cPooling.getOutput(), cPooling.getGradient(), cResidual.getGradient(), cPooling.Activation()) || !DeActivation(cPooling.getOutput(), cPooling.getPrevOutput(), Gradient, cPooling.Activation()) || !SumAndNormalize(cPooling.getGradient(), cPooling.getPrevOutput(), cPooling.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; //--- if(!DeActivation(cQKV.getOutput(), cQKV.getPrevOutput(), cQKV.getGradient(), cQKV.Activation()) || !prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getPrevOutput(), cResidual.getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getGradient(), Gradient, prevLayer.Activation()) || !SumAndNormalize(cResidual.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(cMask.getOutput(), cMask.getGradient(), cMask.getGradient(), cMask.Activation()) || !prevLayer.CalcHiddenGradients(cMask.AsObject()) || !SumAndNormalize(prevLayer.getGradient(), cResidual.getPrevOutput(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronComplexMVMHMaskAttention trainable weights | //| from the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronComplexMVMHMaskAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronComplexMVMHMaskAttention *Source = source; if(!cMask.WeightsUpdate(GetPointer(Source.cMask), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(GetPointer(Source.cPooling), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFeedForward[i].WeightsUpdate(GetPointer(Source.cFeedForward[i]), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronComplexMVMHMaskAttention state to the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!cFeedForward[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronComplexMVMHMaskAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!LoadInsideLayer(file_handle, cFeedForward[i].AsObject())) ReturnFalse; //--- uint index = 2; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronComplexMVMHMaskAttention GPU buffers to the | //| given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronComplexMVMHMaskAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); for(uint i = 0; i < cFeedForward.Size(); i++) cFeedForward[i].SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHProbAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, window, window, units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(5, window_key); iHeads = MathMax(1, heads); iUnits = units_count; iTopKQuerys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); iRandomKeys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(TANH); index++; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cRandomK.Init(0, index, OpenCL, iHeads * MathMax(iRandomKeys, iTopKQuerys), optimization, iBatch)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, iTopKQuerys * iHeads * iWindowKey, optimization, iBatch)) ReturnFalse; index++; if(!cPooling.Init(0, index, OpenCL, iHeads * iWindowKey, iHeads * iWindowKey, iWindow, iTopKQuerys, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(TANH); index++; if(!cTranspose[0].Init(0, index, OpenCL, iTopKQuerys, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cScaling.Init(0, index, OpenCL, iTopKQuerys, iTopKQuerys, iUnits, iWindow, optimization, iBatch)) ReturnFalse; cScaling.SetActivationFunction(None); index++; if(!cTranspose[1].Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHProbAttention random keys operation. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::RandomKeys(CBufferFloat* indexes, int random, int units, int heads) { if(!indexes || random > units || indexes.Total() < (random * heads) ) ReturnFalse; //--- matrix ind = matrix::Zeros(random, heads); if(random == units) { for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = (float)r; } } else { double step = double(units) / random; for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = float(int((r + MathRand() / 32767.0) * step)); } } if(!indexes.AssignArray(ind) || !indexes.BufferWrite()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHProbAttention query importance operation. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::QueryImportance(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iRandomKeys, iHeads}; uint local_work_size[3] = {1, iRandomKeys, 1}; uint kernel = def_k_ProbAttentionQeuryImp; setBuffer(kernel, def_k_probat_querys, cQ.getOutputIndex()) setBuffer(kernel, def_k_probat_keys_values, cKV.getOutputIndex()) setBuffer(kernel, def_k_probat_index_keys, cRandomK.getOutputIndex()) setBuffer(kernel, def_k_probat_querys_imp, cQ.getPrevOutIndex()) setArgument(kernel, def_k_probat_dimension, int(iWindowKey)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQ.getPrevOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHProbAttention top k indexes operation. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::TopKIndexes(void) { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iHeads}; uint kernel = def_k_TopKImportanceToIndex; setBuffer(kernel, def_k_imptoind_importance, cQ.getPrevOutIndex()) setBuffer(kernel, def_k_imptoind_indexes, cRandomK.getPrevOutIndex()) setArgument(kernel, def_k_imptoind_tok_k, int(iTopKQuerys)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cRandomK.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHProbAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionOut(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; uint kernel = def_k_QIndexAttention; setBuffer(kernel, def_k_indatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indatt_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indatt_scores, ibScore) setBuffer(kernel, def_k_indatt_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_indatt_dimension, int(iWindowKey)) setArgument(kernel, def_k_indatt_heads_kv, int(iHeads)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHProbAttention attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionInsideGradients(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint kernel = def_k_QIndexAttentionGradients; setBuffer(kernel, def_k_indattgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indattgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_indattgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indattgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_indattgr_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indattgr_scores, ibScore) setBuffer(kernel, def_k_indattgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_indattgr_kunits, int(iUnits)) setArgument(kernel, def_k_indattgr_heads_kv, int(iHeads)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHProbAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(!RandomKeys(cRandomK.getOutput(), iRandomKeys, iUnits, iHeads)) ReturnFalse; if(!QueryImportance() || !TopKIndexes()) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!cTranspose[0].FeedForward(cPooling.AsObject())) ReturnFalse; if(!cScaling.FeedForward(cTranspose[0].AsObject())) ReturnFalse; if(!cTranspose[1].FeedForward(cScaling.AsObject())) ReturnFalse; if(!SumAndNormalize(cTranspose[1].getOutput(), NeuronOCL.getOutput(), cTranspose[1].getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return(CResidualConv::feedForward(cTranspose[1].AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHProbAttention layer. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cQ.getGradient().Fill(0)) ReturnFalse; //--- if(!CResidualConv::calcInputGradients(cTranspose[1].AsObject())) ReturnFalse; //--- if(!cScaling.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!cTranspose[0].CalcHiddenGradients(cScaling.AsObject())) ReturnFalse; if(!cPooling.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(cQKV.Activation() != None) if(!DeActivation(cQKV.getOutput(), cQKV.getGradient(), cQKV.getGradient(), cQKV.Activation())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cTranspose[1].getGradient(), cTranspose[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(cTranspose[1].getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHProbAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cScaling.UpdateInputWeights(cTranspose[0].AsObject())) ReturnFalse; //--- return(CResidualConv::updateInputWeights(cTranspose[1].AsObject())); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHProbAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iTopKQuerys)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iRandomKeys)) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cRandomK.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; if(!cTranspose[0].Save(file_handle)) ReturnFalse; if(!cTranspose[1].Save(file_handle)) ReturnFalse; if(!cScaling.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHProbAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Load(const int file_handle) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- if(!CResidualConv::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iTopKQuerys = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iRandomKeys = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRandomK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMHAttentionOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScaling.AsObject())) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHProbAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMHProbAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- CResidualConv::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cRandomK.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); cTranspose[0].SetOpenCL(OpenCL); cTranspose[1].SetOpenCL(OpenCL); cScaling.SetOpenCL(OpenCL); //--- if(!!OpenCL) ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHProbAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CResidualConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHProbAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(Source.cPooling.AsObject(), tau)) ReturnFalse; if(!cScaling.WeightsUpdate(Source.cScaling.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMaskMultiWinConv layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint windows_total, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint win = window * windows_total + MathMax(windows_total, 1) - 1; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, win, win, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; iWindowsTotal = windows_total; iWindow = window; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMaskMultiWinConv feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!OpenCL || !Output || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowOut, iVariables}; const int kernel = def_k_FeedForwardMaskMultWinConv; setBuffer(kernel, def_k_ffmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_o, getOutputIndex()) setArgument(kernel, def_k_ffmmwc_activation, Activation()) setArgument(kernel, def_k_ffmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_ffmmwc_windows_total, (int)iWindowsTotal) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMaskMultiWinConv layer. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput || !NeuronOCL.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowsTotal, iVariables}; const int kernel = def_k_CalcHiddenGradientMaskMultWinConv; setBuffer(kernel, def_k_chgmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_chgmmwc_masks_g, SecondGradient.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_chgmmwc_activation, NeuronOCL.Activation()) setArgument(kernel, def_k_chgmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_chgmmwc_window_out, (int)iWindowOut) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMaskMultiWinConv trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* second) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !second) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {(iWindow + 1)*iWindowsTotal, iWindowOut, iVariables}; const int kernel = def_k_UpdateWeightsMaskMultWinConvAdam; setBuffer(kernel, def_k_uwmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmmwc_masks, second.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_v, SecondMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_uwmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_windows_total, (int)iWindowsTotal) setArgument(kernel, def_k_uwmmwc_l, lr) setArgument(kernel, def_k_uwmmwc_b1, b1) setArgument(kernel, def_k_uwmmwc_b2, b2) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMaskMultiWinConv state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowsTotal) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMaskMultiWinConv state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindowsTotal = (uint)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronRoPE layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronRoPE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window % 2 > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iVariables = variables; //--- matrix pe = matrix::Zeros(count, iWindow); vector position = vector::Ones(count); position = position.CumSum() - 1; for(uint i = 0; i < iWindow / 2; i++) { vector temp = position / MathPow(10000.0f, 2.0f * i / window); pe.Col(MathCos(temp), i * 2); pe.Col(MathSin(temp), i * 2 + 1); } PositionEncoder.BufferFree(); if(!PositionEncoder.AssignArray(pe)) ReturnFalse; SetActivationFunction(None); //--- return(PositionEncoder.BufferCreate(open_cl)); } //+------------------------------------------------------------------+ //| Runs the CNeuronRoPE feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronRoPE::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_RoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronRoPE | //| layer. | //+------------------------------------------------------------------+ bool CNeuronRoPE::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_CalcHiddenGradRoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronRoPE state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronRoPE::Save(const int file_handle) { if(!CNeuronPositionEncoder::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronRoPE state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronRoPE::Load(const int file_handle) { if(!CNeuronPositionEncoder::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = uint(FileReadInteger(file_handle)); //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAttentNorm layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * period * variables, optimization_type, batch)) ReturnFalse; CNeuronBaseOCL::SetActivationFunction(None); //--- iCount = units_count; iVariables = variables; iPeriod = period; //--- if(!cAttention.Init(0, 0, OpenCL, iPeriod * iVariables, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); //--- if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronAttentNorm attent norm operation. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, MathMin(iPeriod, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atn_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronAttentNorm attent norm grad operation. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMin(iCount, iPeriod), MathMin(iPeriod * iCount, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atng_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAttentNorm feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cAttention.FeedForward()) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; } //--- return(AttentNorm(NeuronOCL)); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronAttentNorm layer. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!AttentNormGrad(NeuronOCL)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAttentNorm trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return(cAttention.UpdateInputWeights()); } //+------------------------------------------------------------------+ //| Averages the CNeuronAttentNorm weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAttentNorm* Source = source; return(cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau)); } //+------------------------------------------------------------------+ //| Saves the CNeuronAttentNorm state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPeriod)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAttentNorm state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iPeriod = (uint)FileReadInteger(file_handle); //--- if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronAttentNorm GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSAttentNorm layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count, variables, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSAttentNorm attent norm operation. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { 1, MathMin(iWindow, uint(OpenCL.GetMaxLocalSize(1))), iCount}; uint local_work_size[3] = { 1, global_work_size[1], 1}; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, cNorm.getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iWindow, iCount)) setArgument(kernel, def_k_atn_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSAttentNorm attent norm grad operation. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMax(iCount, iWindow), MathMin(MathMax(iWindow, iCount), uint(OpenCL.GetMaxLocalSize(1))), iCount }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, cNorm.getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iCount, iWindow)) setArgument(kernel, def_k_atng_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSAttentNorm feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cAttention.getOutput(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; if(!AttentNorm(cTranspose.AsObject())) ReturnFalse; //--- return(CNeuronTransposeOCL::feedForward(cNorm.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSAttentNorm layer. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!AttentNormGrad(cTranspose.AsObject())) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getPrevOutput(), cAttention.getGradient(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!SumAndNormalize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), getPrevOutput(), NeuronOCL.getGradient(), iCount, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSAttentNorm state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSAttentNorm GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronTQMHA layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint period, uint timeframe, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, heads, units_count, window, units_count, optimization_type, batch)) ReturnFalse; //--- FF[0].SetActivationFunction(GELU); //--- iTimeframe = MathMax(1, timeframe); if(!cParams.Init(0, 0, OpenCL, window * units_count, period, optimization, iBatch)) ReturnFalse; if(!cParams.Zeros()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronTQMHA feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!SecondInput) ReturnFalse; //--- int pos = int(SecondInput[0]); pos = (pos / int(iTimeframe)) % cParams.GetPeriod(); if(!cParams.SetPosition(pos) || !cParams.FeedForward()) ReturnFalse; //--- if(!Q_Embedding.FeedForward(cParams.AsObject())) ReturnFalse; //--- if(!KV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronTQMHA | //| layer. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; //--- if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!prevLayer.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!cParams.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow_K, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronTQMHA trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!cParams.UpdateInputWeights()) ReturnFalse; if(!Q_Embedding.UpdateInputWeights(cParams.AsObject())) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronTQMHA weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronCrossAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTQMHA* Source = source; if(!cParams.WeightsUpdate(Source.cParams.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronTQMHA state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Save(const int file_handle) { if(!CNeuronCrossAttention::Save(file_handle)) ReturnFalse; //--- if(!cParams.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iTimeframe)) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronTQMHA state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Load(const int file_handle) { if(!CNeuronCrossAttention::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iTimeframe = (uint)FileReadInteger(file_handle); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronTQMHA GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronTQMHA::SetOpenCL(COpenCLMy *obj) { CNeuronCrossAttention::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSNSMHAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!sparse >= 1 || sparse < 0) ReturnFalse; fSparse = sparse; //--- if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, heads, heads, 1, units * m_units, 1, optimization_type, batch)) ReturnFalse; int index = 0; if(!cNeighbors.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; CBufferFloat* temp = cNeighbors.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cRamdomCandidates.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cProjection[0].Init(index, 0, OpenCL, window, window, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(SoftPlus); index++; if(!cProjection[1].Init(index, 0, OpenCL, 2 * heads, 2 * heads, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(TANH); index++; if(!cScores.Init(0, index, OpenCL, units * m_units * heads, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSNSMHAttention significant neighbors sampling | //| operation. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SignificantNeighborsSampling(CNeuronBaseOCL *NeuronOCL) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint window = cProjection[0].GetWindow(); //--- if(!NeuronOCL || NeuronOCL.Neurons() != (units * window)) ReturnFalse; if(!cNeighbors.SwapOutputs()) ReturnFalse; CBufferFloat *temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, float(units))) ReturnFalse; //--- uint global_work_offset[2] = { 0 }; uint global_work_size[2] = { units, m_units }; uint local_work_size[2] = { 1, m_units }; //--- uint kernel = def_k_SignificantNeighborsSampling; setBuffer(kernel, def_k_sns_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_sns_random_cands, cRamdomCandidates.getOutputIndex()) setBuffer(kernel, def_k_sns_candidates, cNeighbors.getPrevOutIndex()) setBuffer(kernel, def_k_sns_neighbors, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_sns_dimension, window) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cNeighbors.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSNSMHAttention sparse mh scores operation. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScores(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScores; setBuffer(kernel, def_k_smhs_data, cProjection[1].getOutputIndex()) setBuffer(kernel, def_k_smhs_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_indexes, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_smhs_sparse, fSparse) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cNeighbors.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronSNSMHAttention sparse mh scores grad | //| operation. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScoresGrad(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScoresGrad; setBuffer(kernel, def_k_smhs_gr_data_gr, cProjection[1].getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_gr_scores_gr, cScores.getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_indexes, cNeighbors.getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cProjection[1].getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSNSMHAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!SignificantNeighborsSampling(NeuronOCL)) ReturnFalse; CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].FeedForward(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!SparseMHScores()) ReturnFalse; if(!CNeuronConvOCL::feedForward(cScores.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSNSMHAttention layer. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cScores.AsObject())) ReturnFalse; if(!SparseMHScoresGrad()) ReturnFalse; //--- int total = (int)cProjection.Size(); CNeuronBaseOCL* inputs = NULL; for(int i = total - 1; i >= 0; i--) { inputs = (i > 0 ? cProjection[i - 1].AsObject() : prevLayer); if(!inputs.CalcHiddenGradients(cProjection[i].AsObject())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSNSMHAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].UpdateInputWeights(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!CNeuronConvOCL::updateInputWeights(cScores.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSNSMHAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSNSMHAttention* Source = source; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].WeightsUpdate(Source.cProjection[i].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSNSMHAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteFloat(file_handle, fSparse) < sizeof(float)) ReturnFalse; //--- if(!cNeighbors.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSNSMHAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; fSparse = FileReadFloat(file_handle); //--- if(!LoadInsideLayer(file_handle, cNeighbors.AsObject())) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!LoadInsideLayer(file_handle, cProjection[i].AsObject())) ReturnFalse; //--- int index = 1; if(!cRamdomCandidates.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; index += int(cProjection.Size()); if(!cScores.Init(0, index, OpenCL, Neurons() * iWindow, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSNSMHAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronSNSMHAttention::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cNeighbors.SetOpenCL(OpenCL); cRamdomCandidates.SetOpenCL(OpenCL); for(uint i = 0; i < cProjection.Size(); i++) cProjection[i].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGlobalLocalAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint dimension_k, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHFeedForward::Init(numOutputs, myIndex, open_cl, window, 2 * window, units, 1, heads, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cMask.Init(0, index, OpenCL, units, window, heads, m_units, sparse, optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, window, window, 2 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, window, window, 4 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, 2 * dimension_k * heads, 2 * dimension_k * heads, window, units, 1, optimization, iBatch)) ReturnFalse; cW0.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronGlobalLocalAttention global local attention | //| operation. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttention(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttention; setBuffer(kernel, def_k_glatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_out, cMHAttention.getOutputIndex()) setArgument(kernel, def_k_glatt_dimension, dimension_k) setArgument(kernel, def_k_glatt_total_kv, units) setArgument(kernel, def_k_glatt_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttention.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronGlobalLocalAttention global local attention | //| grad operation. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttentionGrad(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttentionGrad; setBuffer(kernel, def_k_glatt_gr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_q_gr, cQ.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_kv_gr, cKV.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask_gr, cMask.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_gr_out_gr, cMHAttention.getGradientIndex()) setArgument(kernel, def_k_glatt_gr_dimension, dimension_k) setArgument(kernel, def_k_glatt_gr_total_q, units) setArgument(kernel, def_k_glatt_gr_total_kv, units) setArgument(kernel, def_k_glatt_gr_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQ.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGlobalLocalAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; if(!cQ.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalLocalAttention()) ReturnFalse; if(!cW0.FeedForward(cMHAttention.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cResidual.getOutput(), cW0.GetFilters(), true, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return(CNeuronMHFeedForward::feedForward(cResidual.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronGlobalLocalAttention layer. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMHFeedForward::calcInputGradients(cResidual.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cResidual.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttention.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!GlobalLocalAttentionGrad()) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQ.AsObject())) ReturnFalse; if(!DeActivation(cResidual.getOutput(), cResidual.getGradient(), cResidual.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cKV.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMask.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGlobalLocalAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQ.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return(CNeuronMHFeedForward::updateInputWeights(cResidual.AsObject())); } //+------------------------------------------------------------------+ //| Averages the CNeuronGlobalLocalAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGlobalLocalAttention* Source = source; if(!cMask.WeightsUpdate(Source.cMask.AsObject(), tau)) ReturnFalse; if(!cQ.WeightsUpdate(Source.cQ.AsObject(), tau)) ReturnFalse; if(!cKV.WeightsUpdate(Source.cKV.AsObject(), tau)) ReturnFalse; if(!cW0.WeightsUpdate(Source.cW0.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronGlobalLocalAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Save(const int file_handle) { if(!CNeuronMHFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cMask.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGlobalLocalAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Load(const int file_handle) { if(!CNeuronMHFeedForward::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- int index = 3; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index += 2; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGlobalLocalAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronGlobalLocalAttention::SetOpenCL(COpenCLMy *obj) { CNeuronMHFeedForward::SetOpenCL(obj); //--- cMask.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronExtralonger layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint heads, uint dimension_k, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronMLMHAttentionOCL *att = NULL; CNeuronGlobalLocalAttention *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, layers, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, att_layers, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, variables, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronExtralonger feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Concatenate if(!Concat(cTimeModule[-1].getOutput(), cMixModule[-1].getOutput(), cSpatialModule[-1].getOutput(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- return(CNeuronMHAttentionPooling::feedForward(cConcatResults.AsObject())); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronExtralonger layer. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcatResults.AsObject())) ReturnFalse; //--- DeConcatenate if(!cTimeModule[-1] || !cMixModule[-1] || !cSpatialModule[-1] || !DeConcat(cTimeModule[-1].getGradient(), cMixModule[-1].getGradient(), cSpatialModule[-1].getGradient(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; //--- Spatial Module for(int i = cSpatialModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cSpatialModule[i - 1] : NeuronOCL); next = cSpatialModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Mix Module for(int i = cMixModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cMixModule[i - 1] : cProjectionT[-1]); next = cMixModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Time Module CBufferFloat *temp = current.getGradient(); if(!current.SetGradient(current.getPrevOutput(), false)) ReturnFalse; for(int i = cTimeModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cTimeModule[i - 1] : cProjectionT[-1]); next = cTimeModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormalize(temp, current.getGradient(), temp, 1, false, 0, 0, 0, 1) || !current.SetGradient(temp, false)) ReturnFalse; //--- Time projection temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; for(int i = cProjectionT.Total() - 1; i >= 0; i--) { current = (i > 0 ? cProjectionT[i - 1] : NeuronOCL); next = cProjectionT[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, SecondGradient, SecondActivation)) ReturnFalse; } if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronExtralonger trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return(CNeuronMHAttentionPooling::updateInputWeights(cConcatResults.AsObject())); } //+------------------------------------------------------------------+ //| Averages the CNeuronExtralonger weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronExtralonger *Source = source; if(!cProjectionT.WeightsUpdate(Source.cProjectionT.AsObject(), tau)) ReturnFalse; if(!cTimeModule.WeightsUpdate(Source.cTimeModule.AsObject(), tau)) ReturnFalse; if(!cMixModule.WeightsUpdate(Source.cMixModule.AsObject(), tau)) ReturnFalse; if(!cSpatialModule.WeightsUpdate(Source.cSpatialModule.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronExtralonger state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cProjectionT.Save(file_handle)) ReturnFalse; if(!cTimeModule.Save(file_handle)) ReturnFalse; if(!cSpatialModule.Save(file_handle)) ReturnFalse; if(!cMixModule.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronExtralonger state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!cProjectionT.Load(file_handle)) ReturnFalse; if(!cTimeModule.Load(file_handle)) ReturnFalse; if(!cSpatialModule.Load(file_handle)) ReturnFalse; if(!cMixModule.Load(file_handle)) ReturnFalse; //--- int index = cProjectionT.Total() + cTimeModule.Total() + cSpatialModule.Total() + cMixModule.Total(); if(!cConcatResults.Init(0, index, OpenCL, iHeads * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronExtralonger GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronExtralonger::SetOpenCL(COpenCLMy *obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cProjectionT.SetOpenCL(OpenCL); cTimeModule.SetOpenCL(OpenCL); cSpatialModule.SetOpenCL(OpenCL); cMixModule.SetOpenCL(OpenCL); cConcatResults.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGraphAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units * units, optimization, iBatch)) ReturnFalse; cScores.SetHeads(units); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronGraphAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!MatMul(cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronGraphAttention layer. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGraphAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronGraphAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronGraphAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGraphAttention state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 2; if(!cScores.Init(0, index, OpenCL, cGraphs.Neurons(), optimization, iBatch)) ReturnFalse; cScores.SetHeads(cValue.GetUnits()); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGraphAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronGraphAttention layer. | //+------------------------------------------------------------------+ void CNeuronGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSparseGraphAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units, units, sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSparseGraphAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!SparseMatMul(cScores.GetIndexes(), cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSparseGraphAttention layer. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!SparseMatMulGrad(cScores.GetIndexes(), cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSparseGraphAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSparseGraphAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSparseGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSparseGraphAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSparseGraphAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScores.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 3; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSparseGraphAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronSparseGraphAttention layer. | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronExtralongerGraph layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronExtralongerGraph::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint experts, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronGraphAttention *att = NULL; CNeuronGlobLocGraphAtt *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); for(uint i = 0; i < layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; for(uint i = 0; i < att_layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, experts, sparse, emb_dimension, uint(sparse * (dimension + emb_dimension)), optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, variables, dimension + emb_dimension, experts, sparse, emb_dimension, m_units, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronGateLineAttention execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cState.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronGateLineAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint dimension, uint dimension_k, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConv::Init(numOutputs, myIndex, open_cl, heads * dimension_k, heads * dimension_k, dimension, variables, 1, optimization_type, batch)) ReturnFalse; //--- iDimensionK = dimension_k; iHeads = heads; iVariables = variables; //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, dimension, dimension, 3 * iHeads * iDimensionK, iVariables, 1, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(SIGMOID); index++; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cGate.Init(0, index, OpenCL, dimension, dimension, iDimensionK * iHeads, iVariables, 1, optimization, iBatch)) ReturnFalse; cGate.SetActivationFunction(SIGMOID); index++; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return(Clear()); } //+------------------------------------------------------------------+ //| Runs the CNeuronGateLineAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cGate.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!DeConcat(cQ.getOutput(), cK.getOutput(), cV.getOutput(), cQKV.getOutput(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; //--- if(!MatMul(cK.getOutput(), cV.getOutput(), cKV.getOutput(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!cState.SwapOutputs()) ReturnFalse; if(!DiagMatMul(cGate.getOutput(), cState.getPrevOutput(), cState.getOutput(), iDimensionK, iDimensionK, iHeads * iVariables, cState.Activation())) ReturnFalse; if(!SumAndNormalize(cState.getOutput(), cKV.getOutput(), cState.getOutput(), iDimensionK, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cQ.getOutput(), cState.getOutput(), cMHAttention.getOutput(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!CNeuronSpikeConv::feedForward(cMHAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronGateLineAttention layer. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConv::calcInputGradients(cMHAttention.AsObject())) ReturnFalse; //--- if(!MatMulGrad(cQ.getOutput(), cQ.getGradient(), cState.getOutput(), cState.getGradient(), cMHAttention.getGradient(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!DeActivation(cKV.getOutput(), cKV.getGradient(), cState.getGradient(), cKV.Activation())) ReturnFalse; if(!DiagMatMulGrad(cGate.getOutput(), cGate.getGradient(), cState.getPrevOutput(), cKV.getPrevOutput(), cState.getGradient(), iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cGate) if(!MatMulGrad(cK.getOutput(), cK.getGradient(), cV.getOutput(), cV.getGradient(), cKV.getGradient(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!Concat(cQ.getGradient(), cK.getGradient(), cV.getGradient(), cQKV.getGradient(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cQKV) //--- if(!NeuronOCL.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGate.AsObject())) ReturnFalse; if(!SumAndNormalize(PrevOutput, temp, temp, GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronGateLineAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGate.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConv::updateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronGateLineAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGateLineAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cGate.WeightsUpdate(Source.cGate.AsObject(), tau)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronGateLineAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Save(const int file_handle) { if(!CNeuronSpikeConv::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cGate.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iDimensionK) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronGateLineAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Load(const int file_handle) { if(!CNeuronSpikeConv::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGate.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iDimensionK = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- int index = 1; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index += 2; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); cState.Clear(); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return(Clear()); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronGateLineAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronGateLineAttention::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConv::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cGate.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cV.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cState.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHFAT layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window, 2 * window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFields() * cKV_Embedding.GetFields() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHFAT feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHFAT attention operation. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Attention(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFields(); uint total_k = cKV_Embedding.GetFields(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, total_k, iHeads}; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFAT; setBuffer(kernel, def_k_mhfat_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_mhfat_dimension, dimension) setArgument(kernel, def_k_mhfat_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHFAT attention gradients operation. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::AttentionGradients(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFields(); uint total_k = cKV_Embedding.GetFields(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, (uint)MathMin(MathMax(total_k, MathMax(total_q, dimension)), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFATGrad; setBuffer(kernel, def_k_mhfat_gr_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_q_gr, cQ_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_kv_gr, cKV_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_scale_gr, cScale.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_gr_gradients, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_mhfat_gr_dimension, dimension) setArgument(kernel, def_k_mhfat_gr_total_k, total_k) setArgument(kernel, def_k_mhfat_gr_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cKV_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cQ_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cScale.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronMHFAT | //| layer. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHFAT trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHFAT state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iHeads), INT_VALUE) < INT_VALUE) ReturnFalse; if(!cQ_Embedding.Save(file_handle)) ReturnFalse; if(!cKV_Embedding.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHFAT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(!LoadInsideLayer(file_handle, cQ_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; uint index = 3; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index += 2; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHFAT weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHFAT::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHFAT* Source = source; dWeightsUpdate(cQ_Embedding, Source, tau); dWeightsUpdate(cKV_Embedding, Source, tau); dWeightsUpdate(cScale, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHFAT GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMHFAT::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL && ibScoreIndex >= 0) { OpenCL.BufferFree(ibScoreIndex); ibScoreIndex = INVALID_HANDLE; } //--- CNeuronMSRes::SetOpenCL(obj); //--- cQ_Embedding.SetOpenCL(OpenCL); cKV_Embedding.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMHFAT layer. | //+------------------------------------------------------------------+ void CNeuronMHFAT::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); //--- cQ_Embedding.TrainMode(bTrain); cKV_Embedding.TrainMode(bTrain); cScale.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHCrossFAT layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint window_cross, uint fields_cross, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window_cross, 2 * window_key * iHeads, fields_cross, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFields() * cKV_Embedding.GetFields() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- index++; if(!cContext.Init(0, index, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFields(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHCrossFAT feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(cContext.getOutput() != Context) if(!cContext.SetOutput(Context, true)) ReturnFalse; //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(cContext.AsObject())) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHCrossFAT layer. | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondGradient) ReturnFalse; if(cContext.getGradient() != SecondGradient) if(!cContext.SetGradient(SecondGradient, true)) ReturnFalse; cContext.SetActivationFunction(SecondActivation); //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cContext.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHCrossFAT trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHCrossFAT state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Load(const int file_handle) { if(!CNeuronMHFAT::Load(file_handle)) ReturnFalse; if(!cContext.Init(0, 6, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFields(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHCrossFAT GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMHCrossFAT::SetOpenCL(COpenCLMy *obj) { CNeuronMHFAT::SetOpenCL(obj); cContext.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Initializes the CNeuronAutoToken layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint embed_size, uint units_in, uint groups, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, embed_size * topK * groups, optimization_type, batch)) ReturnFalse; activation = None; //--- iEmbedingSize = embed_size; iTopK = topK; iUnits = units_in; iGroups = groups; //--- cDNN.Clear(); cW.Clear(); cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); //--- CNeuronFieldAwareConv* conv = NULL; CFieldAwareParams* params = NULL; CNeuronSparseSoftMax* softmax = NULL; uint index = 0; //--- DNN conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, window_in, 2 * embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * embed_size, embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); //--- W index++; params = new CFieldAwareParams(); if(!params || !params.Init(0, index, OpenCL, units_in, groups, (embed_size + 3) / 4, candidates, (topK + 2) / 3, optimization, iBatch) || !cW.Add(params)) DeleteObjAndFalse(params); params.SetActivationFunction(None); index++; softmax = new CNeuronSparseSoftMax(); if(!softmax || !softmax.Init(0, index, OpenCL, groups, units_in, topK, optimization, iBatch) || !cW.Add(softmax)) DeleteObjAndFalse(softmax); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronAutoToken feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).FeedForward()) ReturnFalse; } else if(!curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; if(!SparseConcatenate(softmax.GetIndexes(), softmax.getOutput(), cDNN[-1].getOutput(), Output, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronAutoToken | //| layer. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cW[-1] || !cDNN[-1] || cW[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cW[-1]; if(!SparseConcatenateGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cDNN[-1].getOutput(), cDNN[-1].getGradient(), Gradient, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; Deactivation(cDNN[-1]); //--- CNeuronBaseOCL* next = cDNN[-1]; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = cDNN.Total() - 2; i >= 0; i--) { curr = cDNN[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- W next = cW[-1]; for(int i = cW.Total() - 2; i >= 0; i--) { curr = cW[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronAutoToken trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).UpdateInputWeights()) ReturnFalse; } else if(!curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronAutoToken state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iEmbedingSize); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, iUnits); dFileWriteUInt(file_handle, iGroups); //--- if(!cDNN.Save(file_handle)) ReturnFalse; if(!cW.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronAutoToken state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iEmbedingSize); dFileReadUInt(file_handle, iTopK); dFileReadUInt(file_handle, iUnits); dFileReadUInt(file_handle, iGroups); //--- if(!cDNN.Load(file_handle)) ReturnFalse; if(!cW.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronAutoToken weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAutoToken* Source = source; dWeightsUpdate(cDNN, Source, tau); dWeightsUpdate(cW, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronAutoToken execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cDNN.ClearStates()) ReturnFalse; if(!cW.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronAutoToken GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronAutoToken::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronAutoToken layer. | //+------------------------------------------------------------------+ void CNeuronAutoToken::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cDNN.TrainMode(bTrain); cW.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMultiMixAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension < heads || dimension % heads > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * dimension, optimization_type, batch)) ReturnFalse; activation = None; if(!cParams.Init(0, 0, OpenCL, units * units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMultiMixAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- if(bTrain) if(!cParams.FeedForward()) ReturnFalse; //--- uint heads = cParams.GetFields(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMul(cParams.getOutput(), NeuronOCL.getOutput(), Output, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), Output, Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMultiMixAttention layer. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint heads = cParams.GetFields(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMulGrad(cParams.getOutput(), cParams.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMultiMixAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMultiMixAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMultiMixAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMultiMixAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; dWeightsUpdate(cParams, ((CNeuronMultiMixAttention*)source), tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMultiMixAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMultiMixAttention layer. | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMTmixAttBlock layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cAttention.Init(0, index, OpenCL, dimension, units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint expert_dimension = dimension / heads; uint hidden_dimension = MathMax((expert_dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, expert_dimension, hidden_dimension, windows, steps, heads, units, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMTmixAttBlock feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint dimension = cAttention.GetDimension(); if(!SumAndNormalize(cSharedExpert[-1].getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cAttention.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMTmixAttBlock layer. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(cAttention.getOutput(), PrevOutput, Gradient, cAttention.Activation())) ReturnFalse; uint dimension = cAttention.GetDimension(); if(!cAttention.CalcHiddenGradients(cSharedExpert[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, cAttention.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cAttention.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMTmixAttBlock trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMTmixAttBlock state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cAttention.Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMTmixAttBlock state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMTmixAttBlock weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMTmixAttBlock* Source = source; dWeightsUpdate(cAttention, Source, tau); dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMTmixAttBlock GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cAttention.SetOpenCL(OpenCL); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMTmixAttBlock layer. | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cAttention.TrainMode(bTrain); cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CCrossMHFlashAttention layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint heads, bool mask_future, uint q_units, uint kv_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * heads * q_units, optimization_type, batch)) ReturnFalse; activation = None; //--- if(kv_units < 1) ReturnFalse; //--- iQUnits = q_units; iKVUnits = kv_units; iHeads = heads; iDimension = dimension; bMask = mask_future; //--- if(!cLogSumExp.BufferInit(q_units * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CCrossMHFlashAttention feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iKVUnits, iDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttention; setBuffer(kernel, def_k_mhflat_q, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhflat_kv, Context.GetIndex()) setBuffer(kernel, def_k_mhflat_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflat_out, getOutputIndex()) setArgument(kernel, def_k_mhflat_dimension, iDimension) setArgument(kernel, def_k_mhflat_total_kv, iKVUnits) setArgument(kernel, def_k_mhflat_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CCrossMHFlashAttention layer. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !prevLayer.getOutput() || !prevLayer.getGradient() || !SecondInput || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iKVUnits), iDimension, iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttentionGrad; setBuffer(kernel, def_k_mhflatg_q, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhflatg_q_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhflatg_kv, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhflatg_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflatg_out, getOutputIndex()) setBuffer(kernel, def_k_mhflatg_out_gr, getGradientIndex()) setArgument(kernel, def_k_mhflatg_dimension, iDimension) setArgument(kernel, def_k_mhflatg_total_q, iQUnits) setArgument(kernel, def_k_mhflatg_total_kv, iKVUnits) setArgument(kernel, def_k_mhflatg_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CCrossMHFlashAttention state to the file handle. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iKVUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CCrossMHFlashAttention state from the file handle. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iKVUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, bMask); //--- //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CCrossMHFlashAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CCrossMHFlashAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronBaseOCL::SetOpenCL(obj); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Reports checked-transfer support for CrossMHFlashAttention. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::SupportsOpenCLChecked(void) const { //--- Reject subtypes with an unverified ownership tree. return(Type() == defCrossMHFlashAttention); } //+------------------------------------------------------------------+ //| Transfers logsumexp before publishing the base context. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::SetOpenCLChecked(COpenCLMy *obj) { //--- Validate the target and exact ownership contract before mutation. if(CheckPointer(obj) == POINTER_INVALID || !SupportsOpenCLChecked()) ReturnFalseEx("checked context target or flash ownership"); //--- Preserve allocated logsumexp data from its current device owner. if(cLogSumExp.Total() > 0 && cLogSumExp.GetIndex() >= 0 && cLogSumExp.GetOpenCL() != obj && !cLogSumExp.BufferRead()) ReturnFalseEx("logsumexp read"); //--- Do not overwrite already transferred logsumexp from a stale host cache. if(cLogSumExp.Total() > 0 && (cLogSumExp.GetOpenCL() != obj || cLogSumExp.GetIndex() < 0) && !cLogSumExp.BufferCreate(obj)) ReturnFalseEx("logsumexp create"); //--- Publish the inherited context only after logsumexp transfer succeeds. if(!CNeuronBaseOCL::SetOpenCLChecked(obj)) ReturnFalseEx("base context transfer"); //--- Finalize without resetting the attention configuration or cache. return(true); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMixFeedForward layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint hidden_dimension = MathMax((dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension, hidden_dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, dimension, hidden_dimension, windows, steps, units, 1, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMixFeedForward feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- CNeuronFieldAwareConv* conv = cSharedExpert[-1]; uint dimension = conv.GetFilters(); if(!SumAndNormalize(conv.getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, NeuronOCL.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMixFeedForward layer. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; CNeuronFieldAwareConv* conv = cSharedExpert[0]; uint dimension = conv.GetWindow(); if(!NeuronOCL.CalcHiddenGradients(conv) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMixFeedForward trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMixFeedForward state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMixFeedForward state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMixFeedForward weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMixFeedForward* Source = source; dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMixFeedForward GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMixFeedForward layer. | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronOneTrans layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; //--- if(!CNeuronMixFeedForward::Init(numOutputs, myIndex, open_cl, dimensions[count - 1], units_out, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; CNeuronMultiWindowsConvOCL* mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(None); index++; CNeuronFieldPatternEmbedding* emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(SIGMOID); //--- index++; if(!cSequenceLast.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cSequenceLast.SetActivationFunction(None); index++; if(!cKVSequenceLast.Init(0, index, OpenCL, embed_size, 2 * embed_size * heads, units_s, candidates, topK, optimization, iBatch)) ReturnFalse; cKVSequenceLast.SetActivationFunction(SIGMOID); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cStackKVSequence.Init(0, index, OpenCL, stack_size, 2 * embed_size * heads, units_s, optimization, iBatch)) ReturnFalse; index++; //--- uint units_ns = windows.Size() - units_s; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, (units_ns + stack_size)*embed_size, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; //--- CNeuronAutoToken* select = NULL; CNeuronSparseMoEConv* query = NULL; CCrossMHFlashAttention* attention = NULL; CNeuronSpikeConv* W0 = NULL; CNeuronMTmixAttBlock* ff = NULL; uint units_in = units_ns + stack_size; for(uint i = 0; i < layers; i++) { select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, embed_size, units_in, (layers - i)*units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; units_in = (layers - i) * units_out; query = new CNeuronSparseMoEConv(); if(!query || !query.Init(0, index, OpenCL, embed_size * topK, embed_size * heads, units_in, candidates, topK, optimization, iBatch) || !cFlow.Add(query)) DeleteObjAndFalse(query); query.SetActivationFunction(SIGMOID); index++; attention = new CCrossMHFlashAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, heads, false, units_in, stack_size, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; W0 = new CNeuronSpikeConv(); if(!W0 || !W0.Init(0, index, OpenCL, embed_size * heads, embed_size * heads, embed_size, units_in, 1, optimization, iBatch) || !cFlow.Add(W0)) DeleteObjAndFalse(W0); index++; ff = new CNeuronMTmixAttBlock(); if(!ff || !ff.Init(0, index, OpenCL, embed_size, units_in, heads, embed_size, candidates, topK, optimization, iBatch) || !cFlow.Add(ff)) DeleteObjAndFalse(ff); index++; } //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronOneTrans feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!DeConcat(cSequenceLast.getOutput(), prev.getPrevOutput(), prev.getOutput(), cSequenceLast.Neurons(), prev.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.FeedForward(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(curr.getPrevOutput(), cStackSequence.getOutput(), prev.getOutput(), prev.Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronOneTrans | //| layer. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMixFeedForward::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CBufferFloat* grad = cStackKVSequence.getGradient(); if(!grad.Fill(0)) ReturnFalse; CBufferFloat* out = cStackKVSequence.getOutput(); CBufferFloat* temp = cStackKVSequence.getPrevOutput(); ENUM_ACTIVATION act = (ENUM_ACTIVATION)cStackKVSequence.Activation(); CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(cFlow[i + 1], out, temp, act)) ReturnFalse; if(cFlow[i + 1].Type() == defCrossMHFlashAttention) if(!SumAndNormalize(temp, grad, grad, 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- curr = cPrepare[-1]; if(!curr || !DeConcat(curr.getPrevOutput(), cStackSequence.getGradient(), cFlow[0].getGradient(), cFlow[0].Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.CalcHiddenGradients(cStackKVSequence.AsObject())) ReturnFalse; if(!cSequenceLast.CalcHiddenGradients(cKVSequenceLast.AsObject())) ReturnFalse; temp = cSequenceLast.getGradient(); if(!cSequenceLast.SetGradient(cSequenceLast.getPrevOutput(), false) || !cSequenceLast.CalcHiddenGradients(cStackSequence.AsObject()) || !SumAndNormalize(temp, cSequenceLast.getGradient(), temp, 1, false, 0, 0, 0, 1) || !cSequenceLast.SetGradient(temp, false)) ReturnFalse; if(!Concat(cSequenceLast.getGradient(), curr.getPrevOutput(), curr.getGradient(), cSequenceLast.Neurons(), curr.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; Deactivation(curr); for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(cPrepare[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronOneTrans trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cKVSequenceLast.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.UpdateInputWeights(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronOneTrans weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMixFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronOneTrans* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cKVSequenceLast, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cStackKVSequence, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronOneTrans state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Save(const int file_handle) { //--- if(!CNeuronMixFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cSequenceLast.Save(file_handle)) ReturnFalse; if(!cKVSequenceLast.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cStackKVSequence.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronOneTrans state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Load(const int file_handle) { //--- if(!CNeuronMixFeedForward::Load(file_handle)) ReturnFalse; //--- if(!cPrepare.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKVSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackKVSequence.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronOneTrans execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Clear(void) { //--- if(!CNeuronMixFeedForward::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; if(!cSequenceLast.Clear()) ReturnFalse; if(!cKVSequenceLast.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cStackKVSequence.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronOneTrans GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronOneTrans::SetOpenCL(COpenCLMy *obj) { CNeuronMixFeedForward::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cSequenceLast.SetOpenCL(OpenCL); cKVSequenceLast.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cStackKVSequence.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronOneTrans layer. | //+------------------------------------------------------------------+ void CNeuronOneTrans::TrainMode(bool flag) { CNeuronMixFeedForward::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cFlow.TrainMode(bTrain); cSequenceLast.TrainMode(bTrain); cKVSequenceLast.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cStackKVSequence.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronMHTHCrossAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Init(numOutputs, myIndex, open_cl, dimension_q, units_q, heads, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iQUnits = units_q; iXUnits = unit_x; iHeads = heads; iXDimension = dimension_x; bMask = false; //--- cPrepareQ.Clear(); cW0.Clear(); cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); //--- Query uint index = 0; uint head_size = (dimension_q + heads - 1) / heads; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_q, dimension_q, heads * head_size, units_q, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, head_size, head_size, dimension_x, units_q * heads, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); //--- W0 index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, dimension_x * units_q * heads, optimization, iBatch) || !cW0.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_x, dimension_x, head_size, units_q * heads, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, heads * head_size, heads * head_size, dimension_q, units_q, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); //--- if(!cLogSumExp.BufferInit(units_q * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHTHCrossAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !cW0[0] || !cW0[0].getOutput() || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iXUnits, iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCA; setBuffer(kernel, def_k_mhstca_query, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhstca_X, Context.GetIndex()) setBuffer(kernel, def_k_mhstca_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_output, cW0[0].getOutputIndex()) setArgument(kernel, def_k_mhstca_dimension, iXDimension) setArgument(kernel, def_k_mhstca_total_X, iXUnits) setArgument(kernel, def_k_mhstca_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) dActivation(cW0[0]); #ifdef _DEBUG if(!cW0[0].getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronMHTHCrossAttention attention inside gradients | //| operation. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !cW0[0] || !cW0[0].getOutput() || !prevLayer.getOutput() || !SecondInput || !cW0[0].getGradient() || !prevLayer.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iXUnits), (uint)MathMin( MathMax( MathMax(iQUnits * iHeads, iXUnits), iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCAGrad; setBuffer(kernel, def_k_mhstca_gr_query, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_query_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhstca_gr_X, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_X_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_output, cW0[0].getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_output_gr, cW0[0].getGradientIndex()) setArgument(kernel, def_k_mhstca_gr_dimension, iXDimension) setArgument(kernel, def_k_mhstca_gr_total_q, iQUnits) setArgument(kernel, def_k_mhstca_gr_total_X, iXUnits) setArgument(kernel, def_k_mhstca_gr_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cW0[0].getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronMHTHCrossAttention feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!AttentionOut(prev, Context)) ReturnFalse; prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), prev.getOutput(), prev.Neurons() / iQUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMTmixAttBlock::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronMHTHCrossAttention layer. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronMTmixAttBlock::calcInputGradients(cW0[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cW0.Total() - 2; i >= 0; i--) { curr = cW0[i]; if(!curr || !curr.CalcHiddenGradients(cW0[i + 1])) ReturnFalse; } //--- if(!AttentionInsideGradients(cPrepareQ[-1], SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- for(int i = cPrepareQ.Total() - 2; i >= 0; i--) { curr = cPrepareQ[i]; if(!curr || !curr.CalcHiddenGradients(cPrepareQ[i + 1])) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(curr)) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), cW0[-1].getPrevOutput(), cW0[-1].getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cW0[-1].getPrevOutput(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(prevLayer.getGradient(), cW0[-1].getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronMHTHCrossAttention trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronMTmixAttBlock::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronMHTHCrossAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Save(const int file_handle) { if(!CNeuronMTmixAttBlock::Save(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iXUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iXDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronMHTHCrossAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Load(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Load(file_handle)) ReturnFalse; if(!cW0.Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iXUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iXDimension); uint mask = 0; dFileReadUInt(file_handle, mask); bMask = bool(mask); //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronMHTHCrossAttention weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMTmixAttBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHTHCrossAttention* Source = source; dWeightsUpdate(cPrepareQ, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronMHTHCrossAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronMTmixAttBlock::SetOpenCL(obj); //--- cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronMHTHCrossAttention layer. | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::TrainMode(bool flag) { CNeuronMTmixAttBlock::TrainMode(flag); //--- cPrepareQ.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSTCA layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSTCA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; uint dimension_out = dimensions[count - 1]; uint units_ns = count - units_s - 1; uint bottleneck = (embed_size + topK - 1) / topK; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck * topK, bottleneck * topK, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Prepare arrays cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvOCL* mwc = NULL; CNeuronFieldPatternEmbedding* emb = NULL; CNeuronAutoToken* select = NULL; CNeuronSwiGLUOCL* swiglu = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMHTHCrossAttention* attention = NULL; //--- Prepare inputs uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(SIGMOID); index++; emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(TANH); //--- Sequence/NonSequence index++; if(!cLastSequence.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cLastSequence.SetActivationFunction(None); index++; if(!cLastNonSequence.Init(0, index, OpenCL, units_ns * embed_size, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cScenarios.Init(0, index, OpenCL, embed_size, scenarios, bottleneck, candidates, topK, optimization_type, batch)) ReturnFalse; cScenarios.SetActivationFunction(TANH); //--- Flow index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (scenarios + units_ns), optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, scenarios + units_ns, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); for(uint l = 0; l < layers; l++) { index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut()*topK, swiglu.GetWindowOut()*topK, embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; attention = new CNeuronMHTHCrossAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, units_out, heads, embed_size, stack_size, bottleneck, candidates, topK, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, embed_size, embed_size, bottleneck, units_out, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); if(l < layers - 1) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (l + 2) * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); } } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * layers * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSTCA feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronSTCA::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFields(); uint units_out = GetUnits(); //--- Sequence/NonSequence if(!DeConcat(cLastSequence.getOutput(), cLastNonSequence.getOutput(), prev.getOutput(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; if(!cStackSequence.FeedForward(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.FeedForward()) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(cLastNonSequence.getOutput(), cScenarios.getOutput(), prev.getOutput(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!stack_querys || !prev || !Concat(prev.getOutput(), stack_querys.getOutput(), curr.getOutput(), embedding_size, curr.Neurons() / units_out - embedding_size, units_out)) ReturnFalse; stack_querys = curr; } else if(!curr.FeedForward(prev, cStackSequence.getOutput())) ReturnFalse; if(!stack_querys && curr.Type() == defNeuronMHTHCrossAttention) stack_querys = prev; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronSTCA | //| layer. | //+------------------------------------------------------------------+ bool CNeuronSTCA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) return(false); //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFields(); uint units_out = GetUnits(); CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Flow for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr) ReturnFalse; if(next.Type() == defNeuronBaseOCL) { if(!DeConcat(curr.getGradient(), next.getPrevOutput(), next.getGradient(), embedding_size, next.Neurons() / (units_ns * scenarios) - embedding_size, units_ns * scenarios)) ReturnFalse; Deactivation(curr); } else if(!curr.CalcHiddenGradients(next, cStackSequence.getOutput(), (!stack_querys ? cStackSequence.getGradient() : cStackSequence.getPrevOutput()), (ENUM_ACTIVATION)cStackSequence.Activation())) ReturnFalse; if(curr.Type() == defNeuronMHTHCrossAttention && !!stack_querys) { if(!SumAndNormalize(cStackSequence.getGradient(), cStackSequence.getPrevOutput(), cStackSequence.getGradient(), embedding_size, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0 && curr.Type() == defNeuronBaseOCL) { if(!!stack_querys) { if(!!stack_querys && !SumAndNormalize(curr.getGradient(), stack_querys.getPrevOutput(), curr.getGradient(), embedding_size, false, 0, 0, 0, 0.5f)) ReturnFalse; } stack_querys = curr; } next = curr; } //--- Sequence/NonSequence if(!DeConcat(cLastNonSequence.getGradient(), cScenarios.getGradient(), cFlow[0].getGradient(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; Deactivation(cLastNonSequence); Deactivation(cScenarios); if(!cLastSequence.CalcHiddenGradients(cStackSequence.AsObject())) ReturnFalse; next = cPrepare[-1]; if(!next || !Concat(cLastSequence.getGradient(), cLastNonSequence.getGradient(), next.getGradient(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; //--- Inputs for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSTCA trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronSTCA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Sequence/NonSequence if(!cStackSequence.UpdateInputWeights(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.UpdateInputWeights()) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSTCA state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSTCA::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastSequence.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSTCA state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSTCA::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSTCA weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronSTCA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSTCA* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastSequence, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronSTCA execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronSTCA::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cLastSequence.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cScenarios.Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSTCA GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronSTCA::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cLastSequence.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronSTCA layer. | //+------------------------------------------------------------------+ void CNeuronSTCA::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cLastSequence.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronWeightGenerator layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint scenarios, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, (window + 3) / 4, (window + 3) / 4, window_out, 1, scenarios, optimization_type, batch)) ReturnFalse; if(!cInputProjection.Init(0, 0, OpenCL, window, window, iWindow, GetUnits(), iVariables, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronWeightGenerator feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cInputProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronConvOCL::feedForward(cInputProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronWeightGenerator layer. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronConvOCL::calcInputGradients(cInputProjection.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cInputProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronWeightGenerator trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cInputProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronConvOCL::updateInputWeights(cInputProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronWeightGenerator state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cInputProjection.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronWeightGenerator state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cInputProjection.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronWeightGenerator weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronWeightGenerator* Source = source; dWeightsUpdate(cInputProjection, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronWeightGenerator GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronWeightGenerator::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cInputProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronWeightGenerator layer. | //+------------------------------------------------------------------+ void CNeuronWeightGenerator::TrainMode(bool flag) { CNeuronConvOCL::TrainMode(flag); cInputProjection.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPCGR layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronPCGR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint querys, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeRCDOCL::Init(numOutputs, myIndex, open_cl, scenarios, querys, window_out, optimization_type, batch)) ReturnFalse; activation = None; //--- iWindowIn = window; iWindowOut = window_out; iInsideDimension = (iWindowOut + 3) / 4; iQuerys = querys; iScenarios = scenarios; //--- uint index = 0; if(!cShared.Init(0, index, OpenCL, iWindowIn, iInsideDimension, iQuerys, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cShared.SetActivationFunction(TANH); index++; if(!cScenarios.Init(0, index, OpenCL, iInsideDimension, iScenarios, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(TANH); index++; if(!cQueryVsScenarios.Init(0, index, OpenCL, 2 * iInsideDimension * iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cQueryVsScenarios.SetActivationFunction(None); index++; if(!cWQscenarios.Init(0, index, OpenCL, 2 * iInsideDimension, iWindowIn * iInsideDimension, iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cWQscenarios.SetActivationFunction(None); index++; if(!cGeneratedQuery.Init(0, index, OpenCL, iInsideDimension * iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cGeneratedQuery.SetActivationFunction(TANH); index++; if(!cQScInToScQIn.Init(0, index, OpenCL, iQuerys, iScenarios, iInsideDimension, optimization, iBatch)) ReturnFalse; cQScInToScQIn.SetActivationFunction((ENUM_ACTIVATION)cGeneratedQuery.Activation()); index++; if(!cWKshared.Init(0, index, OpenCL, iWindowOut, iInsideDimension, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cWKshared.SetActivationFunction(None); index++; if(!cWKscenarios.Init(0, index, OpenCL, iInsideDimension, iInsideDimension * iWindowOut, iScenarios, optimization, iBatch)) ReturnFalse; cWKscenarios.SetActivationFunction(SIGMOID); index++; if(!cWKscT.Init(0, index, OpenCL, iScenarios, cWKshared.Neurons(), optimization, iBatch)) ReturnFalse; cWKscT.SetActivationFunction((ENUM_ACTIVATION)cWKscenarios.Activation()); index++; if(!cWKT.Init(0, index, OpenCL, cWKscT.Neurons(), optimization, iBatch)) ReturnFalse; cWKT.SetActivationFunction(None); index++; if(!cWK.Init(0, index, OpenCL, cWKshared.Neurons(), iScenarios, optimization, iBatch)) ReturnFalse; cWK.SetActivationFunction((ENUM_ACTIVATION)cWKT.Activation()); index++; if(!cQueryMod.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cQueryMod.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPCGR feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronPCGR::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Shared Query if(!cShared.FeedForward(NeuronOCL)) ReturnFalse; //--- Generator if(bTrain) { if(!cScenarios.FeedForward()) ReturnFalse; if(!cWKshared.FeedForward()) ReturnFalse; } //--- Private Query if(!ConcatVecMatrix(cShared.getOutput(), cScenarios.getOutput(), cQueryVsScenarios.getOutput(), iInsideDimension, iScenarios, iInsideDimension, iQuerys, false)) ReturnFalse; if(!cWQscenarios.FeedForward(cQueryVsScenarios.AsObject())) ReturnFalse; if(!MatMul(cWQscenarios.getOutput(), NeuronOCL.getOutput(), cGeneratedQuery.getOutput(), iInsideDimension * iScenarios, iWindowIn, 1, iQuerys, true)) ReturnFalse; dActivation(cGeneratedQuery); //--- Shared + Private if(!SumVecMatrix(cShared.getOutput(), cGeneratedQuery.getOutput(), cGeneratedQuery.getOutput(), iInsideDimension, iQuerys, 0, 0, 0, 1)) ReturnFalse; if(!Normalize(cGeneratedQuery.getOutput(), iInsideDimension)) ReturnFalse; //--- W Key if(!cWKscenarios.FeedForward(cScenarios.AsObject())) ReturnFalse; if(!cWKscT.FeedForward(cWKscenarios.AsObject())) ReturnFalse; if(!ScalarToVector(cWKshared.getOutput(), cWKscT.getOutput(), cWKT.getOutput(), iScenarios)) ReturnFalse; if(!cWK.FeedForward(cWKT.AsObject())) ReturnFalse; //--- Modify Query if(!cQScInToScQIn.FeedForward(cGeneratedQuery.AsObject())) ReturnFalse; if(!MatMul(cQScInToScQIn.getOutput(), cWK.getOutput(), cQueryMod.getOutput(), iQuerys, iInsideDimension, iWindowOut, iScenarios, true)) ReturnFalse; if(!Normalize(cQueryMod.getOutput(), iWindowOut)) ReturnFalse; if(!CNeuronTransposeRCDOCL::feedForward(cQueryMod.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronPCGR | //| layer. | //+------------------------------------------------------------------+ bool CNeuronPCGR::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Modify Query if(!CNeuronTransposeRCDOCL::calcInputGradients(cQueryMod.AsObject())) ReturnFalse; if(!MatMulGrad(cQScInToScQIn.getOutput(), cQScInToScQIn.getGradient(), cWK.getOutput(), cWK.getGradient(), cQueryMod.getGradient(), iQuerys, iInsideDimension, iWindowOut, iScenarios, true)) ReturnFalse; if(!cGeneratedQuery.CalcHiddenGradients(cQScInToScQIn.AsObject())) ReturnFalse; //--- W Key if(!cWKT.CalcHiddenGradients(cWK.AsObject())) ReturnFalse; if(!ScalarToVectorGrad(cWKshared.getOutput(), cWKshared.getGradient(), cWKscT.getOutput(), cWKscT.getGradient(), cWKT.getGradient(), iScenarios)) ReturnFalse; if(!cWKscenarios.CalcHiddenGradients(cWKscT.AsObject())) ReturnFalse; Deactivation(cWKshared); Deactivation(cWKscenarios); if(!cScenarios.CalcHiddenGradients(cWKscenarios.AsObject())) ReturnFalse; //--- Shared + Private if(!SumVecMatrixGrad(cShared.getGradient(), cGeneratedQuery.getPrevOutput(), cGeneratedQuery.getGradient(), iInsideDimension, iQuerys, 0, 0, 0, 1)) ReturnFalse; //--- Private Query Deactivation(cGeneratedQuery); if(!MatMulGrad(cWQscenarios.getOutput(), cWQscenarios.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), cGeneratedQuery.getGradient(), iInsideDimension * iScenarios, iWindowIn, 1, iQuerys, true)) ReturnFalse; if(!cQueryVsScenarios.CalcHiddenGradients(cWQscenarios.AsObject())) ReturnFalse; if(!ConcatVecMatrixGrad(cShared.getPrevOutput(), cScenarios.getPrevOutput(), cQueryVsScenarios.getGradient(), iInsideDimension, iScenarios, iInsideDimension, iQuerys, false)) ReturnFalse; if(!SumAndNormalize(cShared.getGradient(), cShared.getPrevOutput(), cShared.getGradient(), iInsideDimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cShared); if(cScenarios.Activation() != None) if(!DeActivation(cScenarios.getOutput(), cScenarios.getPrevOutput(), cScenarios.getPrevOutput(), cScenarios.Activation())) ReturnFalse; if(!SumAndNormalize(cScenarios.getGradient(), cScenarios.getPrevOutput(), cScenarios.getGradient(), iInsideDimension, false, 0, 0, 0, 1)) ReturnFalse; //--- Shared Query if(!NeuronOCL.CalcHiddenGradients(cShared.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), iWindowIn, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPCGR trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronPCGR::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Shared Query if(!cShared.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cScenarios.UpdateInputWeights()) ReturnFalse; if(!cWKshared.UpdateInputWeights()) ReturnFalse; //--- Private Query if(!cWQscenarios.UpdateInputWeights(cQueryVsScenarios.AsObject())) ReturnFalse; //--- W Key if(!cWKscenarios.UpdateInputWeights(cScenarios.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPCGR state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPCGR::Save(const int file_handle) { if(!CNeuronTransposeRCDOCL::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iWindowIn); dFileWriteUInt(file_handle, iWindowOut); dFileWriteUInt(file_handle, iInsideDimension); dFileWriteUInt(file_handle, iQuerys); dFileWriteUInt(file_handle, iScenarios); //--- if(!cShared.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cQueryVsScenarios.Save(file_handle)) ReturnFalse; if(!cWQscenarios.Save(file_handle)) ReturnFalse; if(!cGeneratedQuery.Save(file_handle)) ReturnFalse; if(!cQScInToScQIn.Save(file_handle)) ReturnFalse; if(!cWKshared.Save(file_handle)) ReturnFalse; if(!cWKscenarios.Save(file_handle)) ReturnFalse; if(!cWKscT.Save(file_handle)) ReturnFalse; if(!cWKT.Save(file_handle)) ReturnFalse; if(!cWK.Save(file_handle)) ReturnFalse; if(!cQueryMod.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPCGR state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPCGR::Load(const int file_handle) { if(!CNeuronTransposeRCDOCL::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iWindowIn); dFileReadUInt(file_handle, iWindowOut); dFileReadUInt(file_handle, iInsideDimension); dFileReadUInt(file_handle, iQuerys); dFileReadUInt(file_handle, iScenarios); //--- if(!LoadInsideLayer(file_handle, cShared.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryVsScenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWQscenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGeneratedQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQScInToScQIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKshared.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKscenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKscT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryMod.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronPCGR weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronPCGR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTransposeRCDOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronPCGR* Source = source; //--- Shared Query dWeightsUpdate(cShared, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cWKshared, Source, tau); dWeightsUpdate(cWQscenarios, Source, tau); dWeightsUpdate(cWKscenarios, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPCGR GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronPCGR::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeRCDOCL::SetOpenCL(obj); //--- cShared.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cQueryVsScenarios.SetOpenCL(OpenCL); cWQscenarios.SetOpenCL(OpenCL); cGeneratedQuery.SetOpenCL(OpenCL); cQScInToScQIn.SetOpenCL(OpenCL); cWKshared.SetOpenCL(OpenCL); cWKscenarios.SetOpenCL(OpenCL); cWKscT.SetOpenCL(OpenCL); cWKT.SetOpenCL(OpenCL); cWK.SetOpenCL(OpenCL); cQueryMod.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronPCGR layer. | //+------------------------------------------------------------------+ void CNeuronPCGR::TrainMode(bool flag) { CNeuronTransposeRCDOCL::TrainMode(flag); //--- cShared.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cQueryVsScenarios.TrainMode(bTrain); cWQscenarios.TrainMode(bTrain); cGeneratedQuery.TrainMode(bTrain); cQScInToScQIn.TrainMode(bTrain); cWKshared.TrainMode(bTrain); cWKscenarios.TrainMode(bTrain); cWKscT.TrainMode(bTrain); cWKT.TrainMode(bTrain); cWK.TrainMode(bTrain); cQueryMod.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronADS layer and allocates its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronADS::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; uint dimension_out = dimensions[count - 1]; uint units_ns = count - units_s - 1; uint bottleneck = uint(0.5 * embed_size * log(topK + 1)); if(bottleneck < 8 * heads || (bottleneck % heads) != 0) bottleneck = MathMax((bottleneck + heads - 1) / heads, 4) * heads; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck * topK, bottleneck * topK, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Prepare arrays cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvOCL* mwc = NULL; CNeuronFieldPatternEmbedding* emb = NULL; CNeuronAutoToken* select = NULL; CNeuronSwiGLUOCL* swiglu = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMHTHCrossAttention* attention = NULL; //--- Prepare inputs uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(SIGMOID); index++; emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(TANH); //--- Sequence/NonSequence index++; if(!cLastSequence.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cLastSequence.SetActivationFunction(None); index++; if(!cLastNonSequence.Init(0, index, OpenCL, units_ns * embed_size, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cQuerys.Init(0, index, OpenCL, embed_size, bottleneck, units_ns, scenarios, bottleneck, candidates, topK, optimization_type, batch)) ReturnFalse; //--- Flow for(uint l = 0; l < layers; l++) { index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; attention = new CNeuronMHTHCrossAttention(); if(!attention || !attention.Init(0, index, OpenCL, bottleneck, units_ns * scenarios, heads, embed_size, stack_size, bottleneck, candidates, topK, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), bottleneck, units_ns, scenarios, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); if(l < layers - 1) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, bottleneck * (l + 2) * units_ns * scenarios, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, bottleneck, bottleneck, neuron.Neurons() / bottleneck, units_ns * scenarios, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck * topK, bottleneck * topK, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); } } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, bottleneck * layers * units_ns * scenarios, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, bottleneck, bottleneck, neuron.Neurons() / bottleneck, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronADS feed-forward pass and writes the layer | //| output. | //+------------------------------------------------------------------+ bool CNeuronADS::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cQuerys.GetCount(); uint bottleneck = cQuerys.GetDimension(); uint units_out = GetUnits(); //--- Sequence/NonSequence if(!DeConcat(cLastSequence.getOutput(), cLastNonSequence.getOutput(), prev.getOutput(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; if(!cStackSequence.FeedForward(cLastSequence.AsObject())) ReturnFalse; //--- PCRG if(!cQuerys.FeedForward(cLastNonSequence.AsObject())) ReturnFalse; //--- Flow prev = cQuerys.AsObject(); stack_querys = prev; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!stack_querys || !prev || !Concat(prev.getOutput(), stack_querys.getOutput(), curr.getOutput(), bottleneck, curr.Neurons() / (units_ns * scenarios) - bottleneck, units_ns * scenarios)) ReturnFalse; stack_querys = curr; } else if(!curr.FeedForward(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the CNeuronADS | //| layer. | //+------------------------------------------------------------------+ bool CNeuronADS::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) return(false); //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cQuerys.GetCount(); uint bottleneck = cQuerys.GetDimension(); uint units_out = GetUnits(); //--- CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Flow for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr) ReturnFalse; if(next.Type() == defNeuronBaseOCL) { if(!DeConcat(curr.getGradient(), next.getPrevOutput(), next.getGradient(), bottleneck, next.Neurons() / (units_ns * scenarios) - bottleneck, units_ns * scenarios)) ReturnFalse; Deactivation(curr); } else if(!curr.CalcHiddenGradients(next, cStackSequence.getOutput(), (!stack_querys ? cStackSequence.getGradient() : cStackSequence.getPrevOutput()), (ENUM_ACTIVATION)cStackSequence.Activation())) ReturnFalse; if(curr.Type() == defNeuronMHTHCrossAttention && !!stack_querys) { if(!SumAndNormalize(cStackSequence.getGradient(), cStackSequence.getPrevOutput(), cStackSequence.getGradient(), embedding_size, false, 0, 0, 0, 0.5f)) ReturnFalse; } if(curr.Type() == defNeuronBaseOCL) { if(!!stack_querys && !SumAndNormalize(curr.getGradient(), stack_querys.getPrevOutput(), curr.getGradient(), bottleneck, false, 0, 0, 0, 0.5f)) ReturnFalse; stack_querys = curr; } next = curr; } //--- PCRG if(!cQuerys.CalcHiddenGradients(cFlow[0])) ReturnFalse; if(!!stack_querys) { if(!DeConcat(stack_querys.getPrevOutput(), cQuerys.getPrevOutput(), stack_querys.getGradient(), bottleneck, bottleneck, units_ns * scenarios)) ReturnFalse; if(cQuerys.Activation() != None && !DeActivation(cQuerys.getOutput(), cQuerys.getPrevOutput(), cQuerys.getPrevOutput(), cQuerys.Activation())) ReturnFalse; if(!SumAndNormalize(cQuerys.getGradient(), cQuerys.getPrevOutput(), cQuerys.getGradient(), bottleneck, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- Sequence/NonSequence if(!cLastNonSequence.CalcHiddenGradients(cQuerys.AsObject())) ReturnFalse; if(!cLastSequence.CalcHiddenGradients(cStackSequence.AsObject())) ReturnFalse; next = cPrepare[-1]; if(!next || !Concat(cLastSequence.getGradient(), cLastNonSequence.getGradient(), next.getGradient(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; //--- Inputs for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronADS trainable weights from the accumulated | //| gradients. | //+------------------------------------------------------------------+ bool CNeuronADS::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- PCRG if(!cQuerys.UpdateInputWeights(cLastNonSequence.AsObject())) ReturnFalse; //--- Flow prev = cQuerys.AsObject(); for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronADS state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronADS::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastSequence.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cQuerys.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronADS state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronADS::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQuerys.AsObject())) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronADS weights toward the source model with | //| factor tau. | //+------------------------------------------------------------------+ bool CNeuronADS::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronADS* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastSequence, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cQuerys, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronADS execution state and its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronADS::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cLastSequence.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cQuerys.Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronADS GPU buffers to the given OpenCL context. | //+------------------------------------------------------------------+ void CNeuronADS::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cLastSequence.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cQuerys.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronADS layer. | //+------------------------------------------------------------------+ void CNeuronADS::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cLastSequence.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cQuerys.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPerTokenFFN layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFieldAwareConv::Init(numOutputs, myIndex, open_cl, 2 * window, window, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; activation = None; if(!cProj.Init(0, 0, OpenCL, window, CNeuronFieldAwareConv::GetWindow(), GetFields(), embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cProj.SetActivationFunction(SoftPlus); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPerTokenFFN feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProj.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::feedForward(cProj.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), Output, Output, GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPerTokenFFN layer. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldAwareConv::calcInputGradients(cProj.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; CBufferFloat* temp = Gradient; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; temp = PrevOutput; } if(!SumAndNormalize(NeuronOCL.getGradient(), temp, NeuronOCL.getGradient(), GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPerTokenFFN trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::updateInputWeights(cProj.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPerTokenFFN state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Save(const int file_handle) { if(!CNeuronFieldAwareConv::Save(file_handle)) ReturnFalse; if(!cProj.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPerTokenFFN state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Load(const int file_handle) { if(!CNeuronFieldAwareConv::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProj.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronPerTokenFFN weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldAwareConv::WeightsUpdate(source, tau)) ReturnFalse; CNeuronPerTokenFFN* Source = source; dWeightsUpdate(cProj, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPerTokenFFN GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronPerTokenFFN::SetOpenCL(COpenCLMy *obj) { CNeuronFieldAwareConv::SetOpenCL(obj); cProj.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronPerTokenFFN layer. | //+------------------------------------------------------------------+ void CNeuronPerTokenFFN::TrainMode(bool flag) { CNeuronFieldAwareConv::TrainMode(flag); cProj.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronScenariosToken layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint fields, uint window_out, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFieldPatternEmbedding::Init(numOutputs, myIndex, open_cl, (window_in + 1) / 2, scenarios, window_out, candidates, topK, optimization_type, batch)) ReturnFalse; if(!cScenariosGenerator.Init(0, 0, OpenCL, window_in, (window_in + 1) / 2, fields, scenarios, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronScenariosToken feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cScenariosGenerator.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronFieldPatternEmbedding::feedForward(cScenariosGenerator.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronScenariosToken layer. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldPatternEmbedding::calcInputGradients(cScenariosGenerator.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cScenariosGenerator.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronScenariosToken trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cScenariosGenerator.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldPatternEmbedding::updateInputWeights(cScenariosGenerator.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronScenariosToken state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Save(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Save(file_handle)) ReturnFalse; if(!cScenariosGenerator.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronScenariosToken state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Load(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenariosGenerator.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronScenariosToken weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldPatternEmbedding::WeightsUpdate(source, tau)) ReturnFalse; CNeuronScenariosToken* Source = source; dWeightsUpdate(cScenariosGenerator, Source, tau) //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronScenariosToken execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Clear(void) { if(!CNeuronFieldPatternEmbedding::Clear()) ReturnFalse; if(!cScenariosGenerator.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronScenariosToken GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronScenariosToken::SetOpenCL(COpenCLMy *obj) { CNeuronFieldPatternEmbedding::SetOpenCL(obj); cScenariosGenerator.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronScenariosToken layer. | //+------------------------------------------------------------------+ void CNeuronScenariosToken::TrainMode(bool flag) { CNeuronFieldPatternEmbedding::TrainMode(flag); cScenariosGenerator.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronUnifiedTokenizer layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(count < units) ReturnFalse; //--- if(!CNeuronFieldPatternEmbedding::Init(numOutputs, myIndex, open_cl, embed_size, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- uint index = 0; uint total_windows = 0; for(uint i = 0; i < count; i++) total_windows += dimensions[i]; if(!cNorm.Init(0, index, OpenCL, total_windows, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; if(!cProj.Init(0, index, OpenCL, dimensions, embed_size, 1, 1, optimization, iBatch)) ReturnFalse; cProj.SetActivationFunction(SIGMOID); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronUnifiedTokenizer feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cProj.FeedForward(cNorm.AsObject())) ReturnFalse; if(!CNeuronFieldPatternEmbedding::feedForward(cProj.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronUnifiedTokenizer layer. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldPatternEmbedding::calcInputGradients(cProj.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronUnifiedTokenizer trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cProj.UpdateInputWeights(cNorm.AsObject())) ReturnFalse; if(!CNeuronFieldPatternEmbedding::updateInputWeights(cProj.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronUnifiedTokenizer state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Save(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(!cProj.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronUnifiedTokenizer state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Load(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProj.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronUnifiedTokenizer weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldPatternEmbedding::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUnifiedTokenizer* Source = source; dWeightsUpdate(cNorm, Source, tau); dWeightsUpdate(cProj, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronUnifiedTokenizer GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronUnifiedTokenizer::SetOpenCL(COpenCLMy *obj) { CNeuronFieldPatternEmbedding::SetOpenCL(obj); cNorm.SetOpenCL(OpenCL); cProj.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronUnifiedTokenizer layer. | //+------------------------------------------------------------------+ void CNeuronUnifiedTokenizer::TrainMode(bool flag) { CNeuronFieldPatternEmbedding::TrainMode(flag); cNorm.TrainMode(bTrain); cProj.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDomainAwareAttention layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { cLogSumExp.BufferFree(); //--- if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, dimension_q, units_q, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iQUnits = units_q; iXUnits = unit_x; iHeads = heads; iXDimension = dimension_x; //--- cPrepareQ.Clear(); cW0.Clear(); cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); //--- Query uint index = 0; uint head_size = (dimension_q + heads - 1) / heads; { CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension_q, heads * head_size, units_q, embed_size, candidates, topK, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, head_size, dimension_x, units_q * heads, embed_size, candidates, topK, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); } //--- W0 index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, dimension_x * units_q * heads, optimization, iBatch) || !cW0.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_x, dimension_x, head_size, units_q * heads, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, heads * head_size, heads * head_size, dimension_q, units_q, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); //--- if(!cLogSumExp.BufferInit(units_q * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDomainAwareAttention attention out operation. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !cW0[0] || !cW0[0].getOutput() || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iXUnits, iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCA; setBuffer(kernel, def_k_mhstca_query, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhstca_X, Context.GetIndex()) setBuffer(kernel, def_k_mhstca_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_output, cW0[0].getOutputIndex()) setArgument(kernel, def_k_mhstca_dimension, iXDimension) setArgument(kernel, def_k_mhstca_total_X, iXUnits) setArgument(kernel, def_k_mhstca_mask_future, int(false)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) dActivation(cW0[0]); #ifdef _DEBUG if(!cW0[0].getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDomainAwareAttention attention inside | //| gradients operation. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !cW0[0] || !cW0[0].getOutput() || !prevLayer.getOutput() || !SecondInput || !cW0[0].getGradient() || !prevLayer.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iXUnits), (uint)MathMin( MathMax( MathMax(iQUnits * iHeads, iXUnits), iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCAGrad; setBuffer(kernel, def_k_mhstca_gr_query, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_query_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhstca_gr_X, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_X_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_output, cW0[0].getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_output_gr, cW0[0].getGradientIndex()) setArgument(kernel, def_k_mhstca_gr_dimension, iXDimension) setArgument(kernel, def_k_mhstca_gr_total_q, iQUnits) setArgument(kernel, def_k_mhstca_gr_total_X, iXUnits) setArgument(kernel, def_k_mhstca_gr_mask_future, int(false)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDomainAwareAttention feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!AttentionOut(prev, Context)) ReturnFalse; prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), prev.getOutput(), prev.Neurons() / iQUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronPerTokenFFN::feedForward(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronDomainAwareAttention layer. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronPerTokenFFN::calcInputGradients(cW0[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cW0.Total() - 2; i >= 0; i--) { curr = cW0[i]; if(!curr || !curr.CalcHiddenGradients(cW0[i + 1])) ReturnFalse; } //--- if(!AttentionInsideGradients(cPrepareQ[-1], SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- for(int i = cPrepareQ.Total() - 2; i >= 0; i--) { curr = cPrepareQ[i]; if(!curr || !curr.CalcHiddenGradients(cPrepareQ[i + 1])) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(curr)) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), cW0[-1].getPrevOutput(), cW0[-1].getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cW0[-1].getPrevOutput(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(prevLayer.getGradient(), cW0[-1].getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDomainAwareAttention trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronPerTokenFFN::updateInputWeights(prev)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronDomainAwareAttention state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iXUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iXDimension); //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDomainAwareAttention state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); //--- if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Load(file_handle)) ReturnFalse; if(!cW0.Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iXUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iXDimension); //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronDomainAwareAttention weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDomainAwareAttention* Source = source; dWeightsUpdate(cPrepareQ, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDomainAwareAttention GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronDomainAwareAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronPerTokenFFN::SetOpenCL(obj); //--- cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronDomainAwareAttention layer. | //+------------------------------------------------------------------+ void CNeuronDomainAwareAttention::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); //--- cPrepareQ.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronFeatureSelfIteration layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint stack_size, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronDomainAwareAttention::Init(numOutputs, myIndex, open_cl, dimension, units, heads, dimension, stack_size * units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; if(!cStack.Init(0, 0, OpenCL, stack_size, dimension, units, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronFeatureSelfIteration feed-forward pass and | //| writes the layer output. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cStack.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronDomainAwareAttention::feedForward(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronFeatureSelfIteration layer. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!CNeuronDomainAwareAttention::calcInputGradients(prevLayer, cStack.getOutput(), cStack.getGradient(), (ENUM_ACTIVATION)cStack.Activation())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronFeatureSelfIteration trainable weights from | //| the accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!CNeuronDomainAwareAttention::updateInputWeights(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronFeatureSelfIteration state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Save(const int file_handle) { if(!CNeuronDomainAwareAttention::Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronFeatureSelfIteration state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Load(const int file_handle) { if(!CNeuronDomainAwareAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronFeatureSelfIteration weights toward the | //| source model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronDomainAwareAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFeatureSelfIteration* Source = source; dWeightsUpdate(cStack, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronFeatureSelfIteration execution state and its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Clear(void) { if(!CNeuronDomainAwareAttention::Clear()) ReturnFalse; if(!cStack.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronFeatureSelfIteration GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronFeatureSelfIteration::SetOpenCL(COpenCLMy *obj) { CNeuronDomainAwareAttention::SetOpenCL(obj); cStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronFeatureSelfIteration layer. | //+------------------------------------------------------------------+ void CNeuronFeatureSelfIteration::TrainMode(bool flag) { CNeuronDomainAwareAttention::TrainMode(flag); cStack.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDomainFused layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint tasks, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, dimension, tasks + scenarios, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cTasks.Init(0, index, OpenCL, dimension * tasks, optimization, iBatch)) ReturnFalse; cTasks.SetActivationFunction(None); index++; if(!cScenarios.Init(0, index, OpenCL, dimension * scenarios, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(None); //--- cSelector.Clear(); cSelector.SetOpenCL(OpenCL); index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension, 2 * dimension, tasks, embed_size, candidates, topK, optimization, iBatch) || !cSelector.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * dimension, scenarios, tasks, embed_size, candidates, topK, optimization, iBatch) || !cSelector.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(None); index++; CNeuronSparseSoftMax* sofmax = new CNeuronSparseSoftMax(); if(!sofmax || !sofmax.Init(0, index, OpenCL, tasks, scenarios, (scenarios + 2) / 3, optimization, iBatch) || !cSelector.Add(sofmax)) DeleteObjAndFalse(sofmax); index++; if(!cScenariosToTask.Init(0, index, OpenCL, dimension * tasks, optimization, iBatch)) ReturnFalse; cScenariosToTask.SetActivationFunction(None); index++; if(!cConcat.Init(0, index, OpenCL, dimension * (tasks + scenarios), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDomainFused feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint dimension = GetWindow(); uint tasks = cTasks.Neurons() / dimension; uint scenarios = cScenarios.Neurons() / dimension; //--- if(!DeConcat(cTasks.getOutput(), cScenarios.getOutput(), NeuronOCL.getOutput(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- Selector CNeuronBaseOCL* prev = cTasks.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSelector.Total(); i++) { curr = cSelector[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cScenarios.getOutput(), cScenariosToTask.getOutput(), tasks, softmax.DimensionOut(), scenarios, dimension)) ReturnFalse; //--- if(!SumAndNormalize(cTasks.getOutput(), cScenariosToTask.getOutput(), cScenariosToTask.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(cScenariosToTask.getOutput(), cScenarios.getOutput(), cConcat.getOutput(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- if(!CNeuronPerTokenFFN::feedForward(cConcat.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronDomainFused layer. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint dimension = GetWindow(); uint tasks = cTasks.Neurons() / dimension; uint scenarios = cScenarios.Neurons() / dimension; //--- if(!CNeuronPerTokenFFN::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(cScenariosToTask.getGradient(), cScenarios.getPrevOutput(), cConcat.getGradient(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- Selector if(cSelector[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cSelector[-1]; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cScenarios.getOutput(), cScenarios.getGradient(), cScenariosToTask.getGradient(), tasks, softmax.DimensionOut(), scenarios, dimension)) ReturnFalse; CNeuronBaseOCL* curr = NULL; for(int i = cSelector.Total() - 2; i >= 0; i--) { curr = cSelector[i]; if(!curr || !curr.CalcHiddenGradients(cSelector[i + 1])) ReturnFalse; } if(!cTasks.CalcHiddenGradients(cSelector[0])) ReturnFalse; //--- if(!SumAndNormalize(cTasks.getGradient(), cScenariosToTask.getGradient(), cTasks.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cScenarios.getGradient(), cScenarios.getPrevOutput(), cScenarios.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!Concat(cTasks.getGradient(), cScenarios.getGradient(), NeuronOCL.getGradient(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDomainFused trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Selector CNeuronBaseOCL* prev = cTasks.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSelector.Total(); i++) { curr = cSelector[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronPerTokenFFN::updateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronDomainFused state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; //--- if(!cTasks.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cSelector.Save(file_handle)) ReturnFalse; if(!cScenariosToTask.Save(file_handle)) ReturnFalse; if(!cConcat.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDomainFused state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Load(const int file_handle) { if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTasks.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cSelector.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenariosToTask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcat.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronDomainFused weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronDomainFused::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDomainFused* Source = source; dWeightsUpdate(cSelector, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDomainFused GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronDomainFused::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenFFN::SetOpenCL(obj); //--- cTasks.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cSelector.SetOpenCL(OpenCL); cScenariosToTask.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronDomainFused layer. | //+------------------------------------------------------------------+ void CNeuronDomainFused::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); //--- cTasks.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cSelector.TrainMode(bTrain); cScenariosToTask.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronPerTokenSwiGLU layer and allocates its | //| GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, window, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; cProj.SetActivationFunction(None); uint index = 1; if(!cGate.Init(0, index, OpenCL, window, CNeuronFieldAwareConv::GetWindow(), GetFields(), embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cGate.SetActivationFunction(SoftPlus); index++; if(!cAfterGate.Init(0, index, OpenCL, cGate.Neurons(), optimization, iBatch)) ReturnFalse; cAfterGate.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronPerTokenSwiGLU feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProj.FeedForward(NeuronOCL)) ReturnFalse; if(!cGate.FeedForward(NeuronOCL)) ReturnFalse; if(!ElementMult(cProj.getOutput(), cGate.getOutput(), cAfterGate.getOutput())) ReturnFalse; if(!CNeuronFieldAwareConv::feedForward(cAfterGate.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronPerTokenSwiGLU layer. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronFieldAwareConv::calcInputGradients(cAfterGate.AsObject())) ReturnFalse; if(!ElementMultGrad(cProj.getOutput(), cProj.getGradient(), cGate.getOutput(), cGate.getGradient(), cAfterGate.getGradient(), cProj.Activation(), cGate.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false) || !NeuronOCL.CalcHiddenGradients(cGate.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronPerTokenSwiGLU trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGate.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::updateInputWeights(cAfterGate.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronPerTokenSwiGLU state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; if(!cGate.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronPerTokenSwiGLU state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Load(const int file_handle) { if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGate.AsObject())) ReturnFalse; if(!cAfterGate.Init(0, 2, OpenCL, cGate.Neurons(), optimization, iBatch)) ReturnFalse; cAfterGate.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronPerTokenSwiGLU weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; CNeuronPerTokenSwiGLU* Source = source; dWeightsUpdate(cGate, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronPerTokenSwiGLU GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronPerTokenSwiGLU::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenFFN::SetOpenCL(obj); cGate.SetOpenCL(OpenCL); cAfterGate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronPerTokenSwiGLU layer. | //+------------------------------------------------------------------+ void CNeuronPerTokenSwiGLU::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); cGate.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronSiameseNorm layer and allocates its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint blocks, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cNormYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNormYIn.SetActivationFunction(None); index++; if(!cSumXYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cSumXYIn.SetActivationFunction(None); index++; if(!cMixer.Init(0, index, OpenCL, window, units, blocks, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; index++; if(!cYOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cYOut.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSiameseNorm feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Type() != Type()) { if(!NeuronOCL.getOutput() || !feedForward(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; } else { CNeuronSiameseNorm* second = NeuronOCL; if(!second.GetYOut() || !second.GetYOut().getOutput() || !feedForward(NeuronOCL, second.GetYOut().getOutput())) ReturnFalse; } return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronSiameseNorm feed-forward pass and writes the | //| layer output. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; if(!Context) return(feedForward(NeuronOCL)); if(Context.Total() < Neurons()) ReturnFalse; //--- uint dimension = cMixer.GetWindow(); uint units = cMixer.GetFields(); //--- Norm(Y) if(!Concat(Context, cNormYIn.getOutput(), cNormYIn.getOutput(), dimension, 0, units)) ReturnFalse; if(!Normalize(cNormYIn.getOutput(), dimension)) ReturnFalse; //--- X + Norm(Y) if(!SumAndNormalize(NeuronOCL.getOutput(), cNormYIn.getOutput(), cSumXYIn.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- Mixing if(!cMixer.FeedForward(cSumXYIn.AsObject())) ReturnFalse; //--- X if(!SumAndNormalize(NeuronOCL.getOutput(), cMixer.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Y if(!SumAndNormalize(Context, cMixer.getOutput(), cYOut.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSiameseNorm layer. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; if(prevLayer.Type() == Type()) { CNeuronSiameseNorm* obj = prevLayer; CNeuronBaseOCL* second = obj.GetYOut(); if(!second.getOutput() || !second.getGradient() || !calcInputGradients(prevLayer, second.getOutput(), second.getGradient(), (ENUM_ACTIVATION)second.Activation())) ReturnFalse; } else { if(!prevLayer.getOutput() || !calcInputGradients(prevLayer, prevLayer.getOutput(), PrevOutput, (ENUM_ACTIVATION)prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getOutput(), PrevOutput, prevLayer.getOutput(), cMixer.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronSiameseNorm layer. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!SecondGradient) return(calcInputGradients(prevLayer)); //--- if(!prevLayer || prevLayer.Neurons() < Neurons() || SecondGradient.Total() < Neurons()) ReturnFalse; //--- uint dimension = cMixer.GetWindow(); uint units = cMixer.GetFields(); //--- Mixing if(!SumAndNormalize(Gradient, cYOut.getGradient(), cMixer.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cMixer); if(!cSumXYIn.CalcHiddenGradients(cMixer.AsObject())) ReturnFalse; if(!SumAndNormalize(cSumXYIn.getGradient(), Gradient, cSumXYIn.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- X if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), cSumXYIn.getGradient(), prevLayer.Activation())) ReturnFalse; //--- Y if(!DeActivation(SecondInput, SecondGradient, cSumXYIn.getGradient(), SecondActivation)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSiameseNorm trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Type() != Type()) { if(updateInputWeights(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; } else { CNeuronSiameseNorm* second = NeuronOCL; if(!updateInputWeights(NeuronOCL, second.GetYOut().getOutput())) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronSiameseNorm trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!cMixer.UpdateInputWeights(cSumXYIn.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronSiameseNorm weights toward the source model | //| with factor tau. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSiameseNorm* Source = source; dWeightsUpdate(cMixer, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronSiameseNorm state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cMixer.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronSiameseNorm state from the file handle. | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMixer.AsObject())) ReturnFalse; //--- uint index = 0; if(!cNormYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNormYIn.SetActivationFunction(None); index++; if(!cSumXYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cSumXYIn.SetActivationFunction(None); index += 2; if(!cYOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cYOut.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronSiameseNorm GPU buffers to the given OpenCL | //| context. | //+------------------------------------------------------------------+ void CNeuronSiameseNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cMixer.SetOpenCL(OpenCL); cNormYIn.SetOpenCL(OpenCL); cSumXYIn.SetOpenCL(OpenCL); cYOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronSiameseNorm layer. | //+------------------------------------------------------------------+ void CNeuronSiameseNorm::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cMixer.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Initializes the CNeuronDictionaryCrossAtt layer and allocates | //| its GPU buffers. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint heads, uint dictionary_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(heads < 1 || dictionary_size < 2) ReturnFalse; if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, 2 * window, 2 * window, window, units, 1, optimization_type, batch)) ReturnFalse; //--- iDictionarySize = dictionary_size; iHeads = heads; iHiddenDimension = (window + heads - 1) / heads; //--- uint index = 0; if(!cDictionaryKeys.Init(0, index, OpenCL, (iDictionarySize * iHiddenDimension * iHeads), optimization, iBatch)) ReturnFalse; cDictionaryKeys.SetActivationFunction(None); index++; if(!cDictionaryValues.Init(0, index, OpenCL, cDictionaryKeys.Neurons(), optimization, iBatch)) ReturnFalse; cDictionaryValues.SetActivationFunction(None); index++; if(!cPrototypes.Init(0, index, OpenCL, cDictionaryKeys.Neurons(), optimization, iBatch)) ReturnFalse; cPrototypes.SetActivationFunction(None); CBufferFloat* buf = cPrototypes.getWeightsParams(); if(!buf) ReturnFalse; buf.Fill(0); index++; if(!cLevels.Init(0, index, OpenCL, units * iHeads, optimization, iBatch)) ReturnFalse; cLevels.SetActivationFunction(SIGMOID); buf = cLevels.getWeightsParams(); if(!buf) ReturnFalse; buf.Fill(-3); bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); if(bufLogSumExp <= 0) ReturnFalse; index++; if(!cQuerys.Init(0, index, OpenCL, window, window, iHeads * iHiddenDimension, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); index++; if(!cSimilarity.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cSimilarity.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, iHiddenDimension * iHeads, iHiddenDimension * iHeads, window, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cProjSimilarity.Init(0, index, OpenCL, iHiddenDimension * iHeads, iHiddenDimension * iHeads, window, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cFeedForward0.Init(0, index, OpenCL, window, window, cConv.GetWindow(), units, 1, optimization, iBatch)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Runs the CNeuronDictionaryCrossAtt feed-forward pass and writes | //| the layer output. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQuerys.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) { if(!cDictionaryKeys.FeedForward() || !cDictionaryValues.FeedForward() || !cPrototypes.FeedForward() || !cLevels.FeedForward()) ReturnFalse; } if(!MHCrossAttvsSim()) ReturnFalse; if(!cW0.FeedForward(cAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cW0.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cProjSimilarity.FeedForward(cSimilarity.AsObject())) ReturnFalse; if(!cFeedForward0.FeedForward(cW0.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cFeedForward0.AsObject())) ReturnFalse; if(!SumAndNormalize(getOutput(), cW0.getOutput(), getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Back-propagates the input gradients through the | //| CNeuronDictionaryCrossAtt layer. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFeedForward0.AsObject())) ReturnFalse; if(!cW0.CalcHiddenGradients(cFeedForward0.AsObject())) ReturnFalse; if(!SumAndNormalize(getGradient(), cW0.getGradient(), cW0.getGradient(), cW0.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!MHCrossAttvsSimGrad()) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQuerys.AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormalize(NeuronOCL.getGradient(), cW0.getGradient(), NeuronOCL.getGradient(), cQuerys.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), cW0.getPrevOutput(), cW0.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cW0.getPrevOutput(), NeuronOCL.getGradient(), cQuerys.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return(true); } //+------------------------------------------------------------------+ //| Updates the CNeuronDictionaryCrossAtt trainable weights from the | //| accumulated gradients. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQuerys.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cDictionaryKeys.UpdateInputWeights()) ReturnFalse; if(!cDictionaryValues.UpdateInputWeights()) ReturnFalse; if(!cPrototypes.UpdateInputWeights()) ReturnFalse; if(!cLevels.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cAttentionOut.AsObject())) ReturnFalse; if(!cProjSimilarity.UpdateInputWeights(cSimilarity.AsObject())) ReturnFalse; if(!cFeedForward0.UpdateInputWeights(cW0.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cFeedForward0.AsObject())) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDictionaryCrossAtt mh cross attvs sim | //| operation. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::MHCrossAttvsSim(void) { if(!OpenCL || bufLogSumExp < 0) ReturnFalse; //--- uint units = cQuerys.GetUnits(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(MathMax(iDictionarySize, iHiddenDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_MHCrossAttvsSim; setBuffer(kernel, def_k_casim_query, cQuerys.getOutputIndex()) setBuffer(kernel, def_k_casim_key, cDictionaryKeys.getOutputIndex()) setBuffer(kernel, def_k_casim_value, cDictionaryValues.getOutputIndex()) setBuffer(kernel, def_k_casim_prototype, cPrototypes.getOutputIndex()) setBuffer(kernel, def_k_casim_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_casim_logsumexp, bufLogSumExp) setBuffer(kernel, def_k_casim_output_at, cAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_casim_similarity, cSimilarity.getOutputIndex()) setArgument(kernel, def_k_casim_dimension, (int)iHiddenDimension) setArgument(kernel, def_k_casim_total_X, (int)iDictionarySize) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; if(!cSimilarity.getOutput().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Handles the CNeuronDictionaryCrossAtt mh cross attvs sim grad | //| operation. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::MHCrossAttvsSimGrad(void) { if(!OpenCL || bufLogSumExp < 0) ReturnFalse; //--- uint units = cQuerys.GetUnits(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(MathMax(MathMax(units, iDictionarySize), iHiddenDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_MHCrossAttvsSimGrad; setBuffer(kernel, def_k_casimgr_query, cQuerys.getOutputIndex()) setBuffer(kernel, def_k_casimgr_key, cDictionaryKeys.getOutputIndex()) setBuffer(kernel, def_k_casimgr_value, cDictionaryValues.getOutputIndex()) setBuffer(kernel, def_k_casimgr_prototype, cPrototypes.getOutputIndex()) setBuffer(kernel, def_k_casimgr_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_casimgr_logsumexp, bufLogSumExp) setBuffer(kernel, def_k_casimgr_output_at, cAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_casimgr_similarity, cSimilarity.getOutputIndex()) setBuffer(kernel, def_k_casimgr_grad_output_at, cAttentionOut.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_similarity, cSimilarity.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_query, cQuerys.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_key, cDictionaryKeys.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_value, cDictionaryValues.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_prototype, cPrototypes.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_levels, cLevels.getGradientIndex()) setArgument(kernel, def_k_casimgr_dimension, (int)iHiddenDimension) setArgument(kernel, def_k_casimgr_total_X, (int)iDictionarySize) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQuerys.getGradient().BufferRead()) ReturnFalse; if(!cDictionaryKeys.getGradient().BufferRead()) ReturnFalse; if(!cDictionaryValues.getGradient().BufferRead()) ReturnFalse; if(!cPrototypes.getGradient().BufferRead()) ReturnFalse; if(!cLevels.getGradient().BufferRead()) ReturnFalse; #endif //--- return(true); } //+------------------------------------------------------------------+ //| Saves the CNeuronDictionaryCrossAtt state to the file handle. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- Learned dictionaries (CParams — carry trainable weights) if(!cDictionaryKeys.Save(file_handle)) ReturnFalse; if(!cDictionaryValues.Save(file_handle)) ReturnFalse; if(!cPrototypes.Save(file_handle)) ReturnFalse; if(!cLevels.Save(file_handle)) ReturnFalse; //--- Computation sub-modules with trainable weights if(!cQuerys.Save(file_handle)) ReturnFalse; //--- cAttentionOut and cSimilarity: no trainable params — re-inited on Load if(!cW0.Save(file_handle)) ReturnFalse; if(!cProjSimilarity.Save(file_handle)) ReturnFalse; if(!cFeedForward0.Save(file_handle)) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Restores the CNeuronDictionaryCrossAtt state from the file | //| handle. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Load(const int file_handle) { //--- Free the transient GPU buffer BEFORE calling the parent, because the parent //--- may change the OpenCL pointer, making the old handle invalid if(bufLogSumExp >= 0 && !!OpenCL) OpenCL.BufferFree(bufLogSumExp); bufLogSumExp = INVALID_HANDLE; //--- if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- Learned dictionaries (CParams) if(!LoadInsideLayer(file_handle, cDictionaryKeys.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDictionaryValues.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrototypes.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLevels.AsObject())) ReturnFalse; //--- Sub-modules with trainable weights if(!LoadInsideLayer(file_handle, cQuerys.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjSimilarity.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward0.AsObject())) ReturnFalse; //--- Restore architectural parameters from loaded sub-objects //--- (these fields are not serialised separately) iHeads = cLevels.Neurons() / cQuerys.GetUnits(); iHiddenDimension = (iHeads > 0) ? cQuerys.GetFilters() / iHeads : 0; iDictionarySize = (iHiddenDimension > 0 && iHeads > 0) ? cDictionaryKeys.Neurons() / (iHiddenDimension * iHeads) : 0; //--- Recreate transient buffer (not persisted) if(!OpenCL) ReturnFalse; bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); if(bufLogSumExp < 0) ReturnFalse; //--- cAttentionOut and cSimilarity: no trainable params — re-initialise from dimensions //--- position in sub-object sequence (see Init) uint index = 5; if(!cAttentionOut.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); index++; if(!cSimilarity.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cSimilarity.SetActivationFunction(None); //--- return(true); } //+------------------------------------------------------------------+ //| Averages the CNeuronDictionaryCrossAtt weights toward the source | //| model with factor tau. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDictionaryCrossAtt *Source = source; //--- Learned dictionaries dWeightsUpdate(cDictionaryKeys, Source, tau); dWeightsUpdate(cDictionaryValues, Source, tau); dWeightsUpdate(cPrototypes, Source, tau); dWeightsUpdate(cLevels, Source, tau); //--- Computation sub-modules with trainable weights //--- cAttentionOut and cSimilarity are excluded — no learnable parameters dWeightsUpdate(cQuerys, Source, tau); dWeightsUpdate(cW0, Source, tau); dWeightsUpdate(cProjSimilarity, Source, tau); dWeightsUpdate(cFeedForward0, Source, tau); //--- return(true); } //+------------------------------------------------------------------+ //| Clears the CNeuronDictionaryCrossAtt execution state and its GPU | //| buffers. | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- CParams objects (cDictionaryKeys, cDictionaryValues, cPrototypes, cLevels) //--- are intentionally excluded: their Output buffer IS the learned parameter tensor if(!cQuerys.Clear()) ReturnFalse; if(!cAttentionOut.Clear()) ReturnFalse; if(!cSimilarity.Clear()) ReturnFalse; if(!cW0.Clear()) ReturnFalse; if(!cProjSimilarity.Clear()) ReturnFalse; if(!cFeedForward0.Clear()) ReturnFalse; //--- return(true); } //+------------------------------------------------------------------+ //| Sets the training flag of the CNeuronDictionaryCrossAtt layer. | //+------------------------------------------------------------------+ void CNeuronDictionaryCrossAtt::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- Learned dictionaries cDictionaryKeys.TrainMode(bTrain); cDictionaryValues.TrainMode(bTrain); cPrototypes.TrainMode(bTrain); cLevels.TrainMode(bTrain); //--- Computation sub-modules cQuerys.TrainMode(bTrain); cAttentionOut.TrainMode(bTrain); cSimilarity.TrainMode(bTrain); cW0.TrainMode(bTrain); cProjSimilarity.TrainMode(bTrain); cFeedForward0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| Rebinds the CNeuronDictionaryCrossAtt GPU buffers to the given | //| OpenCL context. | //+------------------------------------------------------------------+ void CNeuronDictionaryCrossAtt::SetOpenCL(COpenCLMy *obj) { //--- Free the raw buffer on the OLD context before switching if(bufLogSumExp >= 0 && !!OpenCL) OpenCL.BufferFree(bufLogSumExp); bufLogSumExp = INVALID_HANDLE; //--- Switch OpenCL pointer in self and all sub-objects via parent CNeuronSpikeConvBlock::SetOpenCL(obj); //--- Propagate to every sub-object not covered by the parent cDictionaryKeys.SetOpenCL(OpenCL); cDictionaryValues.SetOpenCL(OpenCL); cPrototypes.SetOpenCL(OpenCL); cLevels.SetOpenCL(OpenCL); cQuerys.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cSimilarity.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cProjSimilarity.SetOpenCL(OpenCL); cFeedForward0.SetOpenCL(OpenCL); //--- Recreate transient buffer on the NEW context if(!!OpenCL && cLevels.Neurons() > 0) bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| Constructs dormant UShape members and explicit graph owners | //+------------------------------------------------------------------+ CNeuronPlanDenoiserOCL::CNeuronPlanDenoiserOCL(void) { m_frame_mode = false; m_frame_bound = false; m_frame_live = false; m_frame_failed = false; m_frame_cache_valid = false; m_frame_training = false; m_frame_gradients_ready = false; m_frame_updated = false; m_frame_update_started = false; m_frame_horizon = m_frame_rows = m_frame_width = m_frame_heads = 0; m_frame_max_frames = m_frame_count = m_frame_active_rows = m_frame_next = m_frame_current = 0; m_frame_k = m_frame_v = m_frame_market_horizon = m_frame_dimension = 0; m_frame_condition_count = m_frame_capacity = 0; m_frame_backward_next = m_frame_backward_count = m_frame_update_count = 0; m_frame_work_size = 0; m_frame_decision = m_frame_market_id = m_frame_market_weights = 0; m_frame_cache_decision = m_frame_cache_market = m_frame_cache_weights = 0; m_frame_cache_context = NULL; m_frame_probability_tolerance = 0; ArrayInitialize(m_frame_input, 0); ArrayInitialize(m_frame_output, 0); ArrayInitialize(m_frame_affine_rows, 0); ArrayInitialize(m_frame_tape_size, 0); ArrayInitialize(m_frame_common_size, 0); ArrayInitialize(m_frame_profile, 0); for(uint i = 0; i < 21; i++) m_frame_affine[i] = NULL; for(uint i = 0; i < 24; i++) m_frame_common[i] = NULL; for(uint i = 0; i < 43; i++) m_frame_backward[i] = NULL; for(uint i = 0; i < 21; i++) m_frame_accumulated[i] = NULL; for(uint i = 0; i < 5; i++) m_frame_work[i] = NULL; cNeck = NULL; ZeroMemory(m_config); ZeroMemory(m_composition); ZeroMemory(m_admitted_shape); ZeroMemory(m_admitted_config); ZeroMemory(m_admitted_composition); m_shape_admitted = false; m_contract_admitted = false; m_transfer_pending = false; m_transfer_target = NULL; ArrayInitialize(m_size, 0); ArrayInitialize(m_input, 0); ArrayInitialize(m_output, 0); ArrayInitialize(m_rows, 0); ArrayInitialize(m_input_slot, 0); ArrayInitialize(m_activation, (uint)None); for(uint i = 0; i < 22; i++) m_affine[i] = NULL; for(uint i = 0; i < 84; i++) m_tape[i] = NULL; m_r = 0; m_f = 0; m_capacity = 0; m_phase = 0; m_update_count = 0; m_generation = 0; m_weight_generation = 0; m_identity_admitted = false; m_cache_valid = false; m_configured = false; m_public_valid = false; m_failed = false; m_status = DPCC_CONTRACT_MISMATCH; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_source_index = INVALID_HANDLE; m_second_index = INVALID_HANDLE; m_market_launches = 0; m_cache_hits = 0; m_next_decision = 0; m_next_context = 0; m_next_owner = 0; InvalidateCache(); } //+------------------------------------------------------------------+ //| Releases each native child and directly owned tape exactly once | //+------------------------------------------------------------------+ CNeuronPlanDenoiserOCL::~CNeuronPlanDenoiserOCL(void) { for(uint i = 0; i < 21; i++) DeleteObj(m_frame_affine[i]); for(uint i = 0; i < 24; i++) DeleteObj(m_frame_common[i]); for(int i = 0; i < ArraySize(m_frame_tapes); i++) DeleteObj(m_frame_tapes[i]); for(uint i = 0; i < 43; i++) DeleteObj(m_frame_backward[i]); for(uint i = 0; i < 21; i++) DeleteObj(m_frame_accumulated[i]); for(uint i = 0; i < 5; i++) DeleteObj(m_frame_work[i]); for(uint i = 0; i < 22; i++) DeleteObj(m_affine[i]); for(uint i = 0; i < 84; i++) DeleteObj(m_tape[i]); } //+------------------------------------------------------------------+ //| Invalidates published output after a factual failure | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Fail(const ENUM_DPCC_STATUS status, const bool fatal) { m_status = status; m_public_valid = false; if(fatal) m_failed = true; return(false); } //+------------------------------------------------------------------+ //| Resets market cache provenance without touching parameters | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::InvalidateCache(void) { m_cache_valid = false; m_cache_decision = 0; m_cache_context = 0; m_cache_owner = 0; m_cache_weights = 0; } //+------------------------------------------------------------------+ //| Checks widened graph extents before forming native device | //| indices | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Geometry(void) { const SDPCCShape shape = m_config.shape; if(shape.K == 0 || shape.V == 0 || shape.H == 0 || shape.D == 0 || shape.M == 0 || shape.J == 0 || shape.E == 0 || shape.PlanBatch == 0 || shape.NActions != 6 || shape.StateSize != 13 || shape.Depth != 1 || shape.Heads == 0 || shape.Embedding == 0 || (shape.KeyDimension != 4 && shape.KeyDimension != 8 && shape.KeyDimension != 16 && shape.KeyDimension != 32 && shape.KeyDimension != 64) || (ulong)shape.Heads * shape.KeyDimension != shape.Embedding) return(Fail(DPCC_CONTRACT_MISMATCH, true)); const uint primitives[] = {shape.K, shape.V, shape.H, shape.D, shape.M, shape.J, shape.E, shape.PlanBatch, shape.Embedding, shape.Heads, shape.KeyDimension }; for(uint i = 0; i < (uint)ArraySize(primitives); i++) if(primitives[i] > 16777216) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(shape.K > 64 || shape.J > 64) return(Fail(DPCC_CONTRACT_MISMATCH, true)); const ulong l = shape.PlanBatch; const ulong h = shape.H; const ulong k = shape.K; const ulong w = shape.Embedding; const ulong r = (h + 1) / 2; const ulong bound = UINT_MAX / sizeof(float); ulong f = 0; ulong lh = 0; ulong low_rows = 0; ulong kh = 0; ulong kw = 0; ulong hw = 0; ulong lhw = 0; ulong lrw = 0; ulong khw = 0; ulong khf = 0; ulong lrkw = 0; ulong lh6 = 0; ulong cross_flags = 0; if(!DPCCRolloutProduct(shape.V, (ulong)shape.D + 1, f) || f >= bound) return(Fail(DPCC_CONTRACT_MISMATCH, true)); f++; if(!DPCCRolloutProduct(l, h, lh)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(l, r, low_rows)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(k, h, kh)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(k, w, kw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(h, w, hw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(lh, w, lhw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(low_rows, w, lrw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(kh, w, khw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(kh, f, khf) || khf > bound - 61) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(low_rows, kw, lrkw)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(lh, 6, lh6)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!DPCCRolloutProduct(MathMax(low_rows, h), kw, cross_flags)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); const ulong basic[] = {r, f, lh, low_rows, kh, kw, hw, lhw, lrw, lrkw, khf + 61, 2 * khw, 2 * lhw, 2 * lrw, lh6, cross_flags }; for(uint i = 0; i < (uint)ArraySize(basic); i++) if(basic[i] == 0 || basic[i] > bound || basic[i] > INT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_r = (uint)r; m_f = (uint)f; const ulong sizes[] = { lh6, khf + 61, khf, 45, 16, hw, lhw, lhw, 2 * lhw, lhw, 2 * lrw, lrw, lrw, lrkw, lrw, 2 * lrw, lrw, lhw, lhw, khf + 61, lhw, 2 * lhw, lhw, lhw / shape.KeyDimension, lrw, 2 * lrw, lrw, lrw / shape.KeyDimension, lrkw, 2 * khw, lrkw / shape.KeyDimension, 1, 1, 1, 16, lh6, khf + 61, khf, 45, 16, lhw, lhw, 2 * lrw, lrw, lrw, lrw, lrw, 2 * lrw, lhw, lhw, lhw, lhw, lhw, lhw, 2 * lhw, lrw, lrw, 2 * lrw, lhw, 2 * lhw, lhw, lrw, 2 * lrw, lrw, lrkw, 2 * khw, lrkw, lhw, lhw, lrw, lrw, lrw, khw, khw, khf, khf, khf, 45, lhw, lrw, lhw, lrw, lhw, lrw }; if(ArraySize(sizes) != 84) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_capacity = 1; for(uint i = 0; i < 84; i++) { if(sizes[i] == 0 || sizes[i] > bound || sizes[i] > INT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_size[i] = (uint)sizes[i]; m_capacity = MathMax(m_capacity, m_size[i]); } const uint iw[] = {6, 45, 16, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)(2 * w), (uint)(2 * w), (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)f, (uint)f, (uint)kw, (uint)w, (uint)(2 * w), (uint)w, (uint)w }; const uint ow[] = {(uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)(2 * w), (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)w, (uint)(2 * w), (uint)w, (uint)(2 * w), 6 }; const uint rows[] = {(uint)lh, 1, 1, (uint)lh, (uint)lh, (uint)lh, (uint)lh, (uint)lh, (uint)lh, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)kh, (uint)kh, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)low_rows, (uint)lh }; const uint inputs[] = {0, 3, 4, 6, 6, 6, 80, 7, 8, 10, 11, 11, 11, 81, 12, 2, 2, 13, 14, 15, 16, 18 }; for(uint i = 0; i < 22; i++) { m_input[i] = iw[i]; m_output[i] = ow[i]; m_rows[i] = rows[i]; m_input_slot[i] = inputs[i]; m_activation[i] = (uint)((i == 7 || i == 9 || i == 18) ? TANH : None); ulong parameters = 0; if(!DPCCRolloutProduct(m_output[i], (ulong)m_input[i] + 1, parameters) || parameters > bound) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_capacity = MathMax(m_capacity, (uint)parameters); } const ulong flags = MathMax(lhw, MathMax(lrw, cross_flags)); if(flags == 0 || flags > bound || flags > INT_MAX) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_capacity = MathMax(m_capacity, (uint)flags); m_size[31] = m_capacity; m_size[33] = m_capacity; return(true); } //+------------------------------------------------------------------+ //| Checks exact device ownership and buffer element count | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ValidBuffer(CBufferFloat *buffer, const uint total) const { return(CheckPointer(buffer) != POINTER_INVALID && buffer.Total() == (int)total && buffer.GetIndex() >= 0 && buffer.GetOpenCL() == OpenCL && OpenCL.GetBufferSize(buffer.GetIndex()) == (ulong)total * sizeof(float)); } //+------------------------------------------------------------------+ //| Checks all native affine geometry and actual optimizer counters | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ValidFramePrefix(CBufferFloat *buffer, const uint count) const { if(!OpenCL || count == 0 || count > m_frame_capacity || CheckPointer(buffer) == POINTER_INVALID || buffer.Total() < (int)count || buffer.Total() > (int)m_frame_capacity || buffer.GetIndex() < 0 || buffer.GetOpenCL() != OpenCL) return(false); return(OpenCL.GetBufferSize(buffer.GetIndex()) == (ulong)buffer.Total() * sizeof(float)); } //+------------------------------------------------------------------+ //| Validates the denoiser buffers and training state. | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ValidateOwned(void) const { if(!OpenCL || m_capacity == 0 || m_config.shape.PlanBatch == 0 || m_update_count >= INT_MAX - 1 || lr != 0.0003f || b1 != 0.9f || b2 != 0.999f || (CheckPointer(Weights) != POINTER_INVALID && Weights.Total() != 0)) return(false); for(uint i = 0; i < 84; i++) if(!ValidBuffer(m_tape[i], m_size[i])) return(false); const uint total = m_config.shape.PlanBatch * m_config.shape.H * 6; if(!ValidBuffer(Output, total) || !ValidBuffer(Gradient, total) || !ValidBuffer(PrevOutput, total)) return(false); for(uint i = 0; i < 22; i++) { CNeuronConvOCL *child = m_affine[i]; if(!child || child.GetWindow() != m_input[i] || child.GetInputStride() != m_input[i] || child.GetFilters() != m_output[i] || child.GetVariables() != 1 || child.GetUnits() != m_rows[i] || child.Activation() != None || child.Optimization() != ADAM || child.TrainMode() != bTrain || (CheckPointer(child.getWeights()) != POINTER_INVALID && child.getWeights().Total() != 0) || child.NativeOptimizerStep() != m_update_count + 1 || !ValidBuffer(child.getOutput(), m_rows[i] * m_output[i]) || !ValidBuffer(child.getGradient(), m_rows[i] * m_output[i]) || !ValidBuffer(child.getPrevOutput(), m_rows[i] * m_output[i]) || !ValidBuffer(child.GetWeightsConvReadOnly(), m_output[i] * (m_input[i] + 1)) || !ValidBuffer(child.GetFirstMomentumConv(), m_output[i] * (m_input[i] + 1)) || !ValidBuffer(child.GetSecondMomentumConv(), m_output[i] * (m_input[i] + 1))) return(false); } int indices[219]; uint cursor = 0; indices[cursor++] = Output.GetIndex(); indices[cursor++] = PrevOutput.GetIndex(); indices[cursor++] = Gradient.GetIndex(); for(uint i = 0; i < 84; i++) indices[cursor++] = m_tape[i].GetIndex(); for(uint i = 0; i < 22; i++) { indices[cursor++] = m_affine[i].getOutput().GetIndex(); indices[cursor++] = m_affine[i].getPrevOutput().GetIndex(); indices[cursor++] = m_affine[i].getGradient().GetIndex(); indices[cursor++] = m_affine[i].GetWeightsConvReadOnly().GetIndex(); indices[cursor++] = m_affine[i].GetFirstMomentumConv().GetIndex(); indices[cursor++] = m_affine[i].GetSecondMomentumConv().GetIndex(); } if(cursor != (uint)ArraySize(indices) || !ArraySort(indices)) return(false); for(uint i = 1; i < cursor; i++) if(indices[i] == indices[i - 1]) return(false); return(true); } //+------------------------------------------------------------------+ //| Allocates the graph tape and optionally its independent children | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Allocate(const bool children) { if(!Geometry()) return(false); if(children) for(uint i = 0; i < 22; i++) { DeleteObj(m_affine[i]); m_affine[i] = new CNeuronConvOCL; if(!m_affine[i]) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_affine[i].Init(0, i, OpenCL, m_input[i], m_input[i], m_output[i], m_rows[i], 1, ADAM, iBatch)) return(Fail(DPCC_OPENCL_ERROR, true)); m_affine[i].SetActivationFunction(None); m_affine[i].TrainMode(bTrain); } for(uint i = 0; i < 84; i++) { DeleteObj(m_tape[i]); m_tape[i] = new CBufferFloat; if(!m_tape[i]) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_tape[i].BufferInit(m_size[i], 0)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!m_tape[i].BufferCreate(OpenCL)) return(Fail(DPCC_OPENCL_ERROR, true)); } const uint w = m_config.shape.Embedding; for(uint h = 0; h < m_config.shape.H; h++) for(uint i = 0; i < w / 2; i++) { const double angle = (double)h * MathPow(10000.0, -(double)(2 * i) / (double)w); m_tape[5].Update(h * w + 2 * i, (float)MathSin(angle)); m_tape[5].Update(h * w + 2 * i + 1, (float)MathCos(angle)); } if(!m_tape[5].BufferWrite()) return(Fail(DPCC_OPENCL_ERROR, true)); return(ValidateOwned()); } //+------------------------------------------------------------------+ //| Admits compiled workgroup and local-memory limits for exact Dh | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ResourceAdmission(COpenCLMy *context, string &device) const { if(!context || context.GetContext() == INVALID_HANDLE) return(false); const uint dh = m_config.shape.KeyDimension; if(dh < 4 || dh > 64 || (dh & (dh - 1)) != 0 || context.GetMaxLocalSize(0) < 1 || context.GetMaxLocalSize(1) < dh || context.GetMaxLocalSize(2) < 1 || context.GetMaxWorkGroupSize() < dh) return(false); long local_memory = 0; if(!context.GetLocalMemorySize(local_memory) || local_memory <= 0) return(false); const uint kernels[] = {def_k_MHFlashAttention, def_k_DPCC_MHFlashAttentionGradient, def_k_DPCCAttentionPreflight }; const long minima[] = {1280, 256, 1280}; for(uint i = 0; i < 3; i++) { const long group = context.GetKernelInfoInteger(kernels[i], CL_KERNEL_WORK_GROUP_SIZE); const long compiled_local = context.GetKernelInfoInteger(kernels[i], CL_KERNEL_LOCAL_MEM_SIZE); if(group < dh || compiled_local < 0 || MathMax(compiled_local, minima[i]) > local_memory) return(false); } if(!CLGetInfoString(context.GetContext(), CL_DEVICE_NAME, device) || StringLen(device) == 0) return(false); return(true); } //+------------------------------------------------------------------+ //| Initializes the exact descriptor through qualified Base Init | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Init(uint outputs, uint index, COpenCLMy *context, CLayerDescription *description) { if(m_frame_mode) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_phase != 0 || m_transfer_pending) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } CLayerDescription *desc = description; if(!desc || !context || outputs != 0 || desc.type != Type() || ArraySize(desc.windows) != 9 || ArraySize(desc.units) != 5 || ArraySize(desc.heads) != 1 || desc.windows[0] != 2 || desc.window != 6 || desc.window_out != 13 || desc.layers != 1 || desc.activation != None || desc.optimization != ADAM || desc.batch == 0) return(Fail(DPCC_CONTRACT_MISMATCH)); m_public_valid = false; m_configured = false; m_failed = true; ZeroMemory(m_config); m_config.shape.K = desc.windows[1]; m_config.shape.V = desc.windows[2]; m_config.shape.H = desc.windows[3]; m_config.shape.D = desc.windows[4]; m_config.shape.M = desc.windows[5]; m_config.shape.J = desc.windows[6]; m_config.shape.E = desc.windows[7]; m_config.shape.PlanBatch = desc.windows[8]; m_config.shape.Embedding = desc.units[0]; m_config.shape.Heads = desc.units[1]; m_config.shape.KeyDimension = desc.units[2]; m_config.shape.NActions = desc.units[3]; m_config.shape.StateSize = desc.units[4]; m_config.shape.Depth = desc.layers; if(!Geometry() || desc.variables != m_config.shape.V || desc.step != m_config.shape.H || (ulong)desc.count != (ulong)m_config.shape.PlanBatch * m_config.shape.H * 6 || desc.heads[0] != m_config.shape.Heads) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!ResourceAdmission(context, m_device)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(!CNeuronBaseOCL::Init(outputs, index, context, desc.count, ADAM, desc.batch)) return(Fail(DPCC_OPENCL_ERROR, true)); activation = None; m_update_count = 0; m_generation = 0; m_weight_generation = 0; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_identity_admitted = false; InvalidateCache(); if(!Allocate(true)) return(false); m_admitted_shape = m_config.shape; m_shape_admitted = true; m_contract_admitted = false; ZeroMemory(m_admitted_config); ZeroMemory(m_admitted_composition); m_failed = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Validates normative existing architecture and parallel fields | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CompositionValid(const SDPCCDenoiserComposition &composition) const { const SDPCCDenoiserArchitecture graph = composition.graph; const SDPCCParallelExecution parallel = composition.parallel; const SDPCCShape shape = m_config.shape; return(graph.architecture == "CONDITIONED_USHAPE_TENSOR_V1" && graph.embedding == shape.Embedding && graph.heads == shape.Heads && graph.key_dimension == shape.KeyDimension && graph.depth == 1 && graph.padding_mask && !graph.temporal_mask && graph.graph_id == "DPCC_DENOISER_GRAPH_V1" && graph.normalization == "INPUT_AFFINE_ONLY" && graph.feedforward_activation == "TANH" && graph.attention_backward == "DPCC_MHFlashAttentionGradient" && graph.backward_profile == "DPCC_EXACT_CHAIN_V1" && graph.graph_sha256 == "5c21551e407bbf1c74f9f7315143f906a05f1cd568312ea562f4f84f02e9f9ce" && composition.optimizer_profile == "NN_NATIVE_CONV_ADAM_C6A8_V1" && parallel.version == 1 && parallel.program == "DPCC_TENSOR_V1" && parallel.training_plan_batch == 1 && parallel.deployment_plan_batch == shape.M && (shape.PlanBatch == parallel.training_plan_batch || shape.PlanBatch == parallel.deployment_plan_batch) && parallel.proposal_slots == 9 && parallel.logical_heads == shape.Heads && parallel.key_dimension == shape.KeyDimension && parallel.shared_projection_weights && parallel.self_attention_fold == "M_X_HEAD" && parallel.cross_attention_fold == "M_X_QUERY_SHARED_KV" && parallel.activation_lifetime == "UNTIL_BACKWARD_AND_UPDATE" && parallel.binary_layer_version == 3 && parallel.legacy_slice_use == "REFERENCE_ONLY"); } //+------------------------------------------------------------------+ //| Requires exact Model and Training composition copies | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CompositionSame(const SDPCCDenoiserComposition &first, const SDPCCDenoiserComposition &second) const { return(first.graph.architecture == second.graph.architecture && first.graph.embedding == second.graph.embedding && first.graph.heads == second.graph.heads && first.graph.key_dimension == second.graph.key_dimension && first.graph.depth == second.graph.depth && first.graph.padding_mask == second.graph.padding_mask && first.graph.temporal_mask == second.graph.temporal_mask && first.graph.graph_id == second.graph.graph_id && first.graph.graph_sha256 == second.graph.graph_sha256 && first.graph.normalization == second.graph.normalization && first.graph.feedforward_activation == second.graph.feedforward_activation && first.graph.attention_backward == second.graph.attention_backward && first.graph.backward_profile == second.graph.backward_profile && first.optimizer_profile == second.optimizer_profile && first.parallel.version == second.parallel.version && first.parallel.program == second.parallel.program && first.parallel.training_plan_batch == second.parallel.training_plan_batch && first.parallel.deployment_plan_batch == second.parallel.deployment_plan_batch && first.parallel.proposal_slots == second.parallel.proposal_slots && first.parallel.logical_heads == second.parallel.logical_heads && first.parallel.key_dimension == second.parallel.key_dimension && first.parallel.shared_projection_weights == second.parallel.shared_projection_weights && first.parallel.self_attention_fold == second.parallel.self_attention_fold && first.parallel.cross_attention_fold == second.parallel.cross_attention_fold && first.parallel.activation_lifetime == second.parallel.activation_lifetime && first.parallel.binary_layer_version == second.parallel.binary_layer_version && first.parallel.legacy_slice_use == second.parallel.legacy_slice_use); } //+------------------------------------------------------------------+ //| Freezes descriptor composition and fixed normalization profile | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Configure(const SDPCCConfig &config, const SDPCCDenoiserComposition &model, const SDPCCDenoiserComposition &training) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(m_failed || !DPCCSameShape(m_config.shape, config.shape) || config.contract_version != 2 || config.binary_layer_version != 3 || config.normalization_mode != DPCC_NORMALIZATION_CAUSAL_AFFINE_V2 || config.loss_coordinates != DPCC_LOSS_NORMALIZED || lr != 0.0003f || b1 != 0.9f || b2 != 0.999f || !CompositionValid(model) || !CompositionValid(training) || !CompositionSame(model, training)) return(Fail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 6; i++) if(!MathIsValidNumber(config.action_bias[i]) || !MathIsValidNumber(config.action_scale[i]) || config.action_scale[i] <= 0 || (m_contract_admitted && (m_admitted_config.action_bias[i] != config.action_bias[i] || m_admitted_config.action_scale[i] != config.action_scale[i]))) return(Fail(DPCC_CONTRACT_MISMATCH)); if(m_contract_admitted && !CompositionSame(m_admitted_composition, model)) return(Fail(DPCC_CONTRACT_MISMATCH)); m_config = config; m_composition = model; m_admitted_shape = config.shape; m_admitted_config = config; m_admitted_composition = model; m_shape_admitted = true; m_contract_admitted = true; m_configured = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Admits explicit cache identity only between completed | //| generations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SetContextIdentity(const ulong decision, const ulong context, const ulong owner) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(m_failed || !m_configured || decision == 0 || context == 0 || owner == 0) return(Fail(DPCC_CONTRACT_MISMATCH)); m_next_decision = decision; m_next_context = context; m_next_owner = owner; m_identity_admitted = true; return(true); } //+------------------------------------------------------------------+ //| Cancels an unwritten generation without reporting backward | //| success | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CancelGeneration(void) { if(m_failed || m_phase == 0) return(Fail(DPCC_CONTRACT_MISMATCH)); m_phase = 0; m_public_valid = false; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_source_index = INVALID_HANDLE; m_second_index = INVALID_HANDLE; m_identity_admitted = false; InvalidateCache(); m_status = DPCC_CONTRACT_MISMATCH; return(true); } //+------------------------------------------------------------------+ //| Reduces every numeric flag to one typed device status | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Reduce(const uint count, uint &status) { if(count == 0 || count > m_capacity) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCOperatorReduce; setBuffer(kernel, 0, m_tape[31].GetIndex()); setBuffer(kernel, 1, m_tape[32].GetIndex()); setArgument(kernel, 2, count); uint offset[1] = {0}; uint size[1] = {1}; if(!FrameExecute(kernel, offset, size)) return(false); if(!m_tape[32].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); status = (uint)m_tape[32][0]; return(true); } //+------------------------------------------------------------------+ //| Fails a consumer before any invalid device result is admitted | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Reduce(const uint count) { uint status = 0; if(!Reduce(count, status)) return(false); if(status != 0) return(Fail((ENUM_DPCC_STATUS)status)); return(true); } //+------------------------------------------------------------------+ //| Executes shared exact activation with numeric status | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Activate(CBufferFloat *source, CBufferFloat *destination, const int function) { if(!source || !destination || source.Total() <= 0 || source.Total() > destination.Total() || source.Total() > (int)m_capacity || source.GetOpenCL() != OpenCL || destination.GetOpenCL() != OpenCL || source.GetIndex() < 0 || destination.GetIndex() < 0) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCActivationForward; const uint count = (uint)source.Total(); setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_tape[31].GetIndex()); setArgument(kernel, 3, function); setArgument(kernel, 4, count); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(count)); } //+------------------------------------------------------------------+ //| Checks raw finite values through a separate scratch owner | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Check(CBufferFloat *source) { return(Activate(source, m_tape[33], None)); } //+------------------------------------------------------------------+ //| Copies equal-size raw payloads after finite admission | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Copy(CBufferFloat *source, CBufferFloat *destination) { if(!source || !destination || source.Total() != destination.Total() || !Check(source)) return(false); if(!CopyBufferRaw(source, destination, (uint)source.Total())) return(Fail(DPCC_OPENCL_ERROR)); return(true); } //+------------------------------------------------------------------+ //| Compares raw saved generation bytes on the device | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Compare(CBufferFloat *source, CBufferFloat *snapshot, const uint count, bool &same) { same = false; if(!ValidBuffer(source, count) || !ValidBuffer(snapshot, count)) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDecoderSnapshotCheck; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, snapshot.GetIndex()); setBuffer(kernel, 2, m_tape[31].GetIndex()); setArgument(kernel, 3, (uint)1); setArgument(kernel, 4, (uint)1); setArgument(kernel, 5, (uint)1); setArgument(kernel, 6, count); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); uint status = 0; if(!Reduce(count, status)) return(false); if(status != 0 && status != (uint)DPCC_INVALID_INPUT) return(Fail((ENUM_DPCC_STATUS)status)); same = status == 0; return(true); } //+------------------------------------------------------------------+ //| Adds residual branches with multiplier one and raw numeric flags | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Sum(CBufferFloat *first, CBufferFloat *second, CBufferFloat *destination) { if(!first || !second || !destination || first.Total() != second.Total() || first.Total() != destination.Total() || !ValidBuffer(first, (uint)first.Total()) || !ValidBuffer(second, (uint)second.Total()) || !ValidBuffer(destination, (uint)destination.Total())) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!SumAndNormalize(first, second, destination, 1, false, 0, 0, 0, 1.0f, m_tape[31])) return(Fail(DPCC_OPENCL_ERROR)); return(Reduce((uint)destination.Total())); } //+------------------------------------------------------------------+ //| Broadcasts a shared vector with the exact checked native | //| operation | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Broadcast(CBufferFloat *vector_buffer, CBufferFloat *matrix_buffer, CBufferFloat *destination, const uint width, const uint rows) { if(!ValidBuffer(vector_buffer, width) || !ValidBuffer(matrix_buffer, width * rows) || !ValidBuffer(destination, width * rows)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!SumVecMatrix(vector_buffer, matrix_buffer, destination, width, 1, 0, 0, 0, 1.0f, rows, m_tape[31])) return(Fail(DPCC_OPENCL_ERROR)); return(Reduce(width * rows)); } //+------------------------------------------------------------------+ //| Sums shared broadcast derivatives in fixed row order | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::BroadcastGradient(CBufferFloat *source, CBufferFloat *destination, const uint rows, const uint width) { if(!ValidBuffer(source, rows * width) || !ValidBuffer(destination, width)) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDenoiserBroadcastGradient; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_tape[31].GetIndex()); setArgument(kernel, 3, rows); setArgument(kernel, 4, width); uint offset[1] = {0}; uint size[1] = {width}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(width)); } //+------------------------------------------------------------------+ //| Validates affine products before the native Conv forward | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AffineForward(const uint index, CBufferFloat *source) { if(index >= 22 || !ValidBuffer(source, m_rows[index] * m_input[index])) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronConvOCL *child = m_affine[index]; const uint kernel = def_k_DPCCSharedAffineForwardPreflight; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, source.GetIndex()); setBuffer(kernel, 2, m_tape[31].GetIndex()); setArgument(kernel, 3, m_rows[index]); setArgument(kernel, 4, m_input[index]); setArgument(kernel, 5, m_output[index]); uint offset[2] = {0, 0}; uint size[2] = {m_output[index], m_rows[index]}; if(!FrameExecute(kernel, offset, size)) return(false); if(!Reduce(m_rows[index] * m_output[index])) return(false); if(!child.FeedForwardBuffer(source)) return(Fail(DPCC_OPENCL_ERROR)); return(Check(child.getOutput())); } //+------------------------------------------------------------------+ //| Computes exact logical activation and old-weight input gradients | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AffineGradient(const uint index, CBufferFloat *upstream, CBufferFloat *destination) { if(index >= 22 || !ValidBuffer(upstream, m_rows[index] * m_output[index]) || !ValidBuffer(destination, m_rows[index] * m_input[index])) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronConvOCL *child = m_affine[index]; const uint activation_kernel = def_k_DPCCActivationGradient; setBuffer(activation_kernel, 0, child.getOutputIndex()); setBuffer(activation_kernel, 1, upstream.GetIndex()); setBuffer(activation_kernel, 2, child.getGradientIndex()); setBuffer(activation_kernel, 3, m_tape[31].GetIndex()); setArgument(activation_kernel, 4, (int)m_activation[index]); setArgument(activation_kernel, 5, m_rows[index] * m_output[index]); uint offset[1] = {0}; uint size[1] = {m_rows[index] * m_output[index]}; if(!FrameExecute(activation_kernel, offset, size)) return(false); if(!Reduce(size[0])) return(false); const uint kernel = def_k_DPCCSharedAffineInputGradient; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, child.getGradientIndex()); setBuffer(kernel, 2, destination.GetIndex()); setBuffer(kernel, 3, m_tape[31].GetIndex()); setArgument(kernel, 4, m_rows[index]); setArgument(kernel, 5, m_input[index]); setArgument(kernel, 6, m_output[index]); uint offset2[2] = {0, 0}; uint size2[2] = {m_input[index], m_rows[index]}; if(!FrameExecute(kernel, offset2, size2)) return(false); return(Reduce(m_rows[index] * m_input[index])); } //+------------------------------------------------------------------+ //| Checks all raw token aggregates and native Adam candidate writes | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AdamPreflight(const uint index) { if(index >= 22) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronConvOCL *child = m_affine[index]; const uint step = child.NativeOptimizerStep(); if(step == 0 || step >= INT_MAX || step != m_update_count + 1) return(Fail(DPCC_CONTRACT_MISMATCH)); const float corrected = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)step)) / (1.0 - MathPow((double)b1, (double)step))); if(!MathIsValidNumber(corrected) || corrected <= 0) return(Fail(DPCC_NUMERIC_ERROR)); const uint kernel = def_k_DPCCSharedAffineAdamPreflight; setBuffer(kernel, 0, child.GetWeightsConv().GetIndex()); setBuffer(kernel, 1, child.getGradientIndex()); setBuffer(kernel, 2, m_tape[m_input_slot[index]].GetIndex()); setBuffer(kernel, 3, child.GetFirstMomentumConv().GetIndex()); setBuffer(kernel, 4, child.GetSecondMomentumConv().GetIndex()); setBuffer(kernel, 5, m_tape[31].GetIndex()); setArgument(kernel, 6, m_rows[index]); setArgument(kernel, 7, m_input[index]); setArgument(kernel, 8, m_output[index]); setArgument(kernel, 9, corrected); setArgument(kernel, 10, b1); setArgument(kernel, 11, b2); uint offset[1] = {0}; uint size[1] = {m_output[index] * (m_input[index] + 1)}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(size[0])); } //+------------------------------------------------------------------+ //| Checks actual native parameter and moment buffers | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CheckParameters(const uint index) { if(index >= 22) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!Check(m_affine[index].GetWeightsConv())) return(false); if(!Check(m_affine[index].GetFirstMomentumConv())) return(false); if(!Check(m_affine[index].GetSecondMomentumConv())) return(false); return(true); } //+------------------------------------------------------------------+ //| Dispatches isolated candidate/head self-attention permutations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SelfLayout(const uint mode, const uint n, CBufferFloat *source0, CBufferFloat *source1, CBufferFloat *source2, CBufferFloat *destination0, CBufferFloat *destination1) { const uint count = m_config.shape.PlanBatch * n * m_config.shape.Embedding; const uint source_size = (mode == 2 ? count * 2 : count); if(mode > 3 || !ValidBuffer(source0, source_size) || !ValidBuffer(destination0, count)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(mode == 0 && (!ValidBuffer(source1, count) || !ValidBuffer(source2, count) || !ValidBuffer(destination1, 2 * count))) return(Fail(DPCC_CONTRACT_MISMATCH)); if(mode == 2 && !ValidBuffer(destination1, count)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(source0.GetIndex() == destination0.GetIndex() || ((mode == 0 || mode == 2) && source0.GetIndex() == destination1.GetIndex())) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *bound_source1 = source0; if(CheckPointer(source1) != POINTER_INVALID) bound_source1 = source1; CBufferFloat *bound_source2 = source0; if(CheckPointer(source2) != POINTER_INVALID) bound_source2 = source2; CBufferFloat *bound_destination1 = destination0; if(CheckPointer(destination1) != POINTER_INVALID) bound_destination1 = destination1; const uint kernel = def_k_DPCCDenoiserSelfLayout; setBuffer(kernel, 0, source0.GetIndex()); setBuffer(kernel, 1, bound_source1.GetIndex()); setBuffer(kernel, 2, bound_source2.GetIndex()); setBuffer(kernel, 3, destination0.GetIndex()); setBuffer(kernel, 4, bound_destination1.GetIndex()); setBuffer(kernel, 5, m_tape[31].GetIndex()); setArgument(kernel, 6, m_config.shape.PlanBatch); setArgument(kernel, 7, n); setArgument(kernel, 8, m_config.shape.Heads); setArgument(kernel, 9, m_config.shape.KeyDimension); setArgument(kernel, 10, mode); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(count)); } //+------------------------------------------------------------------+ //| Dispatches market cross-attention layout and exact sum-K | //| adjoints | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CrossLayout(const uint mode, CBufferFloat *source0, CBufferFloat *source1, CBufferFloat *destination0, CBufferFloat *destination1) { const SDPCCShape shape = m_config.shape; const uint query_rows = shape.PlanBatch * m_r * shape.Embedding; const uint market_rows = shape.K * shape.H * shape.Embedding; const uint source_size = (mode == 0 ? query_rows : (mode == 1 ? market_rows : (mode == 2 ? query_rows * shape.K : market_rows * 2))); const uint count = (mode == 0 ? query_rows * shape.K : (mode == 2 ? query_rows : market_rows)); const uint destination_size = (mode == 1 ? count * 2 : count); if(mode > 3 || !ValidBuffer(source0, source_size) || !ValidBuffer(destination0, destination_size) || source0.GetIndex() == destination0.GetIndex()) return(Fail(DPCC_CONTRACT_MISMATCH)); if(mode == 1 && !ValidBuffer(source1, market_rows)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(mode == 3 && (!ValidBuffer(destination1, market_rows) || source0.GetIndex() == destination1.GetIndex())) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *bound_source1 = source0; if(CheckPointer(source1) != POINTER_INVALID) bound_source1 = source1; CBufferFloat *bound_destination1 = destination0; if(CheckPointer(destination1) != POINTER_INVALID) bound_destination1 = destination1; const uint kernel = def_k_DPCCDenoiserCrossLayout; setBuffer(kernel, 0, source0.GetIndex()); setBuffer(kernel, 1, bound_source1.GetIndex()); setBuffer(kernel, 2, destination0.GetIndex()); setBuffer(kernel, 3, bound_destination1.GetIndex()); setBuffer(kernel, 4, m_tape[31].GetIndex()); setArgument(kernel, 5, shape.PlanBatch); setArgument(kernel, 6, m_r); setArgument(kernel, 7, shape.H); setArgument(kernel, 8, shape.K); setArgument(kernel, 9, shape.Heads); setArgument(kernel, 10, shape.KeyDimension); setArgument(kernel, 11, mode); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(count)); } //+------------------------------------------------------------------+ //| Maps padded pairs and exact cropped inverse derivatives | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::PairMap(CBufferFloat *source, CBufferFloat *destination, const uint mode) { const SDPCCShape shape = m_config.shape; const uint original = shape.PlanBatch * shape.H * shape.Embedding; const uint paired = shape.PlanBatch * m_r * 2 * shape.Embedding; if(mode > 1 || !ValidBuffer(source, mode == 0 ? original : paired) || !ValidBuffer(destination, mode == 0 ? paired : original) || source.GetIndex() == destination.GetIndex()) return(Fail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDenoiserPairMap; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_tape[31].GetIndex()); setArgument(kernel, 3, shape.PlanBatch); setArgument(kernel, 4, shape.H); setArgument(kernel, 5, shape.Embedding); setArgument(kernel, 6, mode); uint offset[1] = {0}; uint size[1] = {mode == 0 ? paired : original}; if(!FrameExecute(kernel, offset, size)) return(false); return(Reduce(size[0])); } //+------------------------------------------------------------------+ //| Runs guarded native Flash forward or the exact DPCC backward | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Attention(const uint block, const bool backward) { if(block > 2) return(Fail(DPCC_CONTRACT_MISMATCH)); const SDPCCShape shape = m_config.shape; const uint qslots[] = {20, 24, 28}; const uint kvslots[] = {21, 25, 29}; const uint outslots[] = {22, 26, 13}; const uint lseslots[] = {23, 27, 30}; const uint dqslots[] = {58, 61, 64}; const uint dkvslots[] = {59, 62, 65}; const uint doutslots[] = {60, 63, 66}; const uint nq = (block == 0 ? shape.H : (block == 1 ? m_r : shape.PlanBatch * m_r)); const uint nk = (block == 0 ? shape.H : (block == 1 ? m_r : shape.H)); const uint groups = (block == 2 ? shape.K * shape.Heads : shape.PlanBatch * shape.Heads); const uint dh = shape.KeyDimension; CBufferFloat *q = m_tape[qslots[block]]; CBufferFloat *kv = m_tape[kvslots[block]]; CBufferFloat *lse = m_tape[lseslots[block]]; CBufferFloat *out = m_tape[outslots[block]]; CBufferFloat *dout = m_tape[doutslots[block]]; string device; if(!ResourceAdmission(OpenCL, device) || device != m_device || !ValidBuffer(q, nq * groups * dh) || !ValidBuffer(kv, nk * 2 * groups * dh) || !ValidBuffer(lse, nq * groups) || !ValidBuffer(out, nq * groups * dh) || !ValidBuffer(dout, nq * groups * dh)) return(Fail(DPCC_CONTRACT_MISMATCH)); uint offset[3] = {0, 0, 0}; uint size[3] = {MathMax(nq, nk), dh, groups}; uint local[3] = {1, dh, 1}; const uint check_kernel = def_k_DPCCAttentionPreflight; setBuffer(check_kernel, 0, q.GetIndex()); setBuffer(check_kernel, 1, kv.GetIndex()); setBuffer(check_kernel, 2, lse.GetIndex()); setBuffer(check_kernel, 3, out.GetIndex()); setBuffer(check_kernel, 4, dout.GetIndex()); setBuffer(check_kernel, 5, m_tape[31].GetIndex()); setArgument(check_kernel, 6, (int)dh); setArgument(check_kernel, 7, (int)nq); setArgument(check_kernel, 8, (int)nk); setArgument(check_kernel, 9, (int)0); setArgument(check_kernel, 10, (uint)(backward ? 1 : 0)); kernelExecuteLoc(check_kernel, offset, size, local) if(!Reduce(size[0] * dh * groups)) return(false); if(backward) { const uint kernel = def_k_DPCC_MHFlashAttentionGradient; setBuffer(kernel, 0, q.GetIndex()); setBuffer(kernel, 1, m_tape[dqslots[block]].GetIndex()); setBuffer(kernel, 2, kv.GetIndex()); setBuffer(kernel, 3, m_tape[dkvslots[block]].GetIndex()); setBuffer(kernel, 4, lse.GetIndex()); setBuffer(kernel, 5, out.GetIndex()); setBuffer(kernel, 6, dout.GetIndex()); setArgument(kernel, 7, (int)dh); setArgument(kernel, 8, (int)nq); setArgument(kernel, 9, (int)nk); setArgument(kernel, 10, (int)0); kernelExecuteLoc(kernel, offset, size, local) if(!Check(m_tape[dqslots[block]])) return(false); return(Check(m_tape[dkvslots[block]])); } const uint kernel = def_k_MHFlashAttention; setBuffer(kernel, 0, q.GetIndex()); setBuffer(kernel, 1, kv.GetIndex()); setBuffer(kernel, 2, lse.GetIndex()); setBuffer(kernel, 3, out.GetIndex()); setArgument(kernel, 4, (int)dh); setArgument(kernel, 5, (int)nk); setArgument(kernel, 6, (int)0); size[0] = nq; kernelExecuteLoc(kernel, offset, size, local) if(!Check(lse)) return(false); return(Check(out)); } //+------------------------------------------------------------------+ //| Builds an exact SELF residual from four independent affines | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SelfForward(const uint block, const uint first, const uint source_slot, const uint destination_slot, const uint n) { for(uint i = 0; i < 3; i++) if(!AffineForward(first + i, m_tape[source_slot])) return(false); const uint q = (block == 0 ? 20 : 24); const uint kv = (block == 0 ? 21 : 25); const uint out = (block == 0 ? 22 : 26); const uint rows = (block == 0 ? 80 : 81); if(!SelfLayout(0, n, m_affine[first].getOutput(), m_affine[first + 1].getOutput(), m_affine[first + 2].getOutput(), m_tape[q], m_tape[kv])) return(false); if(!Attention(block, false)) return(false); if(!SelfLayout(1, n, m_tape[out], NULL, NULL, m_tape[rows], NULL)) return(false); if(!AffineForward(first + 3, m_tape[rows])) return(false); return(Sum(m_tape[source_slot], m_affine[first + 3].getOutput(), m_tape[destination_slot])); } //+------------------------------------------------------------------+ //| Reverses SELF attention and adds the identity branch exactly | //| once | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SelfGradient(const uint block, const uint first, CBufferFloat *upstream, CBufferFloat *destination, const uint n) { const uint scratch0 = (block == 0 ? 52 : 55); const uint scratch1 = (block == 0 ? 53 : 56); const uint scratch2 = (block == 0 ? 82 : 83); const uint dq = (block == 0 ? 58 : 61); const uint dkv = (block == 0 ? 59 : 62); const uint dout = (block == 0 ? 60 : 63); if(!AffineGradient(first + 3, upstream, m_tape[scratch0])) return(false); if(!SelfLayout(3, n, m_tape[scratch0], NULL, NULL, m_tape[dout], NULL)) return(false); if(!Attention(block, true)) return(false); if(!SelfLayout(1, n, m_tape[dq], NULL, NULL, m_affine[first].getGradient(), NULL)) return(false); if(!SelfLayout(2, n, m_tape[dkv], NULL, NULL, m_affine[first + 1].getGradient(), m_affine[first + 2].getGradient())) return(false); if(!AffineGradient(first, m_affine[first].getGradient(), m_tape[scratch0])) return(false); if(!AffineGradient(first + 1, m_affine[first + 1].getGradient(), m_tape[scratch1])) return(false); if(!Sum(m_tape[scratch0], m_tape[scratch1], m_tape[scratch2])) return(false); if(!AffineGradient(first + 2, m_affine[first + 2].getGradient(), m_tape[scratch1])) return(false); if(!Sum(m_tape[scratch2], m_tape[scratch1], m_tape[scratch0])) return(false); return(Sum(upstream, m_tape[scratch0], destination)); } //+------------------------------------------------------------------+ //| Builds exact TANH feedforward and its unit residual | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FeedForwardResidual(const uint first, const uint source_slot, const uint activation_slot, const uint destination_slot) { if(!AffineForward(first, m_tape[source_slot])) return(false); if(!Activate(m_affine[first].getOutput(), m_tape[activation_slot], TANH)) return(false); if(!AffineForward(first + 1, m_tape[activation_slot])) return(false); return(Sum(m_tape[source_slot], m_affine[first + 1].getOutput(), m_tape[destination_slot])); } //+------------------------------------------------------------------+ //| Reverses the TANH feedforward with exact native preactivations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FeedForwardGradient(const uint first, CBufferFloat *upstream, CBufferFloat *destination, const uint scratch0, const uint scratch1) { if(!AffineGradient(first + 1, upstream, m_tape[scratch0])) return(false); if(!AffineGradient(first, m_tape[scratch0], m_tape[scratch1])) return(false); return(Sum(upstream, m_tape[scratch1], destination)); } //+------------------------------------------------------------------+ //| Rejects every external alias of a directly owned buffer | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ExternalBuffer(CBufferFloat *buffer) const { if(!buffer || buffer.GetIndex() < 0 || buffer.GetOpenCL() != OpenCL) return(false); CBufferFloat *base[] = {Output, PrevOutput, Gradient, Weights, FirstMomentum, SecondMomentum, DeltaWeights}; for(uint i = 0; i < (uint)ArraySize(base); i++) if(CheckPointer(base[i]) != POINTER_INVALID && (buffer == base[i] || buffer.GetIndex() == base[i].GetIndex())) return(false); for(uint i = 0; i < 84; i++) if(buffer == m_tape[i] || buffer.GetIndex() == m_tape[i].GetIndex()) return(false); for(uint i = 0; i < 22; i++) { CBufferFloat *owners[] = {m_affine[i].getOutput(), m_affine[i].getGradient(), m_affine[i].getPrevOutput(), m_affine[i].GetWeightsConvReadOnly(), m_affine[i].GetFirstMomentumConv(), m_affine[i].GetSecondMomentumConv() }; for(uint j = 0; j < (uint)ArraySize(owners); j++) if(CheckPointer(owners[j]) != POINTER_INVALID && (buffer == owners[j] || buffer.GetIndex() == owners[j].GetIndex())) return(false); } return(true); } //+------------------------------------------------------------------+ //| Admits one raw plan and one explicit shared market context | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SourceValid(CNeuronBaseOCL *source, CBufferFloat *second) const { if(CheckPointer(source) == POINTER_INVALID || source == GetPointer(this) || source.Activation() != None || !ValidBuffer(source.getOutput(), m_size[0]) || !ValidBuffer(second, m_size[1]) || !ExternalBuffer(source.getOutput()) || !ExternalBuffer(second) || source.getOutput().GetIndex() == second.GetIndex()) return(false); if(bTrain && (!ValidBuffer(source.getGradient(), m_size[0]) || !ExternalBuffer(source.getGradient()) || source.getGradient().GetIndex() == source.getOutput().GetIndex() || source.getGradient().GetIndex() == second.GetIndex())) return(false); return(true); } //+------------------------------------------------------------------+ //| Checks the source object and exact retained device handles | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SameSource(CNeuronBaseOCL *source, CBufferFloat *second) const { if(!SourceValid(source, second) || source != m_source || source.getOutput() != m_source_output || source.getOutputIndex() != m_source_index || second != m_second || second.GetIndex() != m_second_index) return(false); if(m_second_owner && (m_second_owner.Activation() != None || m_second_owner.getOutput() != second || m_second_owner.getOutputIndex() != m_second_index)) return(false); return(true); } //+------------------------------------------------------------------+ //| Rejects any mutation of plan or context in the live transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CheckSnapshot(void) { bool same = false; if(!Compare(m_source_output, m_tape[0], m_size[0], same)) return(false); if(!same) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!Compare(m_second, m_tape[1], m_size[1], same)) return(false); if(!same) return(Fail(DPCC_CONTRACT_MISMATCH)); return(true); } //+------------------------------------------------------------------+ //| Requires explicitly supplied context for every forward | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::feedForward(CNeuronBaseOCL *source) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } return(Fail(DPCC_CONTRACT_MISMATCH)); } //+------------------------------------------------------------------+ //| Executes the complete prescribed G00 through G15 graph | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::feedForward(CNeuronBaseOCL *source, CBufferFloat *second) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } m_public_valid = false; m_status = DPCC_OK; if(m_failed || !m_configured || !ValidateOwned() || !SourceValid(source, second)) return(Fail(DPCC_CONTRACT_MISMATCH)); const SDPCCShape shape = m_config.shape; const uint width = shape.Embedding; const uint rows = shape.PlanBatch * shape.H; if(!Copy(source.getOutput(), m_tape[0])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(second, m_tape[1])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!DeConcat(m_tape[2], m_tape[3], m_tape[4], m_tape[1], (int)m_size[2], 45, 16, 1)) return(Fail(DPCC_OPENCL_ERROR)); bool cache = m_identity_admitted && m_cache_valid && m_cache_weights == m_weight_generation && m_cache_decision == m_next_decision && m_cache_context == m_next_context && m_cache_owner == m_next_owner; if(cache) { bool same = false; if(!Compare(m_tape[2], m_tape[76], m_size[2], same)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); cache = same; if(cache) { if(!Compare(m_tape[3], m_tape[77], 45, same)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); cache = same; } } if(!cache) { InvalidateCache(); if(!AffineForward(1, m_tape[3])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(15, m_tape[2])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(16, m_tape[2])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!CrossLayout(1, m_affine[15].getOutput(), m_affine[16].getOutput(), m_tape[29], NULL)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(m_tape[2], m_tape[76])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(m_tape[3], m_tape[77])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); m_cache_valid = m_identity_admitted; m_cache_decision = m_next_decision; m_cache_context = m_next_context; m_cache_owner = m_next_owner; m_cache_weights = m_weight_generation; m_market_launches++; } else m_cache_hits++; m_identity_admitted = false; if(!AffineForward(0, m_tape[0])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(2, m_tape[4])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Broadcast(m_affine[1].getOutput(), m_affine[0].getOutput(), m_tape[6], width, rows)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Broadcast(m_affine[2].getOutput(), m_tape[6], m_tape[78], width, rows)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Broadcast(m_tape[5], m_tape[78], m_tape[6], shape.H * width, shape.PlanBatch)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!SelfForward(0, 3, 6, 7, shape.H)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!FeedForwardResidual(7, 7, 8, 9)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!PairMap(m_tape[9], m_tape[10], 0)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(9, m_tape[10])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Activate(m_affine[9].getOutput(), m_tape[11], TANH)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!SelfForward(1, 10, 11, 12, m_r)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(14, m_tape[12])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!CrossLayout(0, m_affine[14].getOutput(), NULL, m_tape[28], NULL)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Attention(2, false)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(17, m_tape[13])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Sum(m_tape[12], m_affine[17].getOutput(), m_tape[14])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!FeedForwardResidual(18, 14, 15, 16)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(20, m_tape[16])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!PairMap(m_affine[20].getOutput(), m_tape[17], 1)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Sum(m_tape[17], m_tape[9], m_tape[18])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineForward(21, m_tape[18])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(m_affine[21].getOutput(), Output)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); m_source = source; m_source_output = source.getOutput(); m_source_index = source.getOutputIndex(); m_second = second; m_second_index = second.GetIndex(); m_second_owner = NULL; m_generation++; m_phase = (bTrain ? 1 : 0); m_public_valid = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Admits exact mean epsilon loss before publishing its gradient | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::PrepareTrainingExample(CBufferFloat *raw_actions, CBufferFloat *epsilon, CBufferFloat *context_without_noise, CBufferFloat *account13, const uint j, const double alpha_bar, CBufferFloat *noisy_plan, CBufferFloat *condition) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); SDPCCLayout c, r; uint tokens = 0; if(m_failed || !m_configured || m_transfer_pending || m_phase != 0 || m_config.shape.PlanBatch != 1 || !ValidateOwned() || !DPCCCompositeLayouts(m_config.shape, c, r, tokens) || j == 0 || j > m_config.shape.J || !MathIsValidNumber(alpha_bar) || alpha_bar <= 0 || alpha_bar >= 1) return(Fail(DPCC_CONTRACT_MISMATCH)); const float admitted_alpha = (float)alpha_bar; if(!MathIsValidNumber(admitted_alpha) || admitted_alpha <= 0 || admitted_alpha >= 1) return(Fail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *buffers[] = {raw_actions, epsilon, context_without_noise, account13, noisy_plan, condition}; const uint counts[] = {m_size[0], m_size[0], (uint)c.offsets[10], 13, m_size[0], tokens + 61}; for(uint i = 0; i < 6; i++) { if(!ValidBuffer(buffers[i], counts[i]) || OwnsBufferOrDeviceSlot(buffers[i])) return(Fail(DPCC_CONTRACT_MISMATCH)); for(uint k = 0; k < i; k++) if(BufferAliases(buffers[i], buffers[k])) return(Fail(DPCC_CONTRACT_MISMATCH)); } //--- A newly prepared example is not a forward result or an admitted cache identity. m_public_valid = false; m_identity_admitted = false; //--- Both scratch owners are dead graph readers while phase0 is required. const uint kernel = def_k_DPCCDenoiserPrepare; setBuffer(kernel, 0, raw_actions.GetIndex()); setBuffer(kernel, 1, epsilon.GetIndex()); setBuffer(kernel, 2, m_tape[35].GetIndex()); setBuffer(kernel, 3, m_tape[31].GetIndex()); setArgument(kernel, 4, m_size[0]); setArgument(kernel, 5, admitted_alpha); for(uint i = 0; i < 6; i++) { if(!OpenCL.SetArgument(kernel, 6 + i, m_config.action_bias[i]) || !OpenCL.SetArgument(kernel, 12 + i, m_config.action_scale[i])) return(Fail(DPCC_OPENCL_ERROR)); } uint offset[1] = {0}, size[1] = {m_size[0]}; if(!FrameExecute(kernel, offset, size)) return(false); if(!Reduce(m_size[0])) return(false); if(!DPCCDispatchConditionV1(OpenCL, context_without_noise, account13, m_tape[19], m_tape[31], m_config)) return(Fail(DPCC_OPENCL_ERROR)); if(!Reduce(tokens + 45)) return(false); if(!DPCCWriteConditionStepV1(OpenCL, j, m_config.shape.J, m_tape[19], tokens)) return(Fail(DPCC_OPENCL_ERROR)); if(!CopyBufferRaw(m_tape[35], noisy_plan, m_size[0]) || !CopyBufferRaw(m_tape[19], condition, tokens + 61)) return(Fail(DPCC_OPENCL_ERROR)); m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Builds the epsilon MSE loss and optional gradient. | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::BuildEpsilonMSELoss(CBufferFloat *epsilon, const bool prepare_gradient, double &loss) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); loss = 0; if(m_failed || !m_configured || m_transfer_pending || !m_public_valid || !ValidateOwned() || (prepare_gradient ? (!bTrain || m_phase != 1) : (m_phase != 0 && m_phase != 1)) || !SameSource(m_source, m_second) || !ValidBuffer(epsilon, m_size[0]) || OwnsBufferOrDeviceSlot(epsilon) || m_source.OwnsBufferOrDeviceSlot(epsilon) || BufferAliases(epsilon, m_second) || (m_second_owner && m_second_owner.OwnsBufferOrDeviceSlot(epsilon))) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; if(!CheckSnapshot()) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); //--- Tape35 is the plan-input gradient destination, overwritten by backward. //--- Tape34 is the sixteen-float diagnostic receipt, never a forward reader. const uint kernel = def_k_DPCCDenoiserEpsilonMSE; setBuffer(kernel, 0, Output.GetIndex()); setBuffer(kernel, 1, epsilon.GetIndex()); setBuffer(kernel, 2, m_tape[35].GetIndex()); setBuffer(kernel, 3, m_tape[34].GetIndex()); setArgument(kernel, 4, m_size[0]); uint offset[1] = {0}; uint size[1] = {1}; if(!FrameExecute(kernel, offset, size)) return(false); if(!m_tape[34].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); const uint status = (uint)m_tape[34][0]; const float admitted_loss = m_tape[34][1]; if(status != 0 || !MathIsValidNumber(admitted_loss) || admitted_loss < 0) return(Fail(DPCC_NUMERIC_ERROR)); if(prepare_gradient && !CopyBufferRaw(m_tape[35], Gradient, m_size[0])) return(Fail(DPCC_OPENCL_ERROR)); loss = admitted_loss; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Reject backward without the explicit current secondary buffer | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::calcInputGradients(CNeuronBaseOCL *source) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); return(Fail(DPCC_CONTRACT_MISMATCH)); } //+------------------------------------------------------------------+ //| Reverses every prescribed node using all original native weights | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::calcInputGradients(CNeuronBaseOCL *source, CBufferFloat *second, CBufferFloat *second_gradient, ENUM_ACTIVATION function) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(m_failed || !m_configured || !bTrain || m_phase != 1 || !m_public_valid || !ValidateOwned() || !SameSource(source, second) || function != None) return(Fail(DPCC_CONTRACT_MISMATCH)); if(second_gradient && (!ValidBuffer(second_gradient, m_size[1]) || !ExternalBuffer(second_gradient) || second_gradient.GetIndex() == second.GetIndex() || second_gradient.GetIndex() == source.getOutputIndex() || second_gradient.GetIndex() == source.getGradientIndex())) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; if(!CheckSnapshot()) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(21, Gradient, m_tape[40])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(m_tape[40], m_tape[41])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!PairMap(m_tape[41], m_tape[42], 0)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(20, m_tape[42], m_tape[43])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!FeedForwardGradient(18, m_tape[43], m_tape[44], 57, 55)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(17, m_tape[44], m_tape[66])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Attention(2, true)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!CrossLayout(2, m_tape[64], NULL, m_affine[14].getGradient(), NULL)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!CrossLayout(3, m_tape[65], NULL, m_affine[15].getGradient(), m_affine[16].getGradient())) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(14, m_affine[14].getGradient(), m_tape[55])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Sum(m_tape[44], m_tape[55], m_tape[45])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(15, m_affine[15].getGradient(), m_tape[74])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(16, m_affine[16].getGradient(), m_tape[75])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Sum(m_tape[74], m_tape[75], m_tape[37])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!SelfGradient(1, 10, m_tape[45], m_tape[46], m_r)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(9, m_tape[46], m_tape[47])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!PairMap(m_tape[47], m_tape[48], 1)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Sum(m_tape[40], m_tape[48], m_tape[49])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!FeedForwardGradient(7, m_tape[49], m_tape[50], 54, 52)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!SelfGradient(0, 3, m_tape[50], m_tape[51], m_config.shape.H)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(0, m_tape[51], m_tape[35])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); const uint rows = m_config.shape.PlanBatch * m_config.shape.H; const uint width = m_config.shape.Embedding; if(!BroadcastGradient(m_tape[51], m_affine[1].getGradient(), rows, width)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!BroadcastGradient(m_tape[51], m_affine[2].getGradient(), rows, width)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(1, m_affine[1].getGradient(), m_tape[38])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!AffineGradient(2, m_affine[2].getGradient(), m_tape[39])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Concat(m_tape[37].GetIndex(), m_tape[38].GetIndex(), m_tape[39].GetIndex(), m_tape[39].GetIndex(), m_tape[36].GetIndex(), (int)m_size[37], 45, 16, 0, 1)) return(Fail(DPCC_OPENCL_ERROR)); if(!Check(m_tape[36])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(second_gradient && !Sum(second_gradient, m_tape[36], m_tape[19])) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(!Copy(m_tape[35], source.getGradient())) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); if(second_gradient && !Copy(m_tape[19], second_gradient)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); m_phase = 2; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Rejects an update without an explicit current context | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::updateInputWeights(CNeuronBaseOCL *source) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); return(Fail(DPCC_CONTRACT_MISMATCH)); } //+------------------------------------------------------------------+ //| Admits all Adam candidates before the first of 22 native writes | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::updateInputWeights(CNeuronBaseOCL *source, CBufferFloat *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(m_failed || !m_configured || !bTrain || m_phase != 2 || !m_public_valid || !ValidateOwned() || !SameSource(source, second) || m_update_count >= INT_MAX - 1) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; if(!CheckSnapshot()) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); for(uint i = 0; i < 22; i++) if(!AdamPreflight(i)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); for(uint i = 0; i < 22; i++) if(!m_affine[i].UpdateInputWeightsBuffer(m_tape[m_input_slot[i]])) return(Fail(DPCC_OPENCL_ERROR, true)); m_update_count++; m_weight_generation++; InvalidateCache(); for(uint i = 0; i < 22; i++) if(!CheckParameters(i)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status, true)); if(!ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_phase = 0; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Completes frozen backward without changing any native state | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Complete(CNeuronBaseOCL *source, CBufferFloat *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(m_failed || !m_configured || !bTrain || m_phase != 2 || !m_public_valid || !ValidateOwned() || !SameSource(source, second)) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; if(!CheckSnapshot()) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status)); m_phase = 0; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Dispatches exact forward without parent training shortcuts | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FeedForward(CObject *source, CBufferFloat *second) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } CNeuronBaseOCL *neuron = dynamic_cast(source); if(!neuron) return(Fail(DPCC_INVALID_INPUT)); return(feedForward(neuron, second)); } //+------------------------------------------------------------------+ //| Admits explicit secondary object ownership and linear context | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FeedForwardSecond(CObject *source, CNeuronBaseOCL *second) { if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(!second || second.Activation() != None) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!FeedForward(source, second.getOutput())) return(false); m_second_owner = second; return(true); } //+------------------------------------------------------------------+ //| Dispatches the one completed native optimizer transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::UpdateInputWeights(CObject *source, CBufferFloat *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); CNeuronBaseOCL *neuron = dynamic_cast(source); if(!neuron) return(Fail(DPCC_INVALID_INPUT)); return(updateInputWeights(neuron, second)); } //+------------------------------------------------------------------+ //| Dispatches updates with the same admitted secondary object | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::UpdateInputWeightsSecond(CObject *source, CNeuronBaseOCL *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(!second || second != m_second_owner || second.Activation() != None) return(Fail(DPCC_CONTRACT_MISMATCH)); return(UpdateInputWeights(source, second.getOutput())); } //+------------------------------------------------------------------+ //| Completes the ordinary CNet frozen-buffer backward route | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::UpdateInputState(CObject *source, CBufferFloat *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); CNeuronBaseOCL *neuron = dynamic_cast(source); if(!neuron) return(Fail(DPCC_INVALID_INPUT)); return(Complete(neuron, second)); } //+------------------------------------------------------------------+ //| Completes the frozen route with the original secondary owner | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::UpdateInputStateSecond(CObject *source, CNeuronBaseOCL *second) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(!second || second != m_second_owner || second.Activation() != None) return(Fail(DPCC_CONTRACT_MISMATCH)); return(UpdateInputState(source, second.getOutput())); } //+------------------------------------------------------------------+ //| Keeps the public noisy-plan coordinates physically linear | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::SetActivationFunction(ENUM_ACTIVATION value) { if(value != None) Fail(DPCC_CONTRACT_MISMATCH); } //+------------------------------------------------------------------+ //| Changes train mode only after all live readers have completed | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::TrainMode(bool flag) { if(m_frame_mode) { if(flag) FrameFail(DPCC_BACKWARD_UNSUPPORTED); return; } if(m_phase != 0 || m_transfer_pending) { m_status = DPCC_CONTRACT_MISMATCH; return; } bTrain = flag; for(uint i = 0; i < 22; i++) if(m_affine[i]) m_affine[i].TrainMode(flag); } //+------------------------------------------------------------------+ //| Clears transient graph values while preserving the fixed PE | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Clear(void) { if(m_frame_mode) { if(m_frame_live || !FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH)); ArrayInitialize(m_frame_slots, 0); m_frame_failed = false; m_frame_training = false; m_frame_gradients_ready = false; m_frame_updated = false; m_frame_update_started = false; m_frame_backward_next = 0; m_frame_backward_count = 0; m_frame_cache_valid = false; m_frame_cache_context = NULL; m_frame_cache_decision = 0; m_frame_cache_market = 0; m_frame_cache_weights = 0; m_frame_decision = 0; m_frame_count = 0; m_frame_active_rows = 0; m_frame_next = 0; m_frame_current = 0; m_public_valid = false; m_status = DPCC_OK; return(true); } if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(m_failed || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); m_public_valid = false; if(!CNeuronBaseOCL::Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); if(!PrevOutput.Fill(0)) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 22; i++) if(!m_affine[i].Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 84; i++) if(i != 5 && !m_tape[i].Fill(0)) return(Fail(DPCC_OPENCL_ERROR, true)); InvalidateCache(); m_identity_admitted = false; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_source_index = INVALID_HANDLE; m_second_index = INVALID_HANDLE; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Hashes structural parameters and fixed normalization without L/M | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::HashComposition(ulong &hash) const { const SDPCCShape shape = m_config.shape; const uint dimensions[] = {(uint)Type(), shape.K, shape.V, shape.H, shape.D, shape.Embedding, shape.Heads, shape.KeyDimension, shape.Depth, shape.NActions, shape.StateSize, m_config.normalization_mode, m_config.loss_coordinates, m_composition.parallel.version, m_composition.parallel.proposal_slots, m_composition.parallel.binary_layer_version }; for(uint i = 0; i < (uint)ArraySize(dimensions); i++) DPCCRolloutHashWord(dimensions[i], hash); const string texts[] = {m_composition.graph.architecture, m_composition.graph.graph_id, m_composition.graph.graph_sha256, m_composition.graph.normalization, m_composition.graph.feedforward_activation, m_composition.graph.attention_backward, m_composition.graph.backward_profile, m_composition.optimizer_profile, m_composition.parallel.program, m_composition.parallel.self_attention_fold, m_composition.parallel.cross_attention_fold, m_composition.parallel.activation_lifetime, m_composition.parallel.legacy_slice_use }; for(uint i = 0; i < (uint)ArraySize(texts); i++) { DPCCRolloutHashWord(StringLen(texts[i]), hash); for(int j = 0; j < StringLen(texts[i]); j++) DPCCRolloutHashWord(StringGetCharacter(texts[i], j), hash); } for(uint i = 0; i < 6; i++) { URAGFloatBits raw; raw.value = m_config.action_bias[i]; DPCCRolloutHashWord(raw.bits, hash); raw.value = m_config.action_scale[i]; DPCCRolloutHashWord(raw.bits, hash); } for(uint i = 0; i < 22; i++) { DPCCRolloutHashWord(i, hash); DPCCRolloutHashWord(m_input[i], hash); DPCCRolloutHashWord(m_output[i], hash); DPCCRolloutHashWord(m_activation[i], hash); } } //+------------------------------------------------------------------+ //| Hashes the 22 named weight families independently of moments | //+------------------------------------------------------------------+ ulong CNeuronPlanDenoiserOCL::WeightSignature(void) { if(m_failed || !m_configured || !ValidateOwned()) return(0); ulong hash = 1469598103934665603; HashComposition(hash); for(uint i = 0; i < 22; i++) if(!AppendParameterBufferFingerprint(m_affine[i].GetWeightsConv(), hash, i + 1)) return(0); return(hash); } //+------------------------------------------------------------------+ //| Hashes every execution extent and its immutable computation | //| graph | //+------------------------------------------------------------------+ ulong CNeuronPlanDenoiserOCL::ContractSignature(void) const { if(m_failed || !m_configured) return(0); ulong hash = 1469598103934665603; HashComposition(hash); const uint values[] = {m_config.contract_version, m_config.binary_layer_version, m_config.shape.M, m_config.shape.J, m_config.shape.E, m_config.shape.PlanBatch, m_composition.parallel.training_plan_batch, m_composition.parallel.deployment_plan_batch, m_r, m_f, m_capacity }; for(uint i = 0; i < (uint)ArraySize(values); i++) DPCCRolloutHashWord(values[i], hash); for(uint i = 0; i < 84; i++) DPCCRolloutHashWord(m_size[i], hash); for(uint i = 0; i < 22; i++) { DPCCRolloutHashWord(m_rows[i], hash); DPCCRolloutHashWord(m_input_slot[i], hash); } return(hash); } //+------------------------------------------------------------------+ //| Hashes all native weights, moments and actual child optimizer t | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AppendParameterFingerprint(ulong &fingerprint) { if(m_failed || !m_configured || !ValidateOwned()) return(false); HashComposition(fingerprint); DPCCRolloutHashWord(m_update_count, fingerprint); for(uint i = 0; i < 22; i++) { DPCCRolloutHashWord(i, fingerprint); DPCCRolloutHashWord(m_affine[i].NativeOptimizerStep(), fingerprint); if(!m_affine[i].AppendParameterFingerprint(fingerprint)) return(false); } return(true); } //+------------------------------------------------------------------+ //| Returns the complete independent native trainable fingerprint | //+------------------------------------------------------------------+ ulong CNeuronPlanDenoiserOCL::TrainableStateFingerprint(void) { ulong hash = 1469598103934665603; return(AppendParameterFingerprint(hash) ? hash : 0); } //+------------------------------------------------------------------+ //| Hashes current tape values separately from persistent parameters | //+------------------------------------------------------------------+ ulong CNeuronPlanDenoiserOCL::SnapshotFingerprint(void) { if(m_failed || !m_configured || !m_public_valid || !ValidateOwned()) return(0); ulong hash = ContractSignature(); DPCCRolloutHashWord(m_phase, hash); DPCCRolloutHashWord(m_generation, hash); for(uint i = 0; i < 84; i++) if(!AppendParameterBufferFingerprint(m_tape[i], hash, i + 1)) return(0); if(!AppendParameterBufferFingerprint(Output, hash, 85)) return(0); if(!AppendParameterBufferFingerprint(Gradient, hash, 86)) return(0); for(uint i = 0; i < 22; i++) { if(!AppendParameterBufferFingerprint(m_affine[i].getOutput(), hash, 87 + 2 * i)) return(0); if(!AppendParameterBufferFingerprint(m_affine[i].getGradient(), hash, 88 + 2 * i)) return(0); } return(hash); } //+------------------------------------------------------------------+ //| Publishes native counter, phase and shared market-cache | //| diagnostics | //+------------------------------------------------------------------+ CBufferFloat *CNeuronPlanDenoiserOCL::GetDiagnostics(void) { if(m_failed || !m_configured || !ValidateOwned()) return(NULL); const float values[] = {(float)m_status, (float)m_phase, (float)m_update_count, (float)m_generation, (float)m_market_launches, (float)m_cache_hits, (float)m_cache_valid, (float)m_config.shape.PlanBatch, (float)m_config.shape.H, (float)m_config.shape.K, (float)m_config.shape.Embedding, (float)m_config.shape.Heads, (float)m_config.shape.KeyDimension, (float)m_affine[0].NativeOptimizerStep(), (float)m_affine[21].NativeOptimizerStep(), (float)m_weight_generation }; for(uint i = 0; i < 16; i++) m_tape[34].Update(i, values[i]); if(!m_tape[34].BufferWrite()) { Fail(DPCC_OPENCL_ERROR); return(NULL); } return(m_tape[34]); } //+------------------------------------------------------------------+ //| Imports only weights into a genuinely fresh compatible receiver | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(m_frame_mode) return(FrameFail(DPCC_BACKWARD_UNSUPPORTED)); if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(!source || source.Type() != Type() || tau != 1.0f || m_failed || !m_configured || m_update_count != 0 || m_generation != 0 || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); CNeuronPlanDenoiserOCL *donor = source; if(donor == GetPointer(this) || donor.OpenCL != OpenCL || donor.m_failed || !donor.m_configured || donor.m_phase != 0 || !donor.ValidateOwned() || m_config.shape.K != donor.m_config.shape.K || m_config.shape.V != donor.m_config.shape.V || m_config.shape.D != donor.m_config.shape.D || m_config.shape.H != donor.m_config.shape.H || m_config.shape.Embedding != donor.m_config.shape.Embedding || m_f != donor.m_f || m_config.shape.Heads != donor.m_config.shape.Heads || m_config.shape.KeyDimension != donor.m_config.shape.KeyDimension || m_composition.graph.graph_sha256 != donor.m_composition.graph.graph_sha256 || m_composition.optimizer_profile != donor.m_composition.optimizer_profile) return(Fail(DPCC_CONTRACT_MISMATCH)); ulong receiver_contract = 1469598103934665603; ulong donor_contract = 1469598103934665603; HashComposition(receiver_contract); donor.HashComposition(donor_contract); if(receiver_contract != donor_contract) return(Fail(DPCC_CONTRACT_MISMATCH)); m_status = DPCC_OK; for(uint i = 0; i < 22; i++) { if(!CheckParameters(i) || !donor.CheckParameters(i)) return(Fail(DPCC_NUMERIC_ERROR)); CBufferFloat *moments[] = {m_affine[i].GetFirstMomentumConv(), m_affine[i].GetSecondMomentumConv()}; for(uint j = 0; j < 2; j++) { if(!moments[j].BufferRead()) return(Fail(DPCC_OPENCL_ERROR)); for(int element = 0; element < moments[j].Total(); element++) if(moments[j][element] != 0.0f) return(Fail(DPCC_CONTRACT_MISMATCH)); } } m_public_valid = false; InvalidateCache(); for(uint i = 0; i < 22; i++) if(!m_affine[i].WeightsUpdate(donor.m_affine[i], 1.0f)) return(Fail(DPCC_OPENCL_ERROR, true)); m_weight_generation++; for(uint i = 0; i < 22; i++) if(!CheckParameters(i)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status, true)); if(!Clear()) return(false); m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Blends only ABI3 affine weights after a complete idle preflight | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::UpdateFrameWeightsFrom(CNeuronPlanDenoiserOCL *source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source == GetPointer(this) || source.Type() != Type() || !MathIsValidNumber(tau) || tau < 0 || tau > 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); //--- Validate both transactions before inspecting any shared weight owner. CNeuronPlanDenoiserOCL *owners[] = {GetPointer(this), source}; for(uint side = 0; side < 2; side++) { CNeuronPlanDenoiserOCL *owner = owners[side]; if(!owner.m_frame_mode || !owner.m_frame_bound || owner.m_frame_live || owner.m_frame_failed || owner.m_failed || owner.m_transfer_pending || owner.m_phase != 0 || owner.optimization != ADAM || owner.iBatch != 1 || owner.m_frame_training || owner.m_frame_gradients_ready || owner.m_frame_updated || owner.m_frame_update_started || owner.m_frame_count != 0 || owner.m_frame_active_rows != 0 || owner.m_frame_next != 0 || owner.m_frame_current != 0 || owner.m_frame_backward_next != 0 || owner.m_frame_backward_count != 0 || owner.m_frame_decision != 0 || owner.m_frame_update_count > (uint)(INT_MAX - 1) || !owner.FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint slot = 0; slot < owner.m_frame_max_frames; slot++) if(owner.m_frame_slots[slot] != 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } //--- A fresh ABI3 TargetActor may adopt the source identity only here: //--- CNeuronDPCCOCL::WeightsUpdate has already matched the frozen Market pointers. const bool adopt_market_weights = (m_frame_market_weights == 0 && source.m_frame_market_weights != 0); if(source.OpenCL != OpenCL || m_frame_horizon != source.m_frame_horizon || m_frame_rows != source.m_frame_rows || m_frame_width != source.m_frame_width || m_frame_heads != source.m_frame_heads || m_frame_max_frames != source.m_frame_max_frames || m_frame_k != source.m_frame_k || m_frame_v != source.m_frame_v || m_frame_market_horizon != source.m_frame_market_horizon || m_frame_dimension != source.m_frame_dimension || m_frame_condition_count != source.m_frame_condition_count || m_frame_capacity != source.m_frame_capacity || m_frame_work_size != source.m_frame_work_size || (m_frame_market_weights != source.m_frame_market_weights && !adopt_market_weights) || m_frame_probability_tolerance != source.m_frame_probability_tolerance || !MathIsValidNumber(m_frame_probability_tolerance) || m_frame_probability_tolerance < 0 || m_frame_probability_tolerance >= 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 218; i++) if(!MathIsValidNumber(m_frame_profile[i]) || m_frame_profile[i] != source.m_frame_profile[i] || (i >= 109 && m_frame_profile[i] <= 0)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 43; i++) if(m_frame_tape_size[i] != source.m_frame_tape_size[i]) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 24; i++) if(m_frame_common_size[i] != source.m_frame_common_size[i]) return(FrameFail(DPCC_CONTRACT_MISMATCH)); //--- Preflight all 21 canonical geometries and disjoint parameter buffers. const ulong w = m_frame_width; const ulong rows = (ulong)m_frame_rows * m_frame_horizon; const ulong low_rows = (ulong)m_frame_rows * ((m_frame_horizon + 1) / 2); const ulong market_tokens = (ulong)m_frame_k * m_frame_v * m_frame_market_horizon; const ulong inputs[] = {109, (ulong)m_frame_dimension + 1, (ulong)m_frame_dimension + 1, w, 6, 16, w, w, w, w, w, 2 * w, 2 * w, w, w, w, w, w, 2 * w, w, w }; const ulong outputs[] = {w, w, w, w, w, w, w, w, w, w, 2 * w, w, w, w, w, w, w, 2 * w, w, 2 * w, 6 }; const ulong units[] = {1, market_tokens, market_tokens, 1, rows, 1, rows, rows, rows, rows, rows, rows, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, rows }; CBufferFloat *target_weights[21], *source_weights[21]; for(uint i = 0; i < 21; i++) { ulong count = 0; if(!DPCCRolloutProduct(inputs[i] + 1, outputs[i], count) || count == 0 || count > (ulong)INT_MAX || count > (ulong)UINT_MAX / sizeof(float)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint side = 0; side < 2; side++) { CNeuronPlanDenoiserOCL *owner = owners[side]; CNeuronDPCCFrameAffineOCL *child = owner.m_frame_affine[i]; if(owner.m_frame_input[i] != inputs[i] || owner.m_frame_output[i] != outputs[i] || owner.m_frame_affine_rows[i] != units[i] || child.GetWindow() != inputs[i] || child.GetInputStride() != inputs[i] || child.GetFilters() != outputs[i] || child.GetUnits() != units[i] || child.GetVariables() != 1 || child.Type() != defNeuronConvOCL || child.Optimization() != ADAM || !owner.ValidBuffer(child.GetWeightsConvReadOnly(), (uint)count) || !owner.ValidBuffer(child.GetFirstMomentumConv(), (uint)count) || !owner.ValidBuffer(child.GetSecondMomentumConv(), (uint)count) || BufferAliases(child.GetWeightsConvReadOnly(), child.GetFirstMomentumConv()) || BufferAliases(child.GetWeightsConvReadOnly(), child.GetSecondMomentumConv())) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } target_weights[i] = m_frame_affine[i].GetWeightsConvReadOnly(); source_weights[i] = source.m_frame_affine[i].GetWeightsConvReadOnly(); if(source.OwnsBufferOrDeviceSlot(target_weights[i]) || OwnsBufferOrDeviceSlot(source_weights[i])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint j = 0; j < 21; j++) if(i != j && (m_frame_affine[j].OwnsBufferOrDeviceSlot(target_weights[i]) || source.m_frame_affine[j].OwnsBufferOrDeviceSlot(source_weights[i]))) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } //--- Preserve Adam moments and clocks; an attempted dispatch may have written. const uint kernel = def_k_SoftUpdate; bool write_started = false; for(uint i = 0; i < 21; i++) { if(!OpenCL.SetArgumentBuffer(kernel, def_k_su_target, target_weights[i].GetIndex())) return(FrameFail(DPCC_OPENCL_ERROR, write_started)); if(!OpenCL.SetArgumentBuffer(kernel, def_k_su_source, source_weights[i].GetIndex())) return(FrameFail(DPCC_OPENCL_ERROR, write_started)); if(!OpenCL.SetArgument(kernel, def_k_su_tau, tau)) return(FrameFail(DPCC_OPENCL_ERROR, write_started)); uint offset[1] = {0}; uint size[1] = {(uint)target_weights[i].Total()}; write_started = true; if(!OpenCL.Execute(kernel, 1, offset, size)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } if(adopt_market_weights) m_frame_market_weights = source.m_frame_market_weights; m_frame_cache_valid = false; m_frame_cache_context = NULL; m_frame_cache_decision = m_frame_cache_market = m_frame_cache_weights = 0; m_public_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Admits initial Load context attachment without moving live | //| buffers | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::AttachOpenCL(COpenCLMy *context) { if(m_frame_mode) { if(context != OpenCL) FrameFail(DPCC_CONTRACT_MISMATCH); return; } if(m_phase != 0 || m_capacity != 0) { m_status = DPCC_CONTRACT_MISMATCH; return; } CNeuronBaseOCL::AttachOpenCL(context); } //+------------------------------------------------------------------+ //| Checks the complete exact native child tree transfer capability | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SupportsOpenCLChecked(void) const { if(m_frame_mode) return(!m_frame_live && !m_frame_failed && FrameOwned()); if(Type() != defNeuronPlanDenoiserOCL || (m_failed && !m_transfer_pending) || m_capacity == 0 || (m_transfer_pending && CheckPointer(m_transfer_target) == POINTER_INVALID)) return(false); for(uint i = 0; i < 22; i++) if(!m_affine[i] || !m_affine[i].SupportsOpenCLChecked()) return(false); for(uint i = 0; i < 84; i++) if(!m_tape[i] || m_tape[i].Total() != (int)m_size[i]) return(false); return(true); } //+------------------------------------------------------------------+ //| Transfers all owners before publishing the qualified Base | //| context | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SetOpenCLChecked(COpenCLMy *context) { if(m_frame_mode) return(FrameSetOpenCL(context)); if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(m_transfer_pending && context != m_transfer_target) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } string device; if(!context || !SupportsOpenCLChecked() || !ResourceAdmission(context, device)) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!m_transfer_pending && context == OpenCL && ValidateOwned()) return(true); m_transfer_target = context; m_transfer_pending = true; m_failed = true; m_public_valid = false; InvalidateCache(); m_identity_admitted = false; for(uint i = 0; i < 84; i++) if(m_tape[i].GetIndex() >= 0 && m_tape[i].GetOpenCL() != context && !m_tape[i].BufferRead()) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 22; i++) if(!m_affine[i].SetOpenCLChecked(context)) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 84; i++) if((m_tape[i].GetOpenCL() != context || m_tape[i].GetIndex() < 0) && !m_tape[i].BufferCreate(context)) return(Fail(DPCC_OPENCL_ERROR, true)); if(!CNeuronBaseOCL::SetOpenCLChecked(context)) return(Fail(DPCC_OPENCL_ERROR, true)); m_device = device; if(!ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_transfer_pending = false; m_transfer_target = NULL; m_failed = false; m_source = NULL; m_source_output = NULL; m_second = NULL; m_second_owner = NULL; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Routes legacy rebinding through checked complete graph ownership | //+------------------------------------------------------------------+ void CNeuronPlanDenoiserOCL::SetOpenCL(COpenCLMy *context) { SetOpenCLChecked(context); } //+------------------------------------------------------------------+ //| Reconstructs the exact descriptor without invoking parent UShape | //+------------------------------------------------------------------+ CLayerDescription *CNeuronPlanDenoiserOCL::GetLayerInfo(void) { if(m_failed || m_capacity == 0) return(NULL); CLayerDescription *description = new CLayerDescription; if(!description) return(NULL); const SDPCCShape shape = m_config.shape; description.type = Type(); description.count = shape.PlanBatch * shape.H * 6; description.window = 6; description.window_out = 13; description.variables = shape.V; description.step = shape.H; description.layers = 1; description.batch = iBatch; description.activation = None; description.optimization = ADAM; if(ArrayResize(description.windows, 9) != 9 || ArrayResize(description.units, 5) != 5 || ArrayResize(description.heads, 1) != 1) { DeleteObj(description); return(NULL); } const uint windows[] = {2, shape.K, shape.V, shape.H, shape.D, shape.M, shape.J, shape.E, shape.PlanBatch}; const uint units[] = {shape.Embedding, shape.Heads, shape.KeyDimension, 6, 13}; for(uint i = 0; i < 9; i++) description.windows[i] = windows[i]; for(uint i = 0; i < 5; i++) description.units[i] = units[i]; description.heads[0] = shape.Heads; return(description); } //+------------------------------------------------------------------+ //| Saves Base then exact version-three graph and 22 native children | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Save(const int file_handle) { if(m_frame_mode) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_phase != 0) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(file_handle == INVALID_HANDLE || m_failed || !m_configured || !ValidateOwned()) return(Fail(DPCC_CONTRACT_MISMATCH)); if(!CNeuronBaseOCL::Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); const SDPCCShape shape = m_config.shape; const SDPCCDenoiserArchitecture graph = m_composition.graph; const SDPCCParallelExecution parallel = m_composition.parallel; const uint header[] = {0x44503543, 3, 2, shape.K, shape.V, shape.H, shape.D, shape.M, shape.J, shape.E, shape.PlanBatch, shape.NActions, shape.StateSize, shape.Embedding, shape.Heads, shape.KeyDimension, shape.Depth, m_update_count, m_config.normalization_mode, m_config.loss_coordinates, graph.embedding, graph.heads, graph.key_dimension, graph.depth, (uint)graph.padding_mask, (uint)graph.temporal_mask, parallel.version, parallel.training_plan_batch, parallel.deployment_plan_batch, parallel.proposal_slots, parallel.logical_heads, parallel.key_dimension, (uint)parallel.shared_projection_weights, parallel.binary_layer_version, 84, 22 }; for(uint i = 0; i < (uint)ArraySize(header); i++) if(FileWriteInteger(file_handle, (int)header[i], INT_VALUE) != INT_VALUE) return(Fail(DPCC_INVALID_INPUT)); for(uint i = 0; i < 6; i++) { if(FileWriteFloat(file_handle, m_config.action_bias[i]) != sizeof(float)) return(Fail(DPCC_INVALID_INPUT)); if(FileWriteFloat(file_handle, m_config.action_scale[i]) != sizeof(float)) return(Fail(DPCC_INVALID_INPUT)); } const string texts[] = {graph.architecture, graph.graph_id, graph.graph_sha256, graph.normalization, graph.feedforward_activation, graph.attention_backward, graph.backward_profile, m_composition.optimizer_profile, parallel.program, parallel.self_attention_fold, parallel.cross_attention_fold, parallel.activation_lifetime, parallel.legacy_slice_use }; for(uint i = 0; i < (uint)ArraySize(texts); i++) { const int length = StringLen(texts[i]); if(FileWriteInteger(file_handle, length, INT_VALUE) != INT_VALUE) return(Fail(DPCC_INVALID_INPUT)); if(FileWriteString(file_handle, texts[i], length) == 0) return(Fail(DPCC_INVALID_INPUT)); } for(uint i = 0; i < 22; i++) if(!m_affine[i].Save(file_handle)) return(Fail(DPCC_INVALID_INPUT)); if(FileWriteInteger(file_handle, 0x454E4435, INT_VALUE) != INT_VALUE) return(Fail(DPCC_INVALID_INPUT)); return(true); } //+------------------------------------------------------------------+ //| Loads saved geometry with actual counters and no live | //| transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::Load(const int file_handle) { if(m_frame_mode) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_transfer_pending || (m_phase != 0 && !m_failed)) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(file_handle == INVALID_HANDLE || !OpenCL) return(Fail(DPCC_INVALID_INPUT, true)); const bool existing_shape = m_shape_admitted; const bool existing_contract = m_contract_admitted; const SDPCCShape expected_shape = m_admitted_shape; const SDPCCConfig expected = m_admitted_config; const SDPCCDenoiserComposition expected_composition = m_admitted_composition; m_public_valid = false; m_configured = false; m_failed = true; m_phase = 0; InvalidateCache(); m_identity_admitted = false; if(!CNeuronBaseOCL::Load(file_handle)) return(Fail(DPCC_INVALID_INPUT, true)); uint header[36]; if(FileReadArray(file_handle, header, 0, 36) != 36 || header[0] != 0x44503543 || header[1] != 3 || header[2] != 2 || header[17] >= INT_MAX - 1 || header[24] > 1 || header[25] > 1 || header[32] > 1 || header[34] != 84 || header[35] != 22) return(Fail(DPCC_CONTRACT_MISMATCH, true)); ZeroMemory(m_config); m_config.shape.K = header[3]; m_config.shape.V = header[4]; m_config.shape.H = header[5]; m_config.shape.D = header[6]; m_config.shape.M = header[7]; m_config.shape.J = header[8]; m_config.shape.E = header[9]; m_config.shape.PlanBatch = header[10]; m_config.shape.NActions = header[11]; m_config.shape.StateSize = header[12]; m_config.shape.Embedding = header[13]; m_config.shape.Heads = header[14]; m_config.shape.KeyDimension = header[15]; m_config.shape.Depth = header[16]; m_config.contract_version = header[2]; m_config.binary_layer_version = header[1]; m_config.normalization_mode = header[18]; m_config.loss_coordinates = header[19]; if(!Geometry() || activation != None || optimization != ADAM || iBatch == 0 || !ValidBuffer(Output, m_size[0]) || !ValidBuffer(PrevOutput, m_size[0]) || !ValidBuffer(Gradient, m_size[0]) || !ResourceAdmission(OpenCL, m_device)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); if(existing_shape && !DPCCSameShape(expected_shape, m_config.shape)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); for(uint i = 0; i < 6; i++) { if(FileIsEnding(file_handle)) return(Fail(DPCC_INVALID_INPUT, true)); m_config.action_bias[i] = FileReadFloat(file_handle); if(FileIsEnding(file_handle)) return(Fail(DPCC_INVALID_INPUT, true)); m_config.action_scale[i] = FileReadFloat(file_handle); if(!MathIsValidNumber(m_config.action_bias[i]) || !MathIsValidNumber(m_config.action_scale[i]) || m_config.action_scale[i] <= 0 || (existing_contract && (m_config.action_bias[i] != expected.action_bias[i] || m_config.action_scale[i] != expected.action_scale[i]))) return(Fail(DPCC_CONTRACT_MISMATCH, true)); } string texts[13]; for(uint i = 0; i < 13; i++) { if(FileIsEnding(file_handle)) return(Fail(DPCC_INVALID_INPUT, true)); const int length = FileReadInteger(file_handle, INT_VALUE); if(length <= 0 || length > 128) return(Fail(DPCC_CONTRACT_MISMATCH, true)); texts[i] = FileReadString(file_handle, length); if(StringLen(texts[i]) != length) return(Fail(DPCC_INVALID_INPUT, true)); } ZeroMemory(m_composition); m_composition.graph.architecture = texts[0]; m_composition.graph.graph_id = texts[1]; m_composition.graph.graph_sha256 = texts[2]; m_composition.graph.normalization = texts[3]; m_composition.graph.feedforward_activation = texts[4]; m_composition.graph.attention_backward = texts[5]; m_composition.graph.backward_profile = texts[6]; m_composition.optimizer_profile = texts[7]; m_composition.parallel.program = texts[8]; m_composition.parallel.self_attention_fold = texts[9]; m_composition.parallel.cross_attention_fold = texts[10]; m_composition.parallel.activation_lifetime = texts[11]; m_composition.parallel.legacy_slice_use = texts[12]; m_composition.graph.embedding = header[20]; m_composition.graph.heads = header[21]; m_composition.graph.key_dimension = header[22]; m_composition.graph.depth = header[23]; m_composition.graph.padding_mask = (bool)header[24]; m_composition.graph.temporal_mask = (bool)header[25]; m_composition.parallel.version = header[26]; m_composition.parallel.training_plan_batch = header[27]; m_composition.parallel.deployment_plan_batch = header[28]; m_composition.parallel.proposal_slots = header[29]; m_composition.parallel.logical_heads = header[30]; m_composition.parallel.key_dimension = header[31]; m_composition.parallel.shared_projection_weights = (bool)header[32]; m_composition.parallel.binary_layer_version = header[33]; if(m_config.normalization_mode != DPCC_NORMALIZATION_CAUSAL_AFFINE_V2 || m_config.loss_coordinates != DPCC_LOSS_NORMALIZED || !CompositionValid(m_composition) || (existing_contract && !CompositionSame(expected_composition, m_composition))) return(Fail(DPCC_CONTRACT_MISMATCH, true)); for(uint i = 0; i < 22; i++) { DeleteObj(m_affine[i]); m_affine[i] = new CNeuronConvOCL; if(!m_affine[i]) return(Fail(DPCC_OPENCL_ERROR, true)); m_affine[i].AttachOpenCL(OpenCL); if(FileIsEnding(file_handle) || FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !m_affine[i].Load(file_handle)) return(Fail(DPCC_INVALID_INPUT, true)); } if(FileIsEnding(file_handle) || FileReadInteger(file_handle, INT_VALUE) != 0x454E4435) return(Fail(DPCC_INVALID_INPUT, true)); m_update_count = header[17]; m_generation = 0; m_weight_generation++; if(!Allocate(false)) return(Fail(DPCC_CONTRACT_MISMATCH, true)); m_failed = false; m_configured = true; if(!Clear()) return(Fail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < 22; i++) { if(!CheckParameters(i)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status, true)); if(!AdamPreflight(i)) return(Fail(m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status, true)); } m_admitted_shape = m_config.shape; m_admitted_config = m_config; m_admitted_composition = m_composition; m_shape_admitted = true; m_contract_admitted = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Invalidates an incomplete Action6 forward transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameFail(const ENUM_DPCC_STATUS status, const bool fatal) { m_frame_cache_valid = false; if(m_frame_live) m_frame_failed = true; return(Fail(status, fatal)); } //+------------------------------------------------------------------+ //| Selects the Action6 frame contract on a fresh denoiser | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::InitFrameShape(uint horizon, uint rows, uint channels, uint width, uint heads, uint max_frames) { //--- Admit an isolated shape before any legacy graph has been created. if(m_frame_mode || m_capacity != 0 || m_phase != 0 || m_configured || m_failed || !OpenCL || horizon == 0 || rows == 0 || channels != 6 || heads == 0 || width == 0 || width % heads != 0 || max_frames == 0 || max_frames > 64 || (ulong)horizon * rows > (ulong)INT_MAX / 6 || (ulong)horizon * rows > (ulong)UINT_MAX / sizeof(float) / MathMax(width, (uint)6)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint dimension = width / heads; if(dimension < 4 || dimension > 64 || (dimension & (dimension - 1)) != 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); m_config.shape.KeyDimension = dimension; string device; if(!ResourceAdmission(OpenCL, device)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); //--- This mode owns one weight set and keeps every requested slot distinct. m_frame_horizon = horizon; m_frame_rows = rows; m_frame_width = width; m_frame_heads = heads; m_frame_max_frames = max_frames; m_frame_mode = true; m_device = device; bTrain = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Binds canonical Market geometry, normalization and cache | //| identity | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::BindFrameCondition(const uint scenarios, const uint variables, const uint market_horizon, const uint latent_dimension, const float &scale[], const float &bias[], const ulong market_state_id, const ulong market_weights_id, const float probability_tolerance) { if(market_state_id == 0 || market_weights_id == 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!InitFrameCondition(scenarios, variables, market_horizon, latent_dimension, scale, bias, probability_tolerance)) return(false); m_frame_market_id = market_state_id; m_frame_market_weights = market_weights_id; return(true); } //+------------------------------------------------------------------+ //| Allocates immutable conditioning before a decision is bound | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::InitFrameCondition(const uint scenarios, const uint variables, const uint market_horizon, const uint latent_dimension, const float &scale[], const float &bias[], const float probability_tolerance) { //--- Reject incomplete or mutable contracts before allocating weights. if(!m_frame_mode || m_frame_live || m_failed || !OpenCL || scenarios == 0 || scenarios > 64 || variables == 0 || market_horizon < m_frame_horizon || latent_dimension == 0 || latent_dimension >= INT_MAX || ArraySize(scale) != 109 || ArraySize(bias) != 109 || !MathIsValidNumber(probability_tolerance) || probability_tolerance < 0 || probability_tolerance >= 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); ulong tokens = 0; ulong scalars = 0; if(!DPCCRolloutProduct(variables, market_horizon, tokens) || !DPCCRolloutProduct(tokens, scenarios, tokens) || !DPCCRolloutProduct(tokens, (ulong)latent_dimension + 1, scalars) || scalars > (ulong)UINT_MAX / sizeof(float) - 109 - scenarios || scalars > INT_MAX - 109 - scenarios || tokens > (ulong)UINT_MAX / sizeof(float) / 2 / m_frame_width) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 109; i++) if(!MathIsValidNumber(scale[i]) || scale[i] <= 0 || !MathIsValidNumber(bias[i]) || (m_frame_bound && (scale[i] != m_frame_profile[109 + i] || bias[i] != m_frame_profile[i]))) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_frame_bound && (m_frame_k != scenarios || m_frame_v != variables || m_frame_market_horizon != market_horizon || m_frame_dimension != latent_dimension || m_frame_probability_tolerance != probability_tolerance || !FrameOwned())) return(FrameFail(DPCC_CONTRACT_MISMATCH)); //--- The packed input is canonical fields109, Z/logU tokens, then original Pi. if(!m_frame_bound) { m_frame_k = scenarios; m_frame_v = variables; m_frame_market_horizon = market_horizon; m_frame_dimension = latent_dimension; m_frame_condition_count = (uint)scalars + 109 + scenarios; m_frame_probability_tolerance = probability_tolerance; for(uint i = 0; i < 109; i++) { m_frame_profile[i] = bias[i]; m_frame_profile[109 + i] = scale[i]; } if(!FrameAllocate()) return(false); m_frame_bound = true; if(!FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); } m_frame_market_id = 0; m_frame_market_weights = 0; m_frame_cache_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Allocates shared affine owners and bounded frame tapes once | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAllocate(void) { const ulong w = m_frame_width; const ulong h = m_frame_horizon; const ulong rows = m_frame_rows; const ulong r = (h + 1) / 2; const ulong lh = rows * h; const ulong low_rows = rows * r; const ulong market_tokens = (ulong)m_frame_k * m_frame_v * m_frame_market_horizon; const ulong bound = MathMin((ulong)INT_MAX, (ulong)UINT_MAX / sizeof(float)); const ulong input_widths[] = {109, (ulong)m_frame_dimension + 1, (ulong)m_frame_dimension + 1, w, 6, 16, w, w, w, w, w, 2 * w, 2 * w, w, w, w, w, w, 2 * w, w, w }; const ulong output[] = {w, w, w, w, w, w, w, w, w, w, 2 * w, w, w, w, w, w, w, 2 * w, w, 2 * w, 6 }; const ulong affine_rows[] = {1, market_tokens, market_tokens, 1, lh, 1, lh, lh, lh, lh, lh, lh, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, low_rows, lh }; const ulong tapes[] = {lh * 6, 16, lh * w, lh * w, lh * w, lh * w, 2 * lh * w, lh * w, lh * m_frame_heads, lh * w, lh * w, 2 * lh * w, lh * w, 2 * low_rows * w, low_rows * w, 2 * low_rows * w, low_rows * w, low_rows * m_frame_heads, low_rows * w, low_rows * w, 2 * low_rows * w, low_rows * w, lh * w, lh * w, lh * w, lh * 6 }; const ulong market_scalars = market_tokens * ((ulong)m_frame_dimension + 1); const ulong scenario_width = (ulong)m_frame_k * w; const ulong common[] = {109, market_tokens * ((ulong)m_frame_dimension + 1), m_frame_k, 218, m_frame_condition_count, scenario_width, 2 * market_tokens * w, scenario_width, (ulong)m_frame_k * m_frame_heads, w, 1, 1, 1, h * w, w, 109, market_scalars, market_scalars, market_scalars, m_frame_k, scenario_width, 2 * market_tokens * w, scenario_width, w }; const ulong work_size = MathMax(lh, low_rows) * w; //--- Widen every allocation and affine parameter product before narrowing. m_frame_capacity = 1; for(uint i = 0; i < 21; i++) { ulong parameters = 0; ulong values = 0; if(!DPCCRolloutProduct(input_widths[i] + 1, output[i], parameters) || parameters > bound || !DPCCRolloutProduct(affine_rows[i], output[i], values) || values > bound || input_widths[i] > bound || output[i] > bound || affine_rows[i] > bound) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_input[i] = (uint)input_widths[i]; m_frame_output[i] = (uint)output[i]; m_frame_affine_rows[i] = (uint)affine_rows[i]; m_frame_capacity = MathMax(m_frame_capacity, (uint)MathMax(parameters, values)); if(i >= 4) m_frame_tape_size[26 + i - 4] = (uint)values; } for(uint i = 0; i < 26; i++) { if(tapes[i] == 0 || tapes[i] > bound) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_tape_size[i] = (uint)tapes[i]; m_frame_capacity = MathMax(m_frame_capacity, (uint)tapes[i]); } for(uint i = 0; i < 24; i++) { if(common[i] == 0 || common[i] > bound) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_common_size[i] = (uint)common[i]; m_frame_capacity = MathMax(m_frame_capacity, (uint)common[i]); } if(work_size == 0 || work_size > bound || work_size > INT_MAX) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_work_size = (uint)work_size; m_frame_capacity = MathMax(m_frame_capacity, m_frame_work_size); m_frame_common_size[10] = m_frame_capacity; m_frame_common_size[12] = m_frame_capacity; //--- Create weights once; each slot owns activation storage only. if(!CNeuronBaseOCL::Init(0, 0, OpenCL, (uint)(lh * 6), ADAM, 1)) return(FrameFail(DPCC_OPENCL_ERROR, true)); activation = None; bTrain = false; for(uint i = 0; i < 21; i++) { m_frame_affine[i] = new CNeuronDPCCFrameAffineOCL; if(!m_frame_affine[i]) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_affine[i].Init(0, i, OpenCL, m_frame_input[i], m_frame_input[i], m_frame_output[i], m_frame_affine_rows[i], 1, ADAM, 1)) return(FrameFail(DPCC_OPENCL_ERROR, true)); m_frame_affine[i].SetActivationFunction(None); m_frame_affine[i].TrainMode(false); } for(uint i = 0; i < 24; i++) { m_frame_common[i] = new CBufferFloat; if(!m_frame_common[i]) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_common[i].BufferInit(m_frame_common_size[i], 0)) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_common[i].BufferCreate(OpenCL)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } for(uint i = 0; i < 5; i++) { m_frame_work[i] = new CBufferFloat; if(!m_frame_work[i] || !m_frame_work[i].BufferInit(m_frame_work_size, 0) || !m_frame_work[i].BufferCreate(OpenCL)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } //--- One reusable reverse tape preserves all J immutable forward tapes. for(uint i = 0; i < 43; i++) { m_frame_backward[i] = new CBufferFloat; if(!m_frame_backward[i] || !m_frame_backward[i].BufferInit(m_frame_tape_size[i], 0) || !m_frame_backward[i].BufferCreate(OpenCL)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } //--- Per-affine dW persists across the reverse traversal of every frame. for(uint i = 0; i < 21; i++) { const ulong parameters = (ulong)m_frame_output[i] * (m_frame_input[i] + 1); if(parameters == 0 || parameters > bound) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_accumulated[i] = new CBufferFloat; if(!m_frame_accumulated[i] || !m_frame_accumulated[i].BufferInit((uint)parameters, 0) || !m_frame_accumulated[i].BufferCreate(OpenCL)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } const uint count = m_frame_max_frames * 43; if(ArrayResize(m_frame_tapes, (int)count) != (int)count) return(FrameFail(DPCC_OPENCL_ERROR, true)); for(uint i = 0; i < count; i++) m_frame_tapes[i] = NULL; if(ArrayResize(m_frame_slots, (int)m_frame_max_frames) != (int)m_frame_max_frames) return(FrameFail(DPCC_OPENCL_ERROR, true)); ArrayInitialize(m_frame_slots, 0); for(uint i = 0; i < count; i++) { m_frame_tapes[i] = new CBufferFloat; if(!m_frame_tapes[i]) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_tapes[i].BufferInit(m_frame_tape_size[i % 43], 0)) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_tapes[i].BufferCreate(OpenCL)) return(FrameFail(DPCC_OPENCL_ERROR, true)); } //--- Fixed normalization and PE are immutable owners of this contract. for(uint i = 0; i < 218; i++) if(!m_frame_common[3].Update((int)i, m_frame_profile[i])) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_common[3].BufferWrite()) return(FrameFail(DPCC_OPENCL_ERROR, true)); for(uint token = 0; token < m_frame_horizon; token++) for(uint i = 0; i < m_frame_width / 2; i++) { const double angle = token * MathPow(10000.0, -(double)(2 * i) / m_frame_width); if(!m_frame_common[13].Update((int)(token * m_frame_width + 2 * i), (float)MathSin(angle))) return(FrameFail(DPCC_OPENCL_ERROR, true)); if(!m_frame_common[13].Update((int)(token * m_frame_width + 2 * i + 1), (float)MathCos(angle))) return(FrameFail(DPCC_OPENCL_ERROR, true)); } if(!m_frame_common[13].BufferWrite()) return(FrameFail(DPCC_OPENCL_ERROR, true)); return(true); } //+------------------------------------------------------------------+ //| Checks the complete owned Action6 graph without host tensor | //| reads | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameOwned(void) const { if(!m_frame_mode || !m_frame_bound || !OpenCL || m_failed || bTrain || activation != None || ArraySize(m_frame_tapes) != (int)(43 * m_frame_max_frames) || ArraySize(m_frame_slots) != (int)m_frame_max_frames || !ValidBuffer(Output, m_frame_rows * m_frame_horizon * 6) || !ValidBuffer(PrevOutput, m_frame_rows * m_frame_horizon * 6) || !ValidBuffer(Gradient, m_frame_rows * m_frame_horizon * 6)) return(false); for(uint i = 0; i < 24; i++) if(!ValidBuffer(m_frame_common[i], m_frame_common_size[i])) return(false); for(uint i = 0; i < (uint)ArraySize(m_frame_tapes); i++) if(!ValidBuffer(m_frame_tapes[i], m_frame_tape_size[i % 43])) return(false); for(uint i = 0; i < 43; i++) if(!ValidBuffer(m_frame_backward[i], m_frame_tape_size[i])) return(false); for(uint i = 0; i < 5; i++) if(!ValidBuffer(m_frame_work[i], m_frame_work_size)) return(false); for(uint i = 0; i < 21; i++) { const uint parameters = m_frame_output[i] * (m_frame_input[i] + 1); if(!ValidBuffer(m_frame_accumulated[i], parameters)) return(false); CNeuronConvOCL *child = m_frame_affine[i]; if(!child || child.GetOpenCL() != OpenCL || child.TrainMode() || child.Activation() != None || child.GetWindow() != m_frame_input[i] || child.GetInputStride() != m_frame_input[i] || child.GetFilters() != m_frame_output[i] || child.GetUnits() != m_frame_affine_rows[i] || child.GetVariables() != 1 || child.NativeOptimizerStep() != m_frame_update_count + 1 || !ValidBuffer(child.getOutput(), m_frame_affine_rows[i] * m_frame_output[i]) || !ValidBuffer(child.GetWeightsConvReadOnly(), m_frame_output[i] * (m_frame_input[i] + 1))) return(false); } return(true); } //+------------------------------------------------------------------+ //| Returns a bounded slot owner | //+------------------------------------------------------------------+ CBufferFloat *CNeuronPlanDenoiserOCL::FrameTape(const uint slot, const uint item) const { if(slot >= m_frame_max_frames || item >= 43 || ArraySize(m_frame_tapes) != (int)(43 * m_frame_max_frames)) return(NULL); return(m_frame_tapes[slot * 43 + item]); } //+------------------------------------------------------------------+ //| Begins an inference or complete-training Action6 unroll | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::BeginUnroll(ulong decision_id, uint frames, uint rows, bool need_backward) { if(m_frame_live) { m_status = DPCC_CONTRACT_MISMATCH; return(false); } if(!m_frame_mode || !m_frame_bound || m_failed || !FrameOwned() || decision_id == 0 || frames != m_frame_max_frames || rows == 0 || rows > m_frame_rows || m_frame_market_id == 0 || m_frame_market_weights == 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); m_frame_last_kernel_valid = false; m_frame_last_kernel = 0; m_frame_last_kernel_submit_us = 0; m_frame_last_kernel_name = ""; ArrayInitialize(m_frame_slots, 0); m_frame_decision = decision_id; m_frame_count = frames; m_frame_active_rows = rows; m_frame_next = 0; m_frame_current = 0; m_frame_backward_next = 0; m_frame_backward_count = 0; m_frame_training = need_backward; m_frame_gradients_ready = false; m_frame_updated = false; m_frame_update_started = false; m_frame_failed = false; m_frame_cache_valid = false; m_frame_live = true; m_public_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Releases a completed Action6 frame transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ReleaseUnroll(void) { if(!m_frame_mode || !m_frame_live) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_failed || m_frame_failed || !FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); if(m_frame_next != m_frame_count || m_frame_gradients_ready) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(m_frame_training) { if(!m_frame_updated || !m_frame_update_started || m_frame_backward_count != m_frame_count || m_frame_backward_next != 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint slot = 0; slot < m_frame_count; slot++) if(m_frame_slots[slot] != 2) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } else { if(m_frame_updated || m_frame_update_started || m_frame_backward_count != 0 || m_frame_backward_next != 0) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint slot = 0; slot < m_frame_count; slot++) if(m_frame_slots[slot] != 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } ArrayInitialize(m_frame_slots, 0); m_frame_live = false; m_frame_failed = false; m_frame_training = false; m_frame_gradients_ready = false; m_frame_updated = false; m_frame_update_started = false; m_frame_backward_next = 0; m_frame_backward_count = 0; m_frame_cache_valid = false; m_frame_cache_context = NULL; m_frame_cache_decision = m_frame_cache_market = m_frame_cache_weights = 0; m_frame_decision = 0; m_frame_count = m_frame_active_rows = m_frame_next = m_frame_current = 0; m_public_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Aborts a frame transaction before any shared parameter is | //| written | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AbortUnroll(void) { if(!m_frame_mode || !m_frame_live) return(Fail(DPCC_CONTRACT_MISMATCH)); if(m_frame_update_started || m_frame_updated) return(Fail(DPCC_CONTRACT_MISMATCH)); if(m_failed || !FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); ArrayInitialize(m_frame_slots, 0); m_frame_live = false; m_frame_failed = false; m_frame_training = false; m_frame_gradients_ready = false; m_frame_updated = false; m_frame_update_started = false; m_frame_backward_next = 0; m_frame_backward_count = 0; m_frame_cache_valid = false; m_frame_cache_context = NULL; m_frame_cache_decision = m_frame_cache_market = m_frame_cache_weights = 0; m_frame_decision = 0; m_frame_count = m_frame_active_rows = m_frame_next = m_frame_current = 0; m_public_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Reports one compact status for Action6 device operations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameExecute(const uint kernel, const uint &offset[], const uint &global[]) { const int work_dim = ArraySize(global); string global_text = ""; for(int i = 0; i < work_dim; i++) { if(i > 0) global_text += ","; global_text += (string)global[i]; } m_frame_last_kernel = kernel; m_frame_last_kernel_name = OpenCL.GetKernelName(kernel); m_frame_last_kernel_submit_us = GetMicrosecondCount(); m_frame_last_kernel_valid = true; ResetLastError(); if(!OpenCL.Execute(kernel, work_dim, offset, global)) { const int error_code = GetLastError(); string cl_error = ""; CLGetInfoString(OpenCL.GetContext(), CL_ERROR_DESCRIPTION, cl_error); PrintFormat("DPCC_OPENCL_DISPATCH_FAIL owner=PlanDenoiser kernel=%s id=%u work_dim=%d global={%s} error=%d cl=%s", m_frame_last_kernel_name, kernel, work_dim, global_text, error_code, cl_error); return(FrameFail(DPCC_OPENCL_ERROR)); } return(true); } //+------------------------------------------------------------------+ //| Reduces one frame status and reports failed queue | //| synchronization | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameReduce(const uint count, uint &status) { if(count == 0 || count > m_frame_capacity) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCOperatorReduce; setBuffer(kernel, 0, m_frame_common[10].GetIndex()); const bool previous_valid = m_frame_last_kernel_valid; const uint previous_kernel = m_frame_last_kernel; const string previous_name = m_frame_last_kernel_name; const ulong previous_submit_us = m_frame_last_kernel_submit_us; setBuffer(kernel, 1, m_frame_common[11].GetIndex()); setArgument(kernel, 2, count); uint offset[1] = {0}; uint size[1] = {1}; if(!FrameExecute(kernel, offset, size)) return(false); const ulong sync_started_us = GetMicrosecondCount(); ResetLastError(); if(!m_frame_common[11].BufferRead()) { const int error_code = GetLastError(); const ulong wait_us = GetMicrosecondCount() - sync_started_us; const ulong queued_us = (previous_valid && sync_started_us >= previous_submit_us ? sync_started_us - previous_submit_us : 0); string cl_error = ""; CLGetInfoString(OpenCL.GetContext(), CL_ERROR_DESCRIPTION, cl_error); PrintFormat("DPCC_OPENCL_SYNC_FAIL owner=PlanDenoiser stage=frame_reduce sync=%s(%u) " "previous=%s(%u) count=%u queued_us=%I64u wait_us=%I64u error=%d cl=%s", OpenCL.GetKernelName(kernel), kernel, (previous_valid ? previous_name : "none"), (previous_valid ? previous_kernel : 0), count, queued_us, wait_us, error_code, cl_error); return(FrameFail(DPCC_OPENCL_ERROR)); } m_frame_last_kernel_valid = false; m_frame_last_kernel_name = ""; m_frame_last_kernel_submit_us = 0; const float value = m_frame_common[11][0]; if(!MathIsValidNumber(value) || value < 0 || value > 16777216 || MathFloor(value) != value) return(FrameFail(DPCC_NUMERIC_ERROR)); status = (uint)value; return(true); } //+------------------------------------------------------------------+ //| Admits a finite Action6 result before the next operation | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameReduce(const uint count) { uint status = 0; if(!FrameReduce(count, status)) return(false); if(status != 0) return(FrameFail((ENUM_DPCC_STATUS)status)); return(true); } //+------------------------------------------------------------------+ //| Applies the existing exact activation primitive | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameActivate(CBufferFloat *source, CBufferFloat *destination, const int function) { if(!source || !destination || source.Total() <= 0 || source.Total() > destination.Total() || source.Total() > (int)m_frame_capacity || !ValidBuffer(source, (uint)source.Total()) || !ValidBuffer(destination, (uint)destination.Total())) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCActivationForward; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_frame_common[10].GetIndex()); setArgument(kernel, 3, function); setArgument(kernel, 4, (uint)source.Total()); uint offset[1] = {0}; uint size[1] = {(uint)source.Total()}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(size[0])); } //+------------------------------------------------------------------+ //| Validates a device tensor through a private scratch owner | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameCheck(CBufferFloat *source) { return(FrameActivate(source, m_frame_common[12], None)); } //+------------------------------------------------------------------+ //| Copies an admitted prefix while retaining stable row indices | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameCopy(CBufferFloat *source, CBufferFloat *destination, const uint count) { if(!source || !destination || count == 0 || count > (uint)INT_MAX || source.Total() < (int)count || destination.Total() < (int)count || !ValidBuffer(source, (uint)source.Total()) || !ValidBuffer(destination, (uint)destination.Total())) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!FrameCheck(source)) return(false); const uint kernel = def_k_CopyBufferRaw; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setArgument(kernel, 2, (int)count); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(true); } //+------------------------------------------------------------------+ //| Adds two exact residual branches | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSum(CBufferFloat *first, CBufferFloat *second, CBufferFloat *destination) { if(!first || !second || !destination || first.Total() != second.Total() || first.Total() != destination.Total() || !ValidBuffer(first, (uint)first.Total()) || !ValidBuffer(second, (uint)second.Total()) || !ValidBuffer(destination, (uint)destination.Total())) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!SumAndNormalize(first, second, destination, 1, false, 0, 0, 0, 1.0f, m_frame_common[10])) return(FrameFail(DPCC_OPENCL_ERROR)); return(FrameReduce((uint)destination.Total())); } //+------------------------------------------------------------------+ //| Broadcasts a shared condition, step or positional vector | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameBroadcast(CBufferFloat *vector_buffer, CBufferFloat *matrix_buffer, CBufferFloat *destination, const uint width, const uint rows) { if(!ValidBuffer(vector_buffer, width) || !ValidBuffer(matrix_buffer, width * rows) || !ValidBuffer(destination, width * rows)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!SumVecMatrix(vector_buffer, matrix_buffer, destination, width, 1, 0, 0, 0, 1.0f, rows, m_frame_common[10])) return(FrameFail(DPCC_OPENCL_ERROR)); return(FrameReduce(width * rows)); } //+------------------------------------------------------------------+ //| Runs one shared affine and saves its original frame | //| preactivation | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAffine(const uint block, CBufferFloat *source) { if(block >= 21 || !ValidBuffer(source, m_frame_affine_rows[block] * m_frame_input[block])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CNeuronConvOCL *child = m_frame_affine[block]; const uint kernel = def_k_DPCCSharedAffineForwardPreflight; setBuffer(kernel, 0, child.GetWeightsConvReadOnly().GetIndex()); setBuffer(kernel, 1, source.GetIndex()); setBuffer(kernel, 2, m_frame_common[10].GetIndex()); setArgument(kernel, 3, m_frame_affine_rows[block]); setArgument(kernel, 4, m_frame_input[block]); setArgument(kernel, 5, m_frame_output[block]); uint offset[2] = {0, 0}; uint size[2] = {m_frame_output[block], m_frame_affine_rows[block]}; if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(size[0] * size[1])) return(false); if(!child.FeedForwardBuffer(source)) return(FrameFail(DPCC_OPENCL_ERROR)); if(!FrameCheck(child.getOutput())) return(false); if(block >= 4) return(FrameCopy(child.getOutput(), FrameTape(m_frame_current, 26 + block - 4), m_frame_affine_rows[block] * m_frame_output[block])); return(true); } //+------------------------------------------------------------------+ //| Uses native unmasked attention with an independent group axis | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAttention(CBufferFloat *query, CBufferFloat *key_value, CBufferFloat *lse, CBufferFloat *output, const uint queries, const uint keys, const uint groups) { const uint dh = m_frame_width / m_frame_heads; if(!ValidBuffer(query, queries * groups * dh) || !ValidBuffer(key_value, keys * 2 * groups * dh) || !ValidBuffer(lse, queries * groups) || !ValidBuffer(output, queries * groups * dh)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!FrameCheck(query) || !FrameCheck(key_value)) return(false); const uint kernel = def_k_MHFlashAttention; setBuffer(kernel, 0, query.GetIndex()); setBuffer(kernel, 1, key_value.GetIndex()); setBuffer(kernel, 2, lse.GetIndex()); setBuffer(kernel, 3, output.GetIndex()); setArgument(kernel, 4, (int)dh); setArgument(kernel, 5, (int)keys); setArgument(kernel, 6, (int)0); uint offset[3] = {0, 0, 0}; uint size[3] = {queries, dh, groups}; uint local[3] = {1, dh, 1}; kernelExecuteLoc(kernel, offset, size, local) if(!FrameCheck(lse)) return(false); return(FrameCheck(output)); } //+------------------------------------------------------------------+ //| Preserves each candidate's self-attention tensors and LSE | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSelf(const uint block, const uint source_item, const uint first, const uint length, const uint packed, const uint residual) { if(block > 1 || first + 3 >= 21 || packed + 4 >= 43) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *source = FrameTape(m_frame_current, source_item); for(uint i = 0; i < 3; i++) if(!FrameAffine(first + i, source)) return(false); const uint kernel = def_k_DPCCDenoiserSelfLayout; CBufferFloat *q = FrameTape(m_frame_current, packed); CBufferFloat *kv = FrameTape(m_frame_current, packed + 1); CBufferFloat *attended = FrameTape(m_frame_current, packed + 2); CBufferFloat *lse = FrameTape(m_frame_current, packed + 3); CBufferFloat *unpacked = FrameTape(m_frame_current, packed + 4); setBuffer(kernel, 0, m_frame_affine[first].getOutputIndex()); setBuffer(kernel, 1, m_frame_affine[first + 1].getOutputIndex()); setBuffer(kernel, 2, m_frame_affine[first + 2].getOutputIndex()); setBuffer(kernel, 3, q.GetIndex()); setBuffer(kernel, 4, kv.GetIndex()); setBuffer(kernel, 5, m_frame_common[10].GetIndex()); setArgument(kernel, 6, m_frame_rows); setArgument(kernel, 7, length); setArgument(kernel, 8, m_frame_heads); setArgument(kernel, 9, m_frame_width / m_frame_heads); setArgument(kernel, 10, (uint)0); uint offset[1] = {0}; uint size[1] = {m_frame_rows * length * m_frame_width}; if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); if(!FrameAttention(q, kv, lse, attended, length, length, m_frame_rows * m_frame_heads)) return(false); setBuffer(kernel, 0, attended.GetIndex()); setBuffer(kernel, 3, unpacked.GetIndex()); setArgument(kernel, 10, (uint)1); if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); if(!FrameAffine(first + 3, unpacked)) return(false); return(FrameSum(source, m_frame_affine[first + 3].getOutput(), FrameTape(m_frame_current, residual))); } //+------------------------------------------------------------------+ //| Applies the exact Action6 self-attention tensor permutations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSelfLayout(const uint mode, const uint length, CBufferFloat *source0, CBufferFloat *source1, CBufferFloat *source2, CBufferFloat *destination0, CBufferFloat *destination1) { const uint rows = m_frame_rows * length; const uint count = rows * m_frame_width; const uint source_count = mode == 2 ? 2 * count : count; if(mode > 3 || length == 0 || !ValidFramePrefix(source0, source_count) || !ValidBuffer(destination0, count) || (mode == 0 && (!ValidBuffer(source1, count) || !ValidBuffer(source2, count) || !ValidBuffer(destination1, 2 * count))) || (mode == 2 && !ValidBuffer(destination1, count))) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *bound1 = source1; CBufferFloat *bound2 = source2; CBufferFloat *bound_destination1 = destination1; if(!bound1) bound1 = source0; if(!bound2) bound2 = source0; if(!bound_destination1) bound_destination1 = destination0; const uint kernel = def_k_DPCCDenoiserSelfLayout; setBuffer(kernel, 0, source0.GetIndex()); setBuffer(kernel, 1, bound1.GetIndex()); setBuffer(kernel, 2, bound2.GetIndex()); setBuffer(kernel, 3, destination0.GetIndex()); setBuffer(kernel, 4, bound_destination1.GetIndex()); setBuffer(kernel, 5, m_frame_common[10].GetIndex()); setArgument(kernel, 6, m_frame_rows); setArgument(kernel, 7, length); setArgument(kernel, 8, m_frame_heads); setArgument(kernel, 9, m_frame_width / m_frame_heads); setArgument(kernel, 10, mode); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(size[0])); } //+------------------------------------------------------------------+ //| Reverses one Action6 self-attention residual with shared weights | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSelfBackward(const uint block, const uint source_item, const uint first, const uint length, const uint packed, const uint residual, CBufferFloat *upstream, CBufferFloat *destination) { CBufferFloat *source = FrameTape(m_frame_current, source_item); CBufferFloat *residual_output = FrameTape(m_frame_current, residual); if(block > 1 || first + 3 >= 21 || packed + 4 >= 43 || !ValidBuffer(source, m_frame_rows * length * m_frame_width) || !ValidBuffer(residual_output, m_frame_rows * length * m_frame_width) || !ValidBuffer(upstream, m_frame_rows * length * m_frame_width) || !ValidBuffer(destination, m_frame_rows * length * m_frame_width)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); if(!FrameAffineBackward(first + 3, m_frame_current, upstream, m_frame_work[0])) return(false); if(!FrameSelfLayout(3, length, m_frame_work[0], NULL, NULL, m_frame_backward[packed + 2], NULL)) return(false); if(!FrameAttentionBackward(FrameTape(m_frame_current, packed), FrameTape(m_frame_current, packed + 1), FrameTape(m_frame_current, packed + 3), FrameTape(m_frame_current, packed + 2), m_frame_backward[packed + 2], m_frame_backward[packed], m_frame_backward[packed + 1], length, length, m_frame_rows * m_frame_heads)) return(false); if(!FrameSelfLayout(1, length, m_frame_backward[packed], NULL, NULL, m_frame_affine[first].getGradient(), NULL) || !FrameSelfLayout(2, length, m_frame_backward[packed + 1], NULL, NULL, m_frame_affine[first + 1].getGradient(), m_frame_affine[first + 2].getGradient())) return(false); if(!FrameAffineBackward(first, m_frame_current, m_frame_affine[first].getGradient(), m_frame_work[0]) || !FrameAffineBackward(first + 1, m_frame_current, m_frame_affine[first + 1].getGradient(), m_frame_work[1]) || !FrameAdd(m_frame_work[0], m_frame_work[1], m_frame_work[3], m_frame_rows * length * m_frame_width) || !FrameAffineBackward(first + 2, m_frame_current, m_frame_affine[first + 2].getGradient(), m_frame_work[2]) || !FrameAdd(m_frame_work[3], m_frame_work[2], m_frame_work[4], m_frame_rows * length * m_frame_width) || !FrameAdd(upstream, m_frame_work[4], destination, m_frame_rows * length * m_frame_width)) return(false); return(true); } //+------------------------------------------------------------------+ //| Pads odd horizons once and crops the inverse pair mapping | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FramePair(CBufferFloat *source, CBufferFloat *destination, const uint direction) { const uint plain = m_frame_rows * m_frame_horizon * m_frame_width; const uint padded = m_frame_rows * ((m_frame_horizon + 1) / 2) * 2 * m_frame_width; const bool padded_output = direction == def_k_DPCCDenoiserPairMap_forward_pad || direction == def_k_DPCCDenoiserPairMap_backward_crop; if(direction > def_k_DPCCDenoiserPairMap_backward_pad || !ValidBuffer(source, padded_output ? plain : padded) || !ValidFramePrefix(destination, padded_output ? padded : plain)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDenoiserPairMap; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_frame_common[10].GetIndex()); setArgument(kernel, 3, m_frame_rows); setArgument(kernel, 4, m_frame_horizon); setArgument(kernel, 5, m_frame_width); setArgument(kernel, 6, direction); uint offset[1] = {0}; uint size[1] = {padded_output ? padded : plain}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(size[0])); } //+------------------------------------------------------------------+ //| Checks the immutable condition before reusing its cached outputs | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSameCondition(CBufferFloat *condition) { if(!m_frame_cache_valid || m_frame_cache_context != OpenCL || m_frame_cache_decision != m_frame_decision || m_frame_cache_market != m_frame_market_id || m_frame_cache_weights != m_frame_market_weights || !ValidBuffer(condition, m_frame_condition_count)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDecoderSnapshotCheck; setBuffer(kernel, 0, condition.GetIndex()); setBuffer(kernel, 1, m_frame_common[4].GetIndex()); setBuffer(kernel, 2, m_frame_common[10].GetIndex()); setArgument(kernel, 3, (uint)1); setArgument(kernel, 4, (uint)1); setArgument(kernel, 5, (uint)1); setArgument(kernel, 6, m_frame_condition_count); uint offset[1] = {0}; uint size[1] = {m_frame_condition_count}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(size[0])); } //+------------------------------------------------------------------+ //| Builds shared per-scenario conditioning once for a decision | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameCondition(CBufferFloat *condition) { if(!ValidBuffer(condition, m_frame_condition_count)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint tokens = m_frame_v * m_frame_market_horizon; const uint kernel = def_k_DPCCDenoiserFrameCondition; setBuffer(kernel, 0, condition.GetIndex()); setBuffer(kernel, 1, m_frame_common[3].GetIndex()); setBuffer(kernel, 2, m_frame_common[0].GetIndex()); setBuffer(kernel, 3, m_frame_common[1].GetIndex()); setBuffer(kernel, 4, m_frame_common[2].GetIndex()); setBuffer(kernel, 5, m_frame_common[10].GetIndex()); setArgument(kernel, 6, m_frame_k); setArgument(kernel, 7, tokens); setArgument(kernel, 8, m_frame_dimension); setArgument(kernel, 9, m_frame_probability_tolerance); setArgument(kernel, def_k_DPCCDenoiserFrameCondition_mode, (uint)def_k_DPCCDenoiserFrameCondition_forward); uint offset[1] = {0}; uint size[1] = {m_frame_condition_count}; if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); if(!FrameCopy(condition, m_frame_common[4], m_frame_condition_count)) return(false); if(!FrameAffine(0, m_frame_common[0])) return(false); if(!FrameAffine(1, m_frame_common[1])) return(false); if(!FrameAffine(2, m_frame_common[1])) return(false); //--- Attention groups contain exactly one k and one logical head. const uint layout = def_k_DPCCDenoiserFrameLayout; setBuffer(layout, 0, m_frame_affine[1].getOutputIndex()); setBuffer(layout, 1, m_frame_affine[2].getOutputIndex()); setBuffer(layout, 2, m_frame_affine[0].getOutputIndex()); setBuffer(layout, 3, m_frame_common[2].GetIndex()); setBuffer(layout, 4, m_frame_common[5].GetIndex()); setBuffer(layout, 5, m_frame_common[6].GetIndex()); setBuffer(layout, 6, m_frame_common[9].GetIndex()); setBuffer(layout, 7, m_frame_common[10].GetIndex()); setArgument(layout, 8, m_frame_k); setArgument(layout, 9, tokens); setArgument(layout, 10, m_frame_width); setArgument(layout, 11, m_frame_heads); setArgument(layout, 12, (uint)0); size[0] = m_frame_k * tokens * m_frame_width; if(!FrameExecute(layout, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); if(!FrameAttention(m_frame_common[5], m_frame_common[6], m_frame_common[8], m_frame_common[7], 1, tokens, m_frame_k * m_frame_heads)) return(false); setBuffer(layout, 2, m_frame_common[7].GetIndex()); setArgument(layout, 12, (uint)1); size[0] = m_frame_width; if(!FrameExecute(layout, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); if(!FrameAffine(3, m_frame_common[9])) return(false); if(!FrameCopy(m_frame_affine[3].getOutput(), m_frame_common[14], m_frame_width)) return(false); m_frame_cache_decision = m_frame_decision; m_frame_cache_market = m_frame_market_id; m_frame_cache_weights = m_frame_market_weights; m_frame_cache_context = OpenCL; m_frame_cache_valid = true; return(true); } //+------------------------------------------------------------------+ //| Resolves the exact saved input of each shared Action6 affine | //+------------------------------------------------------------------+ CBufferFloat *CNeuronPlanDenoiserOCL::FrameAffineInput(const uint block, const uint slot) const { if(block >= 21 || slot >= m_frame_max_frames) return(NULL); if(block == 0) return(m_frame_common[0]); if(block == 1 || block == 2) return(m_frame_common[1]); if(block == 3) return(m_frame_common[9]); uint item = 0; if(block == 4) item = 0; else if(block == 5) item = 1; else if(block >= 6 && block <= 8) item = 4; else if(block == 9) item = 9; else if(block == 10) item = 10; else if(block == 11) item = 11; else if(block == 12) item = 13; else if(block >= 13 && block <= 15) item = 34; else if(block == 16) item = 18; else if(block == 17) item = 19; else if(block == 18) item = 20; else if(block == 19) item = 21; else if(block == 20) item = 24; return(FrameTape(slot, item)); } //+------------------------------------------------------------------+ //| Adds exact, unclipped shared-affine parameter gradients by frame | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAccumulateAffine(const uint block, CBufferFloat *upstream, CBufferFloat *source) { if(block >= 21 || !ValidBuffer(upstream, m_frame_affine_rows[block] * m_frame_output[block]) || !ValidBuffer(source, m_frame_affine_rows[block] * m_frame_input[block]) || !ValidBuffer(m_frame_accumulated[block], m_frame_output[block] * (m_frame_input[block] + 1)) || m_frame_affine_rows[block] == 0 || m_frame_affine_rows[block] > def_k_DPCCSharedAffineAdamPreflight_frame_count_mask) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCSharedAffineAdamPreflight; CNeuronConvOCL *child = m_frame_affine[block]; const uint rows = m_frame_affine_rows[block]; const uint mode = def_k_DPCCSharedAffineAdamPreflight_accumulate; const uint tokens = (uint)def_k_DPCCSharedAffineAdamPreflight_frame_tag | (mode << def_k_DPCCSharedAffineAdamPreflight_frame_mode_shift) | rows; if((tokens & (uint)def_k_DPCCSharedAffineAdamPreflight_frame_tag) == 0 || (tokens & (uint)def_k_DPCCSharedAffineAdamPreflight_frame_count_mask) != rows || ((tokens >> def_k_DPCCSharedAffineAdamPreflight_frame_mode_shift) & 0x3) != mode) return(FrameFail(DPCC_CONTRACT_MISMATCH)); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_weights, child.GetWeightsConvReadOnly().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_gradient, upstream.GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_input, source.GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_momentum1, m_frame_accumulated[block].GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_momentum2, child.GetSecondMomentumConv().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_flags, m_frame_common[10].GetIndex()); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_tokens, tokens); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_input_width, m_frame_input[block]); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_output_width, m_frame_output[block]); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_learning_rate, 1.0f); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_beta1, b1); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_beta2, b2); uint offset[1] = {0}; const uint count = m_frame_output[block] * (m_frame_input[block] + 1); uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(count)); } //+------------------------------------------------------------------+ //| Computes one exact shared-affine input VJP using pre-update | //| weights | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAffineBackward(const uint block, const uint slot, CBufferFloat *upstream, CBufferFloat *input_gradient) { if(block >= 21 || !m_frame_training || !m_frame_gradients_ready || slot >= m_frame_count) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *source = FrameAffineInput(block, slot); const uint rows = m_frame_affine_rows[block]; const uint in_width = m_frame_input[block]; const uint out_width = m_frame_output[block]; if(!ValidBuffer(source, rows * in_width) || !ValidBuffer(upstream, rows * out_width) || !ValidFramePrefix(input_gradient, rows * in_width) || BufferAliases(upstream, input_gradient) || BufferAliases(source, input_gradient) || !FrameCheck(source) || !FrameCheck(upstream)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCSharedAffineInputGradient; CNeuronConvOCL *child = m_frame_affine[block]; setBuffer(kernel, def_k_DPCCSharedAffineInputGradient_weights, child.GetWeightsConvReadOnly().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineInputGradient_upstream, upstream.GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineInputGradient_input_gradient, input_gradient.GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineInputGradient_flags, m_frame_common[10].GetIndex()); setArgument(kernel, def_k_DPCCSharedAffineInputGradient_tokens, rows); setArgument(kernel, def_k_DPCCSharedAffineInputGradient_input_width, in_width); setArgument(kernel, def_k_DPCCSharedAffineInputGradient_output_width, out_width); uint offset[2] = {0, 0}; uint size[2] = {in_width, rows}; if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(rows * in_width)) return(false); if(!FrameAccumulateAffine(block, upstream, source)) return(false); return(FrameCheck(input_gradient)); } //+------------------------------------------------------------------+ //| Applies the exact activation derivative to a frame-local tensor | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameActivationBackward(CBufferFloat *preactivation, CBufferFloat *upstream, CBufferFloat *input_gradient, const uint count, const int function) { if(!ValidBuffer(preactivation, count) || !ValidBuffer(upstream, count) || !ValidBuffer(input_gradient, count) || BufferAliases(upstream, input_gradient) || BufferAliases(preactivation, input_gradient) || !FrameCheck(preactivation) || !FrameCheck(upstream)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCActivationGradient; setBuffer(kernel, def_k_DPCCActivationGradient_preactivation, preactivation.GetIndex()); setBuffer(kernel, def_k_DPCCActivationGradient_upstream, upstream.GetIndex()); setBuffer(kernel, def_k_DPCCActivationGradient_input_gradient, input_gradient.GetIndex()); setBuffer(kernel, def_k_DPCCActivationGradient_flags, m_frame_common[10].GetIndex()); setArgument(kernel, def_k_DPCCActivationGradient_activation, function); setArgument(kernel, def_k_DPCCActivationGradient_count, count); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(count)); } //+------------------------------------------------------------------+ //| Adds two gradient branches with finite device-side status | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAdd(CBufferFloat *first, CBufferFloat *second, CBufferFloat *destination, const uint count) { if(!ValidFramePrefix(first, count) || !ValidFramePrefix(second, count) || !ValidFramePrefix(destination, count) || BufferAliases(first, destination) || BufferAliases(second, destination) || !FrameCheck(first) || !FrameCheck(second)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const int kernel = def_k_MatrixSum; setBuffer(kernel, def_k_sum_matrix1, first.GetIndex()); setBuffer(kernel, def_k_sum_matrix2, second.GetIndex()); setBuffer(kernel, def_k_sum_matrix_out, destination.GetIndex()); setArgument(kernel, def_k_sum_dimension, (int)1); setArgument(kernel, def_k_sum_shift_in1, (int)0); setArgument(kernel, def_k_sum_shift_in2, (int)0); setArgument(kernel, def_k_sum_shift_out, (int)0); setArgument(kernel, def_k_sum_multiplyer, 1.0f); setBuffer(kernel, def_k_sum_numeric_flags, m_frame_common[10].GetIndex()); setArgument(kernel, def_k_sum_flags_enabled, (int)1); uint offset[1] = {0}; uint size[1] = {count}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(count)); } //+------------------------------------------------------------------+ //| Reduces one frame-local broadcast derivative with owned scratch | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameBroadcastGradient(CBufferFloat *source, CBufferFloat *destination, const uint rows, const uint width) { if(rows == 0 || width == 0 || !ValidBuffer(source, rows * width) || !ValidBuffer(destination, width) || BufferAliases(source, destination) || !FrameCheck(source)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint kernel = def_k_DPCCDenoiserBroadcastGradient; setBuffer(kernel, 0, source.GetIndex()); setBuffer(kernel, 1, destination.GetIndex()); setBuffer(kernel, 2, m_frame_common[10].GetIndex()); setArgument(kernel, 3, rows); setArgument(kernel, 4, width); uint offset[1] = {0}; uint size[1] = {width}; if(!FrameExecute(kernel, offset, size)) return(false); return(FrameReduce(width)); } //+------------------------------------------------------------------+ //| Validates and runs exact unmasked Action6 attention VJP | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameAttentionBackward(CBufferFloat *query, CBufferFloat *key_value, CBufferFloat *lse, CBufferFloat *output, CBufferFloat *output_gradient, CBufferFloat *query_gradient, CBufferFloat *key_value_gradient, const uint queries, const uint keys, const uint groups) { const uint dh = m_frame_width / m_frame_heads; if(queries == 0 || keys == 0 || groups == 0 || !ValidBuffer(query, queries * groups * dh) || !ValidBuffer(key_value, keys * 2 * groups * dh) || !ValidBuffer(lse, queries * groups) || !ValidBuffer(output, queries * groups * dh) || !ValidBuffer(output_gradient, queries * groups * dh) || !ValidBuffer(query_gradient, queries * groups * dh) || !ValidBuffer(key_value_gradient, keys * 2 * groups * dh) || BufferAliases(query_gradient, query) || BufferAliases(key_value_gradient, key_value) || !FrameCheck(query) || !FrameCheck(key_value) || !FrameCheck(lse) || !FrameCheck(output) || !FrameCheck(output_gradient)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const uint preflight = def_k_DPCCAttentionPreflight; setBuffer(preflight, def_k_DPCCAttentionPreflight_query, query.GetIndex()); setBuffer(preflight, def_k_DPCCAttentionPreflight_key_value, key_value.GetIndex()); setBuffer(preflight, def_k_DPCCAttentionPreflight_logsumexp, lse.GetIndex()); setBuffer(preflight, def_k_DPCCAttentionPreflight_output, output.GetIndex()); setBuffer(preflight, def_k_DPCCAttentionPreflight_output_gr, output_gradient.GetIndex()); setBuffer(preflight, def_k_DPCCAttentionPreflight_flags, m_frame_common[10].GetIndex()); setArgument(preflight, def_k_DPCCAttentionPreflight_dimension, (int)dh); setArgument(preflight, def_k_DPCCAttentionPreflight_total_q, (int)queries); setArgument(preflight, def_k_DPCCAttentionPreflight_total_kv, (int)keys); setArgument(preflight, def_k_DPCCAttentionPreflight_mask_future, (int)0); setArgument(preflight, def_k_DPCCAttentionPreflight_mode, (uint)1); uint offset[3] = {0, 0, 0}; const uint extent = MathMax(queries, keys); uint size[3] = {extent, dh, groups}; uint local[3] = {1, dh, 1}; kernelExecuteLoc(preflight, offset, size, local) if(!FrameReduce(extent * dh * groups)) return(false); const uint kernel = def_k_DPCC_MHFlashAttentionGradient; setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_query, query.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_query_gr, query_gradient.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_key_value, key_value.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_key_value_gr, key_value_gradient.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_logsumexp, lse.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_output, output.GetIndex()); setBuffer(kernel, def_k_DPCC_MHFlashAttentionGradient_output_gr, output_gradient.GetIndex()); setArgument(kernel, def_k_DPCC_MHFlashAttentionGradient_dimension, (int)dh); setArgument(kernel, def_k_DPCC_MHFlashAttentionGradient_total_q, (int)queries); setArgument(kernel, def_k_DPCC_MHFlashAttentionGradient_total_kv, (int)keys); setArgument(kernel, def_k_DPCC_MHFlashAttentionGradient_mask_future, (int)0); kernelExecuteLoc(kernel, offset, size, local) if(!FrameCheck(query_gradient)) return(false); return(FrameCheck(key_value_gradient)); } //+------------------------------------------------------------------+ //| Executes D0-D6 while retaining every nonlinear slot output | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameForward(uint slot, uint j, CBufferFloat *plan, CBufferFloat *condition, CBufferFloat *epsilon_out) { //--- Validate lifecycle, external ownership and exact active-prefix geometry. const uint active_count = m_frame_active_rows * m_frame_horizon * 6; if(!m_frame_mode || !m_frame_live || m_frame_update_started || m_frame_failed || m_failed || !FrameOwned() || slot >= m_frame_count || slot != m_frame_next || j != m_frame_max_frames - slot || m_frame_slots[slot] != 0 || !ValidBuffer(plan, active_count) || !ValidBuffer(condition, m_frame_condition_count) || !ValidBuffer(epsilon_out, active_count)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *external[] = {plan, condition, epsilon_out}; for(uint i = 0; i < 3; i++) { if(OwnsBufferOrDeviceSlot(external[i])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint k = 0; k < i; k++) if(BufferAliases(external[i], external[k])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } m_public_valid = false; m_frame_current = slot; m_status = DPCC_OK; if(m_frame_cache_valid) { if(!FrameSameCondition(condition)) return(false); } else if(!FrameCondition(condition)) return(false); CBufferFloat *saved_plan = FrameTape(slot, 0); const uint prepare = def_k_DPCCDenoiserFrameLayout; setBuffer(prepare, 0, plan.GetIndex()); setBuffer(prepare, 1, plan.GetIndex()); setBuffer(prepare, 2, plan.GetIndex()); setBuffer(prepare, 3, m_frame_common[2].GetIndex()); setBuffer(prepare, 4, m_frame_common[5].GetIndex()); setBuffer(prepare, 5, m_frame_common[6].GetIndex()); setBuffer(prepare, 6, saved_plan.GetIndex()); setBuffer(prepare, 7, m_frame_common[10].GetIndex()); setArgument(prepare, 8, active_count); setArgument(prepare, 9, m_frame_tape_size[0]); setArgument(prepare, 10, m_frame_width); setArgument(prepare, 11, m_frame_heads); setArgument(prepare, 12, (uint)2); uint offset[1] = {0}; uint size[1] = {m_frame_tape_size[0]}; if(!FrameExecute(prepare, offset, size)) return(false); if(!FrameReduce(size[0])) return(false); float encoding[]; string reason; if(!DPCCBuildStepPositionalEncoding(j, m_frame_max_frames, encoding, reason)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); CBufferFloat *step = FrameTape(slot, 1); for(uint i = 0; i < 16; i++) if(!step.Update((int)i, encoding[i])) return(FrameFail(DPCC_OPENCL_ERROR)); if(!step.BufferWrite()) return(FrameFail(DPCC_OPENCL_ERROR)); //--- D0 and two independent self-attention resolutions use the same weights for every slot. if(!FrameAffine(4, saved_plan)) return(false); if(!FrameAffine(5, step)) return(false); const uint rows = m_frame_rows * m_frame_horizon; if(!FrameBroadcast(m_frame_common[14], m_frame_affine[4].getOutput(), FrameTape(slot, 2), m_frame_width, rows)) return(false); if(!FrameBroadcast(m_frame_affine[5].getOutput(), FrameTape(slot, 2), FrameTape(slot, 3), m_frame_width, rows)) return(false); if(!FrameBroadcast(m_frame_common[13], FrameTape(slot, 3), FrameTape(slot, 4), m_frame_horizon * m_frame_width, m_frame_rows)) return(false); if(!FrameSelf(0, 4, 6, m_frame_horizon, 5, 10)) return(false); if(!FrameAffine(10, FrameTape(slot, 10))) return(false); if(!FrameActivate(m_frame_affine[10].getOutput(), FrameTape(slot, 11), TANH)) return(false); if(!FrameAffine(11, FrameTape(slot, 11))) return(false); if(!FrameSum(FrameTape(slot, 10), m_frame_affine[11].getOutput(), FrameTape(slot, 12))) return(false); if(!FramePair(FrameTape(slot, 12), FrameTape(slot, 13), 0)) return(false); if(!FrameAffine(12, FrameTape(slot, 13))) return(false); if(!FrameSelf(1, 34, 13, (m_frame_horizon + 1) / 2, 14, 19)) return(false); if(!FrameAffine(17, FrameTape(slot, 19))) return(false); if(!FrameActivate(m_frame_affine[17].getOutput(), FrameTape(slot, 20), TANH)) return(false); if(!FrameAffine(18, FrameTape(slot, 20))) return(false); if(!FrameSum(FrameTape(slot, 19), m_frame_affine[18].getOutput(), FrameTape(slot, 21))) return(false); if(!FrameAffine(19, FrameTape(slot, 21))) return(false); if(!FramePair(m_frame_affine[19].getOutput(), FrameTape(slot, 22), 1)) return(false); //--- D6 applies TANH after its saved skip sum and publishes epsilon only after admission. if(!FrameSum(FrameTape(slot, 22), FrameTape(slot, 12), FrameTape(slot, 23))) return(false); if(!FrameActivate(FrameTape(slot, 23), FrameTape(slot, 24), TANH)) return(false); if(!FrameAffine(20, FrameTape(slot, 24))) return(false); if(!FrameCopy(m_frame_affine[20].getOutput(), FrameTape(slot, 25), m_frame_tape_size[25])) return(false); if(!FrameCopy(FrameTape(slot, 25), Output, m_frame_tape_size[25])) return(false); if(!FrameCopy(FrameTape(slot, 25), epsilon_out, active_count)) return(false); m_frame_slots[slot] = 1; m_frame_next++; m_public_valid = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Reverses one complete Action6 frame before any shared weight | //| update | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameBackward(uint slot, CBufferFloat *epsilon_gradient, CBufferFloat *plan_gradient, CBufferFloat *condition_gradient) { const uint active_count = m_frame_active_rows * m_frame_horizon * 6; const uint full_rows = m_frame_rows * m_frame_horizon; const uint low_rows = m_frame_rows * ((m_frame_horizon + 1) / 2); const uint full_width = full_rows * m_frame_width; const uint low_width = low_rows * m_frame_width; const uint padded_width = 2 * low_width; const uint tokens = m_frame_v * m_frame_market_horizon; if(!m_frame_mode || !m_frame_live || !m_frame_training || !m_frame_gradients_ready || m_frame_failed || m_failed || !FrameOwned() || slot >= m_frame_count || slot + 1 != m_frame_backward_next || m_frame_slots[slot] != 1 || !ValidBuffer(epsilon_gradient, active_count) || !ValidBuffer(plan_gradient, active_count) || !ValidBuffer(condition_gradient, m_frame_condition_count)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=frame_preflight slot=%u status=%d", slot, (int)m_status); return(FrameFail(DPCC_CONTRACT_MISMATCH)); } CBufferFloat *external[] = {epsilon_gradient, plan_gradient, condition_gradient}; for(uint i = 0; i < 3; i++) { if(OwnsBufferOrDeviceSlot(external[i])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint k = 0; k < i; k++) if(BufferAliases(external[i], external[k])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } if(!FrameCheck(epsilon_gradient) || !FrameCheck(condition_gradient)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=input_check slot=%u status=%d", slot, (int)m_status); return(false); } m_frame_current = slot; m_public_valid = false; m_status = DPCC_OK; //--- Seed only the active epsilon prefix; padded plan rows remain zero. const uint layout = def_k_DPCCDenoiserFrameLayout; setBuffer(layout, def_k_DPCCDenoiserFrameLayout_key, epsilon_gradient.GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_value, m_frame_common[7].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_query, m_frame_common[7].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_pi, m_frame_common[2].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_query, m_frame_common[7].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_kv, m_frame_common[7].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_weighted, m_frame_backward[25].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_flags, m_frame_common[10].GetIndex()); setArgument(layout, def_k_DPCCDenoiserFrameLayout_K, active_count); setArgument(layout, def_k_DPCCDenoiserFrameLayout_tokens, m_frame_tape_size[25]); setArgument(layout, def_k_DPCCDenoiserFrameLayout_width, 6); setArgument(layout, def_k_DPCCDenoiserFrameLayout_heads, (uint)1); setArgument(layout, def_k_DPCCDenoiserFrameLayout_mode, (uint)def_k_DPCCDenoiserFrameLayout_backward_plan_prefix); uint offset[1] = {0}; uint size[1] = {m_frame_tape_size[25]}; if(!FrameExecute(layout, offset, size)) return(false); if(!FrameReduce(size[0])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=plan_seed slot=%u status=%d", slot, (int)m_status); return(false); } //--- D6: output affine, TANH, then split the skip branches. if(!FrameAffineBackward(20, slot, m_frame_backward[25], m_frame_backward[24]) || !FrameActivationBackward(FrameTape(slot, 23), m_frame_backward[24], m_frame_backward[23], full_width, TANH) || !FrameCopy(m_frame_backward[23], m_frame_backward[22], full_width) || !FrameCopy(m_frame_backward[23], m_frame_backward[12], full_width)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d6 slot=%u status=%d", slot, (int)m_status); return(false); } //--- D5 crop is adjointed by padding before the final hidden affine. if(!FramePair(m_frame_backward[22], m_frame_backward[41], def_k_DPCCDenoiserPairMap_backward_crop) || !FrameAffineBackward(19, slot, m_frame_backward[41], m_frame_backward[21])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d5 slot=%u status=%d", slot, (int)m_status); return(false); } //--- D4 feedforward residual and second self-attention branch. if(!FrameAffineBackward(18, slot, m_frame_backward[21], m_frame_backward[20])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_affine18 slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameActivationBackward(FrameTape(slot, 39), m_frame_backward[20], m_frame_backward[39], low_rows * 2 * m_frame_width, TANH)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_activation slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAffineBackward(17, slot, m_frame_backward[39], m_frame_work[0])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_affine17 slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAdd(m_frame_backward[21], m_frame_work[0], m_frame_backward[19], low_width)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_add slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameSelfBackward(1, 34, 13, (m_frame_horizon + 1) / 2, 14, 19, m_frame_backward[19], m_frame_backward[34])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_self_attention slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAffineBackward(12, slot, m_frame_backward[34], m_frame_backward[13])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_affine12 slot=%u status=%d", slot, (int)m_status); return(false); } if(!FramePair(m_frame_backward[13], m_frame_work[0], def_k_DPCCDenoiserPairMap_backward_pad)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_pair slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAdd(m_frame_backward[12], m_frame_work[0], m_frame_work[1], full_width)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_residual slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameCopy(m_frame_work[1], m_frame_backward[12], full_width)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d4_copy slot=%u status=%d", slot, (int)m_status); return(false); } //--- D2 feedforward residual and first self-attention branch. if(!FrameAffineBackward(11, slot, m_frame_backward[12], m_frame_backward[11]) || !FrameActivationBackward(FrameTape(slot, 32), m_frame_backward[11], m_frame_backward[32], full_rows * 2 * m_frame_width, TANH) || !FrameAffineBackward(10, slot, m_frame_backward[32], m_frame_work[0]) || !FrameAdd(m_frame_backward[12], m_frame_work[0], m_frame_backward[10], full_width) || !FrameSelfBackward(0, 4, 6, m_frame_horizon, 5, 10, m_frame_backward[10], m_frame_backward[4])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d2 slot=%u status=%d", slot, (int)m_status); return(false); } //--- D0: action-plan path and the shared step/common broadcasts. if(!FrameAffineBackward(4, slot, m_frame_backward[4], m_frame_backward[0]) || !FrameBroadcastGradient(m_frame_backward[4], m_frame_common[23], full_rows, m_frame_width) || !FrameAffineBackward(5, slot, m_frame_common[23], m_frame_backward[1])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=d0 slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameCopy(m_frame_backward[0], plan_gradient, active_count)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=plan_copy slot=%u status=%d", slot, (int)m_status); return(false); } //--- Reverse the shared cross-attention and original scenario mixture. if(!FrameAffineBackward(3, slot, m_frame_common[23], m_frame_common[20])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=cross_qkv slot=%u status=%d", slot, (int)m_status); return(false); } setBuffer(layout, def_k_DPCCDenoiserFrameLayout_key, m_frame_common[20].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_query, m_frame_common[7].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_pi, m_frame_common[2].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_query, m_frame_common[22].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_kv, m_frame_common[19].GetIndex()); setArgument(layout, def_k_DPCCDenoiserFrameLayout_K, m_frame_k); setArgument(layout, def_k_DPCCDenoiserFrameLayout_tokens, tokens); setArgument(layout, def_k_DPCCDenoiserFrameLayout_width, m_frame_width); setArgument(layout, def_k_DPCCDenoiserFrameLayout_heads, m_frame_heads); setArgument(layout, def_k_DPCCDenoiserFrameLayout_mode, (uint)def_k_DPCCDenoiserFrameLayout_backward_weighted); size[0] = m_frame_k * m_frame_width; if(!FrameExecute(layout, offset, size)) return(false); if(!FrameReduce(size[0]) || !FrameAttentionBackward(m_frame_common[5], m_frame_common[6], m_frame_common[8], m_frame_common[7], m_frame_common[22], m_frame_common[20], m_frame_common[21], 1, tokens, m_frame_k * m_frame_heads)) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=cross_attention slot=%u status=%d", slot, (int)m_status); return(false); } setBuffer(layout, def_k_DPCCDenoiserFrameLayout_key, m_frame_common[20].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_value, m_frame_common[21].GetIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_query, m_frame_affine[0].getGradientIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_packed_kv, m_frame_affine[1].getGradientIndex()); setBuffer(layout, def_k_DPCCDenoiserFrameLayout_weighted, m_frame_affine[2].getGradientIndex()); setArgument(layout, def_k_DPCCDenoiserFrameLayout_K, m_frame_k); setArgument(layout, def_k_DPCCDenoiserFrameLayout_tokens, tokens); setArgument(layout, def_k_DPCCDenoiserFrameLayout_width, m_frame_width); setArgument(layout, def_k_DPCCDenoiserFrameLayout_heads, m_frame_heads); setArgument(layout, def_k_DPCCDenoiserFrameLayout_mode, (uint)def_k_DPCCDenoiserFrameLayout_backward_attention); size[0] = m_frame_k * tokens * m_frame_width; if(!FrameExecute(layout, offset, size)) return(false); if(!FrameReduce(size[0])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=attention_layout slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAffineBackward(0, slot, m_frame_affine[0].getGradient(), m_frame_common[15])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=attention_query_input slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAffineBackward(1, slot, m_frame_affine[1].getGradient(), m_frame_common[16])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=attention_key_input slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAffineBackward(2, slot, m_frame_affine[2].getGradient(), m_frame_common[17])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=attention_value_input slot=%u status=%d", slot, (int)m_status); return(false); } if(!FrameAdd(m_frame_common[16], m_frame_common[17], m_frame_common[18], m_frame_common_size[18])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=attention_key_value_sum slot=%u status=%d", slot, (int)m_status); return(false); } //--- Normalization scales only financial fields; latent and Pi gradients pass through. const uint condition_kernel = def_k_DPCCDenoiserFrameCondition; setBuffer(condition_kernel, 0, condition_gradient.GetIndex()); setBuffer(condition_kernel, 1, m_frame_common[3].GetIndex()); setBuffer(condition_kernel, 2, m_frame_common[15].GetIndex()); setBuffer(condition_kernel, 3, m_frame_common[18].GetIndex()); setBuffer(condition_kernel, 4, m_frame_common[19].GetIndex()); setBuffer(condition_kernel, 5, m_frame_common[10].GetIndex()); setArgument(condition_kernel, 6, m_frame_k); setArgument(condition_kernel, 7, tokens); setArgument(condition_kernel, 8, m_frame_dimension); setArgument(condition_kernel, 9, m_frame_probability_tolerance); setArgument(condition_kernel, def_k_DPCCDenoiserFrameCondition_mode, (uint)def_k_DPCCDenoiserFrameCondition_backward); size[0] = m_frame_condition_count; if(!FrameExecute(condition_kernel, offset, size)) return(false); if(!FrameReduce(size[0])) { PrintFormat("DPCC_FRAME_BWD_FAIL stage=condition_vjp slot=%u status=%d", slot, (int)m_status); return(false); } m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Clears frame-aggregated dW only after the full forward unroll | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ZeroAccumulatedGradients(void) { const uint next_step = m_frame_update_count + 1; if(!m_frame_mode || !m_frame_live || !m_frame_training || m_frame_update_started || m_frame_failed || m_failed || !FrameOwned() || m_frame_next != m_frame_count || m_frame_count == 0 || m_frame_gradients_ready || m_frame_updated || m_frame_backward_count != 0 || m_frame_backward_next != 0 || m_frame_update_count >= INT_MAX - 1 || next_step >= INT_MAX) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint slot = 0; slot < m_frame_count; slot++) if(m_frame_slots[slot] != 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 21; i++) if(m_frame_affine[i].NativeOptimizerStep() != next_step) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 21; i++) if(!m_frame_accumulated[i].Fill(0)) return(FrameFail(DPCC_OPENCL_ERROR)); m_frame_backward_next = m_frame_count; m_frame_backward_count = 0; m_frame_gradients_ready = true; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Reverses frames in exact reverse unroll order and sums dW | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::BackwardFrame(uint slot, CBufferFloat *epsilon_gradient, CBufferFloat *plan_gradient, CBufferFloat *condition_gradient) { const uint active_count = m_frame_active_rows * m_frame_horizon * 6; if(!m_frame_mode || !m_frame_live || !m_frame_training || m_frame_update_started || !m_frame_gradients_ready || m_frame_failed || m_failed || slot >= m_frame_count || slot + 1 != m_frame_backward_next || m_frame_backward_count >= m_frame_count || !ValidBuffer(condition_gradient, m_frame_condition_count)) { const uint slot_state = (slot < m_frame_count ? m_frame_slots[slot] : 0); PrintFormat("DPCC_FRAME_BWD pre s=%u st=%u n=%u next=%u r=%u mode=%d live=%d train=%d ready=%d cond=%d", slot, slot_state, m_frame_count, m_frame_backward_next, m_frame_backward_count, (int)m_frame_mode, (int)m_frame_live, (int)m_frame_training, (int)m_frame_gradients_ready, (int)ValidBuffer(condition_gradient, m_frame_condition_count)); return(FrameFail(DPCC_CONTRACT_MISMATCH)); } CBufferFloat *external[] = {epsilon_gradient, plan_gradient, condition_gradient}; if(!ValidBuffer(epsilon_gradient, active_count) || !ValidBuffer(plan_gradient, active_count)) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 3; i++) { if(OwnsBufferOrDeviceSlot(external[i])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint k = 0; k < i; k++) if(BufferAliases(external[i], external[k])) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } //--- This API accumulates only this unroll's shared-condition VJP. if(m_frame_backward_count == 0 && !condition_gradient.Fill(0)) return(FrameFail(DPCC_OPENCL_ERROR)); if(!FrameBackward(slot, epsilon_gradient, plan_gradient, condition_gradient)) { PrintFormat("DPCC_FRAME_BWD_FAIL inner slot=%u status=%d", slot, (int)m_status); return(false); } m_frame_slots[slot] = 2; m_frame_backward_next--; m_frame_backward_count++; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Preflights all Adam candidates before one shared-parameter | //| update | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ApplyAccumulatedGradients(void) { const uint step = m_frame_update_count + 1; if(!m_frame_mode || !m_frame_live || !m_frame_training || m_frame_update_started || !m_frame_gradients_ready || m_frame_failed || m_failed || !FrameOwned() || m_frame_updated || m_frame_backward_count != m_frame_count || m_frame_backward_next != 0 || m_frame_update_count >= INT_MAX - 1 || step >= INT_MAX) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint slot = 0; slot < m_frame_count; slot++) if(m_frame_slots[slot] != 2) return(FrameFail(DPCC_CONTRACT_MISMATCH)); const float corrected = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)step)) / (1.0 - MathPow((double)b1, (double)step))); if(!MathIsValidNumber(corrected) || corrected <= 0 || !MathIsValidNumber(lr) || lr <= 0 || !MathIsValidNumber(b1) || !MathIsValidNumber(b2) || b1 != 0.9f || b2 != 0.999f) return(FrameFail(DPCC_NUMERIC_ERROR)); //--- Phase one validates every old parameter, moment and Adam candidate. //--- Phase two is the first write; each shared affine is updated once. for(uint phase = 0; phase < 2; phase++) { const uint mode = (phase == 0 ? def_k_DPCCSharedAffineAdamPreflight_update_preflight : def_k_DPCCSharedAffineAdamPreflight_update_apply); for(uint i = 0; i < 21; i++) { const uint count = m_frame_output[i] * (m_frame_input[i] + 1); CNeuronDPCCFrameAffineOCL *child = m_frame_affine[i]; if(!child || child.NativeOptimizerStep() != step || !ValidBuffer(m_frame_accumulated[i], count) || !ValidBuffer(child.GetWeightsConvReadOnly(), count) || !ValidBuffer(child.GetFirstMomentumConv(), count) || !ValidBuffer(child.GetSecondMomentumConv(), count)) return(FrameFail(DPCC_CONTRACT_MISMATCH, m_frame_update_started)); const uint kernel = def_k_DPCCSharedAffineAdamPreflight; setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_weights, child.GetWeightsConv().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_gradient, m_frame_accumulated[i].GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_input, child.GetWeightsConvReadOnly().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_momentum1, child.GetFirstMomentumConv().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_momentum2, child.GetSecondMomentumConv().GetIndex()); setBuffer(kernel, def_k_DPCCSharedAffineAdamPreflight_flags, m_frame_common[10].GetIndex()); const uint rows = 1; if(rows > def_k_DPCCSharedAffineAdamPreflight_frame_count_mask || mode == 0 || mode > 3) return(FrameFail(DPCC_CONTRACT_MISMATCH, m_frame_update_started)); const uint tokens = (uint)def_k_DPCCSharedAffineAdamPreflight_frame_tag | (mode << def_k_DPCCSharedAffineAdamPreflight_frame_mode_shift) | rows; if((tokens & (uint)def_k_DPCCSharedAffineAdamPreflight_frame_tag) == 0 || (tokens & (uint)def_k_DPCCSharedAffineAdamPreflight_frame_count_mask) != rows || ((tokens >> def_k_DPCCSharedAffineAdamPreflight_frame_mode_shift) & 0x3) != mode) return(FrameFail(DPCC_CONTRACT_MISMATCH, m_frame_update_started)); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_tokens, tokens); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_input_width, m_frame_input[i]); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_output_width, m_frame_output[i]); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_learning_rate, corrected); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_beta1, b1); setArgument(kernel, def_k_DPCCSharedAffineAdamPreflight_beta2, b2); uint offset[1] = {0}; uint size[1] = {count}; if(phase == 1) m_frame_update_started = true; if(!FrameExecute(kernel, offset, size)) return(false); if(!FrameReduce(count)) return(m_frame_update_started ? FrameFail(m_status, true) : false); } } for(uint i = 0; i < 21; i++) if(!FrameCheck(m_frame_affine[i].GetWeightsConvReadOnly()) || !FrameCheck(m_frame_affine[i].GetFirstMomentumConv()) || !FrameCheck(m_frame_affine[i].GetSecondMomentumConv())) return(FrameFail(m_status, true)); for(uint i = 0; i < 21; i++) if(!m_frame_affine[i].AdvanceFrameOptimizerStep(step)) return(FrameFail(DPCC_CONTRACT_MISMATCH, true)); m_frame_update_count++; m_frame_updated = true; m_frame_gradients_ready = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Converts every failed dispatch into a failed frame transaction | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ForwardFrame(uint slot, uint j, CBufferFloat *plan, CBufferFloat *condition, CBufferFloat *epsilon_out) { if(!FrameForward(slot, j, plan, condition, epsilon_out)) { const ENUM_DPCC_STATUS reason = (m_status == DPCC_OK ? DPCC_OPENCL_ERROR : m_status); PrintFormat("DPCC_DENOISER_FORWARD_FAIL slot=%u j=%u status=%d", slot, j, (int)reason); return(FrameFail(reason)); } return(true); } //+------------------------------------------------------------------+ //| Admits only the already materialized Action6 OpenCL context | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::FrameSetOpenCL(COpenCLMy *context) { if(m_frame_live || context != OpenCL || !FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH)); m_frame_cache_valid = false; m_status = DPCC_OK; return(true); } //+------------------------------------------------------------------+ //| Fingerprints complete FREE Action6 training state | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::AppendFrameTrainingFingerprint(ulong &fingerprint) { if(!m_frame_mode || !m_frame_bound || m_frame_live || m_frame_failed || m_failed || m_frame_training || m_frame_gradients_ready || m_frame_updated || m_frame_update_started || m_frame_count != 0 || m_frame_active_rows != 0 || m_frame_next != 0 || m_frame_current != 0 || m_frame_backward_next != 0 || m_frame_backward_count != 0 || m_frame_decision != 0 || m_frame_cache_valid || m_frame_cache_context != NULL || m_frame_cache_decision != 0 || m_frame_cache_market != 0 || m_frame_cache_weights != 0 || m_public_valid || m_status != DPCC_OK || m_frame_update_count > (uint)(INT_MAX - 1) || !FrameOwned()) return(false); ulong hash = fingerprint; const uint shape[] = {3, m_frame_horizon, m_frame_rows, m_frame_width, m_frame_heads, m_frame_max_frames, m_frame_k, m_frame_v, m_frame_market_horizon, m_frame_dimension, m_frame_condition_count, m_frame_update_count, 21 }; for(uint i = 0; i < (uint)ArraySize(shape); i++) DPCCRolloutHashWord(shape[i], hash); if(!MathIsValidNumber(m_frame_probability_tolerance) || m_frame_probability_tolerance < 0 || m_frame_probability_tolerance >= 1) return(false); DPCCRolloutHashDouble(m_frame_probability_tolerance, hash); for(uint i = 0; i < 218; i++) { if(!MathIsValidNumber(m_frame_profile[i]) || (i >= 109 && m_frame_profile[i] <= 0)) return(false); DPCCRolloutHashDouble(m_frame_profile[i], hash); } for(uint i = 0; i < 21; i++) { if(CheckPointer(m_frame_affine[i]) == POINTER_INVALID || m_frame_affine[i].NativeOptimizerStep() != m_frame_update_count + 1) return(false); DPCCRolloutHashWord(i, hash); DPCCRolloutHashWord(m_frame_affine[i].NativeOptimizerStep(), hash); if(!m_frame_affine[i].AppendDPCCPolicyTrainingFingerprint(hash)) return(false); } fingerprint = hash; return(true); } //+------------------------------------------------------------------+ //| Saves ABI3 Action6 parameters and child optimizer state | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::SaveFrameTrainingState(const int file_handle) { if(!m_frame_mode) return(false); if(file_handle == INVALID_HANDLE || !m_frame_bound || m_frame_live || m_frame_failed || m_frame_training || m_frame_gradients_ready || m_frame_updated || m_frame_update_started || m_frame_count != 0 || m_frame_active_rows != 0 || m_frame_next != 0 || m_frame_current != 0 || m_frame_backward_next != 0 || m_frame_backward_count != 0 || m_frame_decision != 0 || m_frame_cache_valid || m_frame_cache_context != NULL || m_frame_cache_decision != 0 || m_frame_cache_market != 0 || m_frame_cache_weights != 0 || m_public_valid || m_failed || m_status != DPCC_OK || m_frame_update_count > (uint)(INT_MAX - 1) || !FrameOwned()) return(FrameFail(DPCC_CONTRACT_MISMATCH)); //--- Validate immutable normalization and every native child before writing. if(!MathIsValidNumber(m_frame_probability_tolerance) || m_frame_probability_tolerance < 0 || m_frame_probability_tolerance >= 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); for(uint i = 0; i < 109; i++) if(!MathIsValidNumber(m_frame_profile[i]) || !MathIsValidNumber(m_frame_profile[109 + i]) || m_frame_profile[109 + i] <= 0) return(FrameFail(DPCC_NUMERIC_ERROR)); for(uint i = 0; i < (uint)ArraySize(m_frame_affine); i++) { CNeuronDPCCFrameAffineOCL *child = m_frame_affine[i]; if(CheckPointer(child) == POINTER_INVALID || child.Type() != defNeuronConvOCL || child.NativeOptimizerStep() != m_frame_update_count + 1) return(FrameFail(DPCC_CONTRACT_MISMATCH)); } //--- Bind the frame ABI, conditioning geometry and common optimizer clock. const uint header[] = {0x44504654, 1, (uint)Type(), 3, (uint)ArraySize(m_frame_affine), m_frame_horizon, m_frame_rows, m_frame_width, m_frame_heads, m_frame_max_frames, m_frame_k, m_frame_v, m_frame_market_horizon, m_frame_dimension, m_frame_condition_count, m_frame_update_count, (uint)ArraySize(m_frame_profile) }; for(uint i = 0; i < (uint)ArraySize(header); i++) if(FileWriteInteger(file_handle, (int)header[i], INT_VALUE) != INT_VALUE) return(FrameFail(DPCC_INVALID_INPUT)); if(FileWriteFloat(file_handle, m_frame_probability_tolerance) != sizeof(float)) return(FrameFail(DPCC_INVALID_INPUT)); for(uint i = 0; i < (uint)ArraySize(m_frame_profile); i++) if(FileWriteFloat(file_handle, m_frame_profile[i]) != sizeof(float)) return(FrameFail(DPCC_INVALID_INPUT)); //--- Save children in canonical slot order; each record owns weights and Adam state. for(uint i = 0; i < (uint)ArraySize(m_frame_affine); i++) { CNeuronDPCCFrameAffineOCL *child = m_frame_affine[i]; if(FileWriteInteger(file_handle, (int)i, INT_VALUE) != INT_VALUE) return(FrameFail(DPCC_INVALID_INPUT)); if(FileWriteInteger(file_handle, child.Type(), INT_VALUE) != INT_VALUE) return(FrameFail(DPCC_INVALID_INPUT)); if(!child.SaveDPCCPolicyTrainingStateOnly(file_handle)) return(FrameFail(DPCC_INVALID_INPUT)); } return(true); } //+------------------------------------------------------------------+ //| Reads the ABI3 frame record with optional native child restore | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::ReadFrameTrainingState(const int file_handle, const bool apply) { if(file_handle == INVALID_HANDLE || !m_frame_mode || !m_frame_bound || m_frame_live || m_frame_failed || m_frame_training || m_frame_gradients_ready || m_frame_updated || m_frame_update_started || m_frame_count != 0 || m_frame_active_rows != 0 || m_frame_next != 0 || m_frame_current != 0 || m_frame_backward_next != 0 || m_frame_backward_count != 0 || m_frame_decision != 0 || m_frame_cache_valid || m_frame_cache_context != NULL || m_frame_cache_decision != 0 || m_frame_cache_market != 0 || m_frame_cache_weights != 0 || m_public_valid || m_failed || m_status != DPCC_OK || !FrameOwned()) return(false); if(!MathIsValidNumber(m_frame_probability_tolerance) || m_frame_probability_tolerance < 0 || m_frame_probability_tolerance >= 1) return(false); //--- Verify the parent type, ABI and exact bound geometry before child reads. uint header[17]; ResetLastError(); for(uint i = 0; i < (uint)ArraySize(header); i++) { header[i] = (uint)FileReadInteger(file_handle, INT_VALUE); if(GetLastError() != 0) return(false); } if(header[0] != 0x44504654 || header[1] != 1 || header[2] != (uint)Type() || header[3] != 3 || header[4] != (uint)ArraySize(m_frame_affine) || header[5] != m_frame_horizon || header[6] != m_frame_rows || header[7] != m_frame_width || header[8] != m_frame_heads || header[9] != m_frame_max_frames || header[10] != m_frame_k || header[11] != m_frame_v || header[12] != m_frame_market_horizon || header[13] != m_frame_dimension || header[14] != m_frame_condition_count || header[15] > (uint)(INT_MAX - 1) || header[16] != (uint)ArraySize(m_frame_profile)) return(false); ResetLastError(); const float probability_tolerance = FileReadFloat(file_handle); if(GetLastError() != 0 || !MathIsValidNumber(probability_tolerance) || probability_tolerance != m_frame_probability_tolerance) return(false); for(uint i = 0; i < (uint)ArraySize(m_frame_profile); i++) { ResetLastError(); const float value = FileReadFloat(file_handle); if(GetLastError() != 0 || !MathIsValidNumber(value) || !MathIsValidNumber(m_frame_profile[i]) || value != m_frame_profile[i] || (i >= 109 && m_frame_profile[i] <= 0)) return(false); } //--- Validate or restore all 21 canonical Conv records in slot order. for(uint i = 0; i < (uint)ArraySize(m_frame_affine); i++) { CNeuronDPCCFrameAffineOCL *child = m_frame_affine[i]; if(CheckPointer(child) == POINTER_INVALID || child.Type() != defNeuronConvOCL) return(false); ResetLastError(); const int slot = FileReadInteger(file_handle, INT_VALUE); const int type = FileReadInteger(file_handle, INT_VALUE); if(GetLastError() != 0 || slot != (int)i || type != defNeuronConvOCL) return(false); //--- Reject a mixed child optimizer clock during preflight, before any buffer is applied. const ulong child_start = FileTell(file_handle); if(child_start == ULONG_MAX || child_start > (ulong)LONG_MAX) return(false); ResetLastError(); FileReadInteger(file_handle, INT_VALUE); FileReadInteger(file_handle, INT_VALUE); FileReadInteger(file_handle, INT_VALUE); FileReadInteger(file_handle, INT_VALUE); FileReadInteger(file_handle, INT_VALUE); const long child_step = FileReadLong(file_handle); if(GetLastError() != 0 || child_step != (long)(header[15] + 1) || !FileSeek(file_handle, (long)child_start, SEEK_SET)) return(false); if(!child.LoadDPCCPolicyTrainingStateOnly(file_handle, apply)) return(false); } if(apply) { //--- Reject inconsistent per-child clocks before publishing the restored state. for(uint i = 0; i < (uint)ArraySize(m_frame_affine); i++) if(m_frame_affine[i].NativeOptimizerStep() != header[15] + 1) return(false); m_frame_update_count = header[15]; if(!FrameOwned()) return(false); m_frame_cache_valid = false; m_frame_cache_context = NULL; m_frame_cache_decision = 0; m_frame_cache_market = 0; m_frame_cache_weights = 0; m_public_valid = false; m_status = DPCC_OK; } return(true); } //+------------------------------------------------------------------+ //| Validates then restores one exact Action6 training state | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::LoadFrameTrainingState(const int file_handle) { if(!m_frame_mode) return(false); if(file_handle == INVALID_HANDLE) return(FrameFail(DPCC_INVALID_INPUT)); //--- Preflight every byte before applying any child parameter buffer. const ulong start = FileTell(file_handle); if(start == ULONG_MAX || start > (ulong)LONG_MAX) return(FrameFail(DPCC_INVALID_INPUT)); if(!ReadFrameTrainingState(file_handle, false)) { if(!FileSeek(file_handle, (long)start, SEEK_SET)) return(FrameFail(DPCC_INVALID_INPUT)); return(FrameFail(DPCC_CONTRACT_MISMATCH)); } const ulong end = FileTell(file_handle); const ulong size = FileSize(file_handle); if(end == ULONG_MAX || size == ULONG_MAX || end <= start || end > size || end > (ulong)LONG_MAX) { FileSeek(file_handle, (long)start, SEEK_SET); return(FrameFail(DPCC_INVALID_INPUT)); } if(!FileSeek(file_handle, (long)start, SEEK_SET)) return(FrameFail(DPCC_INVALID_INPUT)); //--- Apply only the already validated record; any write failure requires checkpoint recovery. if(!ReadFrameTrainingState(file_handle, true) || FileTell(file_handle) != end) { FileSeek(file_handle, (long)start, SEEK_SET); return(FrameFail(DPCC_INVALID_INPUT, true)); } return(true); } //+------------------------------------------------------------------+ //| Checks both legacy and Action6 denoiser allocations | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { for(uint i = 0; i < 24; i++) if(BufferAliases(candidate, m_frame_common[i])) return(true); for(int i = 0; i < ArraySize(m_frame_tapes); i++) if(BufferAliases(candidate, m_frame_tapes[i])) return(true); for(uint i = 0; i < 43; i++) if(BufferAliases(candidate, m_frame_backward[i])) return(true); for(uint i = 0; i < 21; i++) if(BufferAliases(candidate, m_frame_accumulated[i])) return(true); for(uint i = 0; i < 5; i++) if(BufferAliases(candidate, m_frame_work[i])) return(true); for(uint i = 0; i < 21; i++) if(CheckPointer(m_frame_affine[i]) != POINTER_INVALID && m_frame_affine[i].OwnsBufferOrDeviceSlot(candidate)) return(true); if(CNeuronUShapeAttention::OwnsBufferOrDeviceSlot(candidate)) return(true); for(uint i = 0; i < 84; i++) if(BufferAliases(candidate, m_tape[i])) return(true); for(uint i = 0; i < 22; i++) if(CheckPointer(m_affine[i]) != POINTER_INVALID && m_affine[i].OwnsBufferOrDeviceSlot(candidate)) return(true); return(false); } //+------------------------------------------------------------------+ //| Function CNeuronMLMHAttentionOCL::OwnsBufferOrDeviceSlot | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { if(CNeuronBaseOCL::OwnsBufferOrDeviceSlot(candidate)) return(true); CCollection *collections[] = {QKV_Tensors, QKV_Weights, S_Tensors, AO_Tensors, FF_Tensors, FF_Weights}; for(uint c = 0; c < (uint)ArraySize(collections); c++) { if(CheckPointer(collections[c]) == POINTER_INVALID) continue; for(int i = 0; i < collections[c].Total(); i++) { CBufferFloat *buffer = dynamic_cast(collections[c].At(i)); if(BufferAliases(candidate, buffer)) return(true); } } return(false); } //+------------------------------------------------------------------+ //| Function CNeuronUShapeAttention::OwnsBufferOrDeviceSlot | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::OwnsBufferOrDeviceSlot(CBufferFloat *candidate) { if(CNeuronBaseOCL::OwnsBufferOrDeviceSlot(candidate)) return(true); for(uint i = 0; i < 2; i++) if(cAttention[i].OwnsBufferOrDeviceSlot(candidate)) return(true); for(uint i = 0; i < 2; i++) if(cMergeSplit[i].OwnsBufferOrDeviceSlot(candidate)) return(true); if(CheckPointer(cNeck) != POINTER_INVALID && cNeck.OwnsBufferOrDeviceSlot(candidate)) return(true); return(false); } //+------------------------------------------------------------------+ //| Function CNeuronPlanDenoiserOCL::CanImportPreparedParameters | //+------------------------------------------------------------------+ bool CNeuronPlanDenoiserOCL::CanImportPreparedParameters(CNeuronPlanDenoiserOCL *donor) { if(!donor || donor == GetPointer(this) || m_failed || !m_configured || m_phase != 0 || m_update_count != 0 || m_generation != 0 || !ValidateOwned() || donor.OpenCL != OpenCL || donor.m_failed || !donor.m_configured || donor.m_phase != 0 || !donor.ValidateOwned() || m_config.shape.K != donor.m_config.shape.K || m_config.shape.V != donor.m_config.shape.V || m_config.shape.D != donor.m_config.shape.D || m_config.shape.H != donor.m_config.shape.H || m_config.shape.Embedding != donor.m_config.shape.Embedding || m_f != donor.m_f || m_config.shape.Heads != donor.m_config.shape.Heads || m_config.shape.KeyDimension != donor.m_config.shape.KeyDimension) return(false); ulong receiver = 1469598103934665603, source = 1469598103934665603; HashComposition(receiver); donor.HashComposition(source); if(receiver != source) return(false); for(uint i = 0; i < 22; i++) { if(m_affine[i].NativeOptimizerStep() != 1 || !CheckParameters(i) || !donor.CheckParameters(i)) return(false); CBufferFloat *moments[] = {m_affine[i].GetFirstMomentumConv(), m_affine[i].GetSecondMomentumConv()}; for(uint j = 0; j < 2; j++) { if(!moments[j] || !moments[j].BufferRead()) return(false); for(int n = 0; n < moments[j].Total(); n++) if(moments[j].At(n) != 0.0f) return(false); } } return(true); } //+------------------------------------------------------------------+ //| Save LSE together with native attention activation buffers. | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::CaptureEvaluationState(SLayerReadState &state) {return(CNeuronBaseOCL::CaptureEvaluationState(state) && state.PutBuffer(GetPointer(cLogSumExp)));} //+------------------------------------------------------------------+ //| Function CCrossMHFlashAttention::RestoreEvaluationState | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::RestoreEvaluationState(SLayerReadState &state) { CBufferFloat *lse = GetPointer(cLogSumExp); return(CNeuronBaseOCL::RestoreEvaluationState(state) && state.GetBuffer(lse) && lse == GetPointer(cLogSumExp)); } //+------------------------------------------------------------------+