//+------------------------------------------------------------------+ //| ProjectName | //| Copyright 2020, CompanyName | //| http://www.companyname.net | //+------------------------------------------------------------------+ #ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE //+------------------------------------------------------------------+ //| Logical method implementations generated from NeuroNet.mqh: Attention //+------------------------------------------------------------------+ //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronAttentionOCL::~CNeuronAttentionOCL(void) { DeleteObj(Querys); DeleteObj(Values); DeleteObj(Scores); DeleteObj(AttentionOut); DeleteObj(FF1); DeleteObj(FF2); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) { Querys = new CNeuronConvOCL(); if(CheckPointer(Querys) == POINTER_INVALID) ReturnFalse; if(!Querys.Init(0, 0, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys.SetActivationFunction(None); } //--- if(CheckPointer(Values) == POINTER_INVALID) { Values = new CNeuronConvOCL(); if(CheckPointer(Values) == POINTER_INVALID) ReturnFalse; if(!Values.Init(0, 2, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values.SetActivationFunction(None); } //--- if(CheckPointer(Scores) == POINTER_INVALID) { Scores = new CBufferFloat(); if(CheckPointer(Scores) == POINTER_INVALID) ReturnFalse; } if(!Scores.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) { AttentionOut = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut) == POINTER_INVALID) ReturnFalse; if(!AttentionOut.Init(0, 3, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); } //--- if(CheckPointer(FF1) == POINTER_INVALID) { FF1 = new CNeuronConvOCL(); if(CheckPointer(FF1) == POINTER_INVALID) ReturnFalse; if(!FF1.Init(0, 4, open_cl, window, window, window * 4, units_count, optimization_type, batch)) ReturnFalse; FF1.SetActivationFunction(LReLU); } //--- if(CheckPointer(FF2) == POINTER_INVALID) { FF2 = new CNeuronConvOCL(); if(CheckPointer(FF2) == POINTER_INVALID) ReturnFalse; if(!FF2.Init(0, 5, open_cl, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; FF2.SetActivationFunction(None); FF2.SetGradientIndex(Gradient.GetIndex()); } //--- iWindow = window; iUnits = units_count; activation = (ENUM_ACTIVATION)FF2.Activation(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) //if(!prevLayer.Output.BufferRead()) // ReturnFalse; #ifdef _DEBUG if(!prevLayer.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, (int)iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, AttentionOut.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, AttentionOut.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(!FF1.FeedForward(AttentionOut)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!AttentionOut.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, AttentionOut.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, Querys.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, Values.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, Values.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, Scores.GetIndex()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!Querys.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Querys)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Values)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, prevLayer.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)(iWindow + 1)) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.1f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getGradientIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Values.UpdateInputWeights(prevLayer)) ReturnFalse; if(!FF1.UpdateInputWeights(AttentionOut)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronAttentionOCL *Source = source; dWeightsUpdate(Querys, Source, tau); dWeightsUpdate(Values, Source, tau); dWeightsUpdate(FF1, Source, tau); dWeightsUpdate(FF2, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(CheckPointer(Querys) == POINTER_INVALID || !Querys.Save(file_handle)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores) == POINTER_INVALID || !Scores.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut) == POINTER_INVALID || !AttentionOut.Save(file_handle)) ReturnFalse; if(CheckPointer(FF1) == POINTER_INVALID || !FF1.Save(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID || !FF2.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iUnits, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) Querys = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values) == POINTER_INVALID) Values = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores) == POINTER_INVALID) Scores = new CBufferFloat(); if(Scores.GetIndex() >= 0) Scores.BufferFree(); if(!Scores.Load(file_handle)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) AttentionOut = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(FF1) == POINTER_INVALID) FF1 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF1.Load(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID) FF2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF2.Load(file_handle)) ReturnFalse; iWindow = FileReadInteger(file_handle); iUnits = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronAttentionOCL::GetLayerInfo() { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; result.window = (int)iWindow; result.count = (int)iUnits; return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMHAttentionOCL::~CNeuronMHAttentionOCL(void) { DeleteObj(Querys2); DeleteObj(Querys3); DeleteObj(Querys4); DeleteObj(Keys2); DeleteObj(Keys3); DeleteObj(Keys4); DeleteObj(Values2); DeleteObj(Values3); DeleteObj(Values4); DeleteObj(Scores2); DeleteObj(Scores3); DeleteObj(Scores4); DeleteObj(Weights0); DeleteObj(AttentionOut2); DeleteObj(AttentionOut3); DeleteObj(AttentionOut4); DeleteObj(AttentionConcatenate); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAttentionOCL::Init(numOutputs, myIndex, open_cl, window, units_count, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) { Querys2 = new CNeuronConvOCL(); if(CheckPointer(Querys2) == POINTER_INVALID) ReturnFalse; if(!Querys2.Init(0, 6, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys2.SetActivationFunction(None); } //--- if(CheckPointer(Querys3) == POINTER_INVALID) { Querys3 = new CNeuronConvOCL(); if(CheckPointer(Querys3) == POINTER_INVALID) ReturnFalse; if(!Querys3.Init(0, 7, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys3.SetActivationFunction(None); } //--- if(CheckPointer(Querys4) == POINTER_INVALID) { Querys4 = new CNeuronConvOCL(); if(CheckPointer(Querys4) == POINTER_INVALID) ReturnFalse; if(!Querys4.Init(0, 8, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys4.SetActivationFunction(None); } //--- if(CheckPointer(Values2) == POINTER_INVALID) { Values2 = new CNeuronConvOCL(); if(CheckPointer(Values2) == POINTER_INVALID) ReturnFalse; if(!Values2.Init(0, 9, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values2.SetActivationFunction(None); } //--- if(CheckPointer(Values3) == POINTER_INVALID) { Values3 = new CNeuronConvOCL(); if(CheckPointer(Values3) == POINTER_INVALID) ReturnFalse; if(!Values3.Init(0, 10, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values3.SetActivationFunction(None); } //--- if(CheckPointer(Values4) == POINTER_INVALID) { Values4 = new CNeuronConvOCL(); if(CheckPointer(Values4) == POINTER_INVALID) ReturnFalse; if(!Values4.Init(0, 11, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values4.SetActivationFunction(None); } //--- if(CheckPointer(Scores2) == POINTER_INVALID) { Scores2 = new CBufferFloat(); if(CheckPointer(Scores2) == POINTER_INVALID) ReturnFalse; } if(!Scores2.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) { Scores3 = new CBufferFloat(); if(CheckPointer(Scores3) == POINTER_INVALID) ReturnFalse; } if(!Scores3.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) { Scores4 = new CBufferFloat(); if(CheckPointer(Scores4) == POINTER_INVALID) ReturnFalse; } if(!Scores4.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) { AttentionOut2 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut2) == POINTER_INVALID) ReturnFalse; if(!AttentionOut2.Init(0, 12, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut2.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) { AttentionOut3 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut3) == POINTER_INVALID) ReturnFalse; if(!AttentionOut3.Init(0, 13, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut3.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) { AttentionOut4 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut4) == POINTER_INVALID) ReturnFalse; if(!AttentionOut4.Init(0, 14, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut4.SetActivationFunction(None); } //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) { AttentionConcatenate = new CNeuronBaseOCL(); if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) ReturnFalse; if(!AttentionConcatenate.Init(0, 15, open_cl, 4 * window * units_count, optimization_type, batch)) ReturnFalse; AttentionConcatenate.SetActivationFunction(None); } //--- if(CheckPointer(Weights0) == POINTER_INVALID) { Weights0 = new CNeuronConvOCL(); if(CheckPointer(Weights0) == POINTER_INVALID) ReturnFalse; if(!Weights0.Init(0, 16, open_cl, 4 * window, 4 * window, window, units_count, optimization_type, batch)) ReturnFalse; Weights0.SetActivationFunction(None); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {1}; setBuffer(def_k_Normalize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, prevLayer.Neurons()); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.FeedForward(prevLayer)) ReturnFalse; //--- Scores Head 1 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()) setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()) kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 2 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys2.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores2.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores2.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores2.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values2.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut2.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut2.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 3 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys3.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores3.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores3.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores3.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values3.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut3.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut3.getOutput().BufferRead()) ReturnFalse; #endif } //--- Scores Head 4 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys4.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores4.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!Scores4.BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores4.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values4.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut4.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut4.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iUnits, iWindow }; setBuffer(def_k_ConcatenateMatrix, def_k_conc_input1, AttentionOut.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window1, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input2, AttentionOut2.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window2, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input3, AttentionOut3.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window3, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input4, AttentionOut4.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window4, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_out, AttentionConcatenate.getOutputIndex()); kernelExecute(def_k_ConcatenateMatrix, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionConcatenate.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.FeedForward(AttentionConcatenate)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getOutput().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normalize, def_k_norm_buffer, Weights0.getOutputIndex()); setArgument(def_k_Normalize, def_k_norm_dimension, Weights0.Neurons()); kernelExecute(def_k_Normalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(!FF1.FeedForward(Weights0)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!Weights0.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!Weights0.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!AttentionConcatenate.CalcHiddenGradients((CObject *)Weights0)) ReturnFalse; //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iWindow}; setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output1, AttentionOut.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window1, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output2, AttentionOut2.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window2, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output3, AttentionOut3.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window3, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output4, AttentionOut4.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window4, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_inputs, AttentionConcatenate.getGradientIndex()); kernelExecute(def_k_DeconcatenateMatrix, global_work_offset, global_work_size) #ifdef _DEBUG if(!AttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!calcHeadGradient(Querys, Values, Scores, AttentionOut, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys2, Values2, Scores2, AttentionOut2, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys3, Values3, Scores3, AttentionOut3, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys4, Values4, Scores4, AttentionOut4, prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix1, AttentionOut.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix2, AttentionOut2.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix3, AttentionOut3.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix4, AttentionOut4.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix5, Weights0.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix_out, prevLayer.getGradientIndex()); setArgument(def_k_Matrix5Sum, def_k_sum5_dimension, (int)iWindow); setArgument(def_k_Matrix5Sum, def_k_sum5_multiplyer, (float)0.2); kernelExecute(def_k_Matrix5Sum, global_work_offset, global_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcHeadGradient(CNeuronConvOCL *query, CNeuronConvOCL *value, CBufferFloat *score, CNeuronBaseOCL *attention, CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, attention.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, query.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, query.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, value.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, value.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, score.GetIndex()); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!query.getGradient().BufferRead()) ReturnFalse; #endif } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)query)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!prevLayer.CalcHiddenGradients((CObject *)value)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow + 1) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.33f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!attention.getGradient().BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer) || !Querys2.UpdateInputWeights(prevLayer) || !Querys3.UpdateInputWeights(prevLayer) || !Querys4.UpdateInputWeights(prevLayer)) ReturnFalse; //--- if(!Values.UpdateInputWeights(prevLayer) || !Values2.UpdateInputWeights(prevLayer) || !Values3.UpdateInputWeights(prevLayer) || !Values4.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Weights0.UpdateInputWeights(AttentionConcatenate)) ReturnFalse; if(!FF1.UpdateInputWeights(Weights0)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMHAttentionOCL *Source = source; if(!Querys.WeightsUpdate(Source.Querys, tau) || !Querys2.WeightsUpdate(Source.Querys2, tau) || !Querys3.WeightsUpdate(Source.Querys3, tau) || !Querys4.WeightsUpdate(Source.Querys4, tau)) ReturnFalse; //--- if(!Values.WeightsUpdate(Source.Values, tau) || !Values2.WeightsUpdate(Source.Values2, tau) || !Values3.WeightsUpdate(Source.Values3, tau) || !Values4.WeightsUpdate(Source.Values4, tau)) ReturnFalse; if(!Weights0.WeightsUpdate(Source.Weights0, tau)) ReturnFalse; if(!FF1.WeightsUpdate(Source.FF1, tau)) ReturnFalse; if(!FF2.WeightsUpdate(Source.FF2, tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Save(const int file_handle) { if(!CNeuronAttentionOCL::Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.Save(file_handle)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores2) == POINTER_INVALID || !Scores2.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut2) == POINTER_INVALID || !AttentionOut2.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.Save(file_handle)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores3) == POINTER_INVALID || !Scores3.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut3) == POINTER_INVALID || !AttentionOut3.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.Save(file_handle)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores4) == POINTER_INVALID || !Scores4.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut4) == POINTER_INVALID || !AttentionOut4.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID || !AttentionConcatenate.Save(file_handle)) ReturnFalse; if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Load(const int file_handle) { if(!CNeuronAttentionOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) Querys2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values2) == POINTER_INVALID) Values2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores2) == POINTER_INVALID) Scores2 = new CBufferFloat(); if(Scores2.GetIndex() >= 0) Scores2.BufferFree(); if(!Scores2.Load(file_handle)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) AttentionOut2 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID) Querys3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values3) == POINTER_INVALID) Values3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) Scores3 = new CBufferFloat(); if(Scores3.GetIndex() >= 0) Scores3.BufferFree(); if(!Scores3.Load(file_handle)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) AttentionOut3 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID) Querys4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values4) == POINTER_INVALID) Values4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) Scores4 = new CBufferFloat(); if(Scores4.GetIndex() >= 0) Scores4.BufferFree(); if(!Scores4.Load(file_handle)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) AttentionOut4 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) AttentionConcatenate = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionConcatenate.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Weights0) == POINTER_INVALID) Weights0 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Weights0.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronAttentionOCL::GetLayerInfo(); if(!result) return result; result.step = 4; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::CNeuronMLMHAttentionOCL(void) : iLayers(0), iHeads(0), iWindow(0), iWindowKey(0), iUnits(0) { QKV_Tensors = new CCollection(); QKV_Weights = new CCollection(); S_Tensors = new CCollection(); AO_Tensors = new CCollection(); FF_Tensors = new CCollection(); FF_Weights = new CCollection(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::~CNeuronMLMHAttentionOCL(void) { DeleteObj(QKV_Tensors); DeleteObj(QKV_Weights); DeleteObj(S_Tensors); DeleteObj(AO_Tensors); DeleteObj(FF_Tensors); DeleteObj(FF_Weights); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of our tensore uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? qkv_weights : 3 * iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? w0 : out), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = { outputs.Total() / (window_out * variables), window_out, variables }; setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_w, weights.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_i, inputs.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_o, outputs.GetIndex()); setArgument(def_k_FeedForwardConv, def_k_ffc_inputs, (int)(inputs.Total() / variables)); setArgument(def_k_FeedForwardConv, def_k_ffc_step, (int)step); setArgument(def_k_FeedForwardConv, def_k_ffc_window_in, (int)window); setArgument(def_k_FeedForwardConv, def_k_ffс_window_out, (int)window_out); setArgument(def_k_FeedForwardConv, def_k_ffc_activation, (int)activ); kernelExecute(def_k_FeedForwardConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHAttentionScore, def_k_mhas_mask, (int)mask); kernelExecute(def_k_MHAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionInputGradients(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total(); global_work_size[1] = variables; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, weights.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, gradient.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, inputs.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, gradient.Total() - shift_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)step); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)window); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)window_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)activ); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)shift_out); kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!inp_gradient.BufferRead()) ReturnFalse; #endif //--- return true;//inp_gradient.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionInsideGradients(CBufferFloat* qkv, CBufferFloat* qkv_g, CBufferFloat* scores, CBufferFloat* gradient) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_score, scores.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()); kernelExecute(def_k_MHAttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv_g.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads, 0, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* momentum1, CBufferFloat* momentum2, uint window, uint window_out, uint step = 0, uint heads = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = weights.Total(); uint global_work_offset_am[3] = {0, 0, 0}; uint global_work_size_am[3] = {window, window_out, variables}; uint local_work_size_am[3] = {window, (heads > 0 ? window_out / heads : 1), variables}; if(weights.GetIndex() < 0) ReturnFalse; float lt = 0; switch(optimization) { case SGD: if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_inputs, inputs.Total()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_learning_rates, lr); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_momentum, alpha); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_in, (int)window); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_out, (int)window_out); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_step, (int)step); ResetLastError(); kernelExecute(def_k_UpdateWeightsConvMomentum, global_work_offset, global_work_size) break; case ADAM: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsConvAdam, global_work_offset, global_work_size) t++; break; case ADAM_MINI: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_v, momentum2.GetIndex()) lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_l, lt) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b1, b1) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b2, b2) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_step, (int)step) ResetLastError(); kernelExecuteLoc(def_k_UpdateWeightsConvAdamMini, global_work_offset_am, global_work_size_am, local_work_size_am) t++; break; //--- default: printf("Error of optimization type %s: %s", __FUNCSIG__, EnumToString(optimization)); ReturnFalse; } #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- global_work_size[0] = window_out; setBuffer(def_k_NormalizeWeights, def_k_norm_buffer, weights.GetIndex()); setArgument(def_k_NormalizeWeights, def_k_norm_dimension, (int)window + 1); kernelExecute(def_k_NormalizeWeights, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- return true;//weights.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayers, INT_VALUE) || !FileWriteInteger(file_handle, iHeads, INT_VALUE) || !FileWriteInteger(file_handle, iWindow, INT_VALUE) || !FileWriteInteger(file_handle, iUnits, INT_VALUE) || !FileWriteInteger(file_handle, iWindowKey, INT_VALUE)) ReturnFalse; //--- Saving objects if(!QKV_Tensors.Save(file_handle) || !QKV_Weights.Save(file_handle) || !S_Tensors.Save(file_handle) || !AO_Tensors.Save(file_handle) || !FF_Tensors.Save(file_handle) || !FF_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 6 - 4); Gradient = FF_Tensors.At(iLayers * 6 - 1); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); QKV_Tensors.SetOpenCL(OpenCL); QKV_Weights.SetOpenCL(OpenCL); S_Tensors.SetOpenCL(OpenCL); AO_Tensors.SetOpenCL(OpenCL); FF_Tensors.SetOpenCL(OpenCL); FF_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMLMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; //--- result.window = (int)iWindow; result.step = (int)iHeads; result.window_out = (int)iWindowKey; result.count = (int)iUnits; result.layers = (int)iLayers; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionScore(CBufferFloat* qkv, CBufferFloat* scores, bool mask = true) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_sparse, m_dSparse); kernelExecute(def_k_MHSparseAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionOut(CBufferFloat* qkv, CBufferFloat* scores, CBufferFloat* out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHSparseAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHSparseAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteFloat(file_handle, m_dSparse) < sizeof(float)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; m_dSparse = FileReadFloat(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* source, CBufferFloat* momentum1, CBufferFloat* momentum2, float tau) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {weights.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, weights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, source.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, momentum1.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, momentum2.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, weights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, source.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMH2AttentionOCL::CNeuronMH2AttentionOCL(void) : iHeads(0), iWindow(0), iUnits(0), iWindowKey(0), ScoreIndex(INVALID_HANDLE) { activation = None; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Transpose.Init(0, 0, OpenCL, iUnits, iWindow, optimization_type, batch)) ReturnFalse; Transpose.SetActivationFunction(None); //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iUnits, iUnits, 2 * iWindowKey * iHeads, iWindow, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) FF[i].SetActivationFunction(None); //--- Gradient.BufferFree(); DeleteObj(Gradient); Gradient = FF[1].getGradient(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Transpose.FeedForward(NeuronOCL) || !KV_Embedding.FeedForward(Transpose.AsObject())) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindow/*K units*/, iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!MHAttentionOut.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(GetPointer(Transpose))) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!FF[1].CalcHiddenGradients((CObject *)GetPointer(FF[0]))) ReturnFalse; if(!FF[0].CalcHiddenGradients((CObject *)GetPointer(AttentionOut))) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!W0.CalcHiddenGradients((CObject *)GetPointer(MHAttentionOut))) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!KV_Embedding.CalcHiddenGradients((CObject *)GetPointer(Transpose))) ReturnFalse; if(!Q_Embedding.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getGradient(), AttentionOut.getGradient(), iWindow, false)) ReturnFalse; if(!Transpose.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), AttentionOut.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!Q_Embedding.Save(file_handle)) ReturnFalse; if(!KV_Embedding.Save(file_handle)) ReturnFalse; if(!Transpose.Save(file_handle)) ReturnFalse; if(!W0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, Q_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, KV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, Transpose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, W0.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, FF[i].AsObject())) ReturnFalse; if(!SetGradient(FF[1].getGradient(), true)) ReturnFalse; //--- iUnits = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization, iBatch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; AttentionOut.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMH2AttentionOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); Q_Embedding.SetOpenCL(OpenCL); KV_Embedding.SetOpenCL(OpenCL); if(Transpose.Neurons() > 0) Transpose.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); W0.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL); FF[0].SetOpenCL(OpenCL); FF[1].SetOpenCL(OpenCL); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMH2AttentionOCL *src = source; if(!Q_Embedding.WeightsUpdate(GetPointer(src.Q_Embedding), tau)) ReturnFalse; if(!KV_Embedding.WeightsUpdate(GetPointer(src.KV_Embedding), tau)) ReturnFalse; if(!W0.WeightsUpdate(GetPointer(src.W0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].WeightsUpdate(GetPointer(src.FF[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint features, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * features, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(features, 1); //--- MHSA uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tensor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of output tensor uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor //--- MHCA uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeads * iUnits; //Size of KV tensor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iUnits + 1) * iWindowKey * iHeads; //Size of weights' matrix of KV tenzor uint scores_ca = iUnits * iWindow * iHeads; //Size of Score tensor //--- FF uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- MHSA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cTranspose.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores_ca, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads cross attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- MHSA //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; for(uint w = 0; w < q_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float kv = (float)(1 / sqrt(iUnits + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* kv)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- MHSA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Transpose(CBufferFloat* in, CBufferFloat* out, int rows, int cols) { if(!in || !out) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {rows, cols}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, in.GetIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, out.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCA(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* score, CBufferFloat* out) { if(!q || !kv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindow, iHeads}; uint local_work_size[3] = {1, iWindow, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, score.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- MHSA //--- Calculate Queries, Keys, Values CBufferFloat *inputs = NeuronOCL.getOutput(); CBufferFloat *qkv = QKV_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 4); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 4); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 8); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- MHCA inputs = temp; CBufferFloat *q = QKV_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2); if(IsStopped() || !Transpose(inputs, tr, iUnits, iWindow)) ReturnFalse; CBufferFloat *kv = QKV_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), tr, kv, iUnits, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Multi-heads cross attention calculation temp = S_Tensors.At(i * 4 + 1); out = AO_Tensors.At(i * 4 + 1); if(IsStopped() || !MHCA(q, kv, temp, out)) ReturnFalse; //--- Cross Attention out calculation temp = FF_Tensors.At(i * 8 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 8 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 8 + 3); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, Output, iWindow, true, 0, 0, i * inputs.Total())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCAInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* score, CBufferFloat* aog) { if(!q || !qg || !kv || !kvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, score.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, aog.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!aog.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *inp = prevLayer.getOutput(); CBufferFloat *grad = prevLayer.getGradient(); //--- for(int i = 0; (i < (int)iLayers && !IsStopped()); i++) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), Gradient, FF_Tensors.At(i * 8 + 2), FF_Tensors.At(i * 8 + 6), 4 * iWindow, iWindow, None, i * inp.Total())) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 8 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(i * 8 + 6), FF_Tensors.At(i * 8 + 1), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradient if(IsStopped() || !SumAndNormalize(Gradient, temp, temp, iWindow, false, i * inp.Total(), 0, 0)) ReturnFalse; out_grad = temp; //--- MHCA //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out_grad, AO_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3), iWindowKey * iHeads, iWindow, None)) ReturnFalse; if(IsStopped() || !MHCAInsideGradients(QKV_Tensors.At(i * 6 + 1), QKV_Tensors.At(i * 6 + 4), QKV_Tensors.At(i * 6 + 2), QKV_Tensors.At(i * 6 + 5), S_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3))) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2 + 1); if(IsStopped() || !Transpose(QKV_Tensors.At(i * 6 + 5), tr, iWindow, iUnits)) ReturnFalse; //--- Sum temp = FF_Tensors.At(i * 8 + 4); if(IsStopped() || !SumAndNormalize(QKV_Tensors.At(i * 6 + 4), tr, temp, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; //--- MHSA //--- Split gradient to multi-heads out_grad = temp; if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out_grad, AO_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 2), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 6), QKV_Tensors.At(i * 6 + 3), S_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 1))) ReturnFalse; //--- if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(i * 6 + 3), inp, tr, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum gradients if(i > 0) { if(IsStopped() || !SumAndNormalize(grad, tr, grad, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormalize(out_grad, grad, grad, iWindow, false)) ReturnFalse; } else if(IsStopped() || !SumAndNormalize(out_grad, tr, grad, iWindow, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(l * 6 + 3), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), iWindow, 3 * iWindowKey * iHeads, 0, 3 * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), QKV_Tensors.At(l * 6 + 4), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), QKV_Tensors.At(l * 6 + 5), cTranspose.At(l * 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), iUnits, 2 * iWindowKey * iHeads, 0, 2 * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), FF_Tensors.At(l * 8 + 4), AO_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), FF_Tensors.At(l * 8 + 5), AO_Tensors.At(l * 4 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(l * 8 + 6), FF_Tensors.At(l * 8 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), FF_Tensors.At(l * 8 + 7), FF_Tensors.At(l * 8 + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle) || !cTranspose.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL) || !cTranspose.SetOpenCL(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMFTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMFTOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12)), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 3), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iUnits = fmax(units_count, 1); iHeads = fmax(fmin(heads, iWindow), 1); iWindowKey = fmax((window + iHeads - 1) / iHeads, 1); iLayers = fmax(layers, 1); iLPIWindow = fmax(lpi_window, 1); iLPIStep = 1; //--- XCA uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tensor uint scores = iWindowKey * iWindowKey * iHeads; //Size of Score tensor uint out = iWindow * iUnits; //Size of output tensor //--- LPI uint lpi1_num = iWindow * iHeads * iUnits; //Size of LPI1 tensor uint lpi1_weights = (iLPIWindow + 1) * iHeads; //Size of weights' matrix of LPI1 tenzor uint lpi2_weights = (iHeads + 1) * 2; //Size of weights' matrix of LPI2 tenzor //--- FF uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- LPI //--- Initilize LPI tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI1 ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI Normalize ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI2 ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi1_weights)) ReturnFalse; for(uint w = 0; w < lpi1_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Normalization int count = (int)lpi1_num * (optimization_type == SGD ? 7 : 9); temp = new CBufferFloat(); if(!temp.BufferInit(count, 0.0f)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi2_weights)) ReturnFalse; for(uint w = 0; w < lpi2_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- LPI temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi2_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } iBatchCount = 1; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiT(CBufferFloat* qkv, CBufferFloat* score, CBufferFloat* out) { if(!OpenCL || !qkv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; uint local_work_size[3] = {iWindowKey, iUnits, 1}; setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_score, score.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_out, out.GetIndex()) ResetLastError(); kernelExecuteLoc(def_k_XCiTFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(qkv, temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNorm(CBufferFloat* inputs, CBufferFloat* options, CBufferFloat* out) { if(!OpenCL || !inputs || !options || !out) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); iBatchCount = MathMin(iBatchCount, iBatch); setBuffer(def_k_BatchFeedForward, def_k_bff_inputs, inputs.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_options, options.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_output, out.GetIndex()) setArgument(def_k_BatchFeedForward, def_k_bff_batch, (int)iBatchCount) setArgument(def_k_BatchFeedForward, def_k_bff_optimization, (int)optimization) setArgument(def_k_BatchFeedForward, def_k_bff_activation, (int)None) ResetLastError(); kernelExecute(def_k_BatchFeedForward, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormInsideGradient(CBufferFloat* inputs, CBufferFloat* inputs_g, CBufferFloat* options, CBufferFloat* out, CBufferFloat* out_g, ENUM_ACTIVATION activ) { if(!OpenCL || !inputs || !inputs_g || !options || !out || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_i, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_options, options.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_g, out_g.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_ig, inputs_g.GetIndex()) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_activation, (int)activ) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_batch, (int)iBatchCount) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_optimization, (int)optimization) ResetLastError(); kernelExecute(def_k_CalcHiddenGradientBatch, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormUpdateWeights(CBufferFloat* options, CBufferFloat* out_g) { if(!OpenCL || !options || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = out_g.Total(); //--- if(optimization == SGD) { setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_learning_rates, lr) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_momentum, alpha) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsMomentum, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif } else { setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_l, lr) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b1, b1) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b2, b2) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!options.BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiTInsideGradients(CBufferFloat* qkv, CBufferFloat* qkvg, CBufferFloat* score, CBufferFloat* aog) { if(!OpenCL || !qkv || !qkvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv_g, qkvg.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_scores, score.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_gradient, aog.GetIndex()) ResetLastError(); kernelExecute(def_k_XCiTInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!aog.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLPIWindow, INT_VALUE) || !FileWriteInteger(file_handle, iLPIStep, INT_VALUE) || !FileWriteInteger(file_handle, iBatchCount, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cLPI.Save(file_handle) || !cLPI_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLPIWindow = FileReadInteger(file_handle, INT_VALUE); iLPIStep = FileReadInteger(file_handle, INT_VALUE); iBatchCount = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!cLPI.Load(file_handle) || !cLPI_Weights.Load(file_handle)) ReturnFalse; if(!cLPI.SetOpenCL(OpenCL) || !cLPI_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 4 - 3); Gradient = FF_Tensors.At(iLayers * 4 - 1); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cLPI.SetOpenCL(OpenCL); cLPI_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint dimension, uint heads, uint units_count, uint prev_window, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- if(prev_window != window) { if(!cProjInput.Init(0, 0, OpenCL, prev_window, prev_window, window, units_count, optimization_type, batch)) ReturnFalse; } //--- iWindowSize = window; iPrevWindowSize = prev_window; iDimension = dimension; iHeads = heads; iUnits = units_count; //--- if(!cQKV.Init(0, 1, OpenCL, window, window, dimension * heads, units_count, optimization_type, batch)) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- if(!cRelativePositionsBias.Init(1, 2, OpenCL, iUnits * iHeads * 3, optimization_type, batch)) ReturnFalse; if(!MHAttentionOut.Init(0, 3, OpenCL, iUnits * iHeads * iDimension, optimization_type, batch)) ReturnFalse; if(!cProj.Init(0, 4, OpenCL, iHeads * iDimension, iHeads * iDimension, window, iUnits, optimization_type, batch)) ReturnFalse; if(!AttentionOut.Init(0, 5, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cFF1.Init(0, 6, OpenCL, window, window, 4 * window, units_count, optimization_type, batch)) ReturnFalse; if(!cFF2.Init(0, 7, OpenCL, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; if(!SAttenOut.Init(0, 8, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cCAtten.Init(0, 9, OpenCL, window, MathMax(window / 2, 3), 8, iUnits, 1, optimization_type, batch)) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; if(iPrevWindowSize != iWindowSize) { if(!cProjInput.FeedForward(inputs) || !cQKV.FeedForward(GetPointer(cProjInput))) ReturnFalse; inputs = GetPointer(cProjInput); } else if(!cQKV.FeedForward(inputs)) ReturnFalse; if(!DOT()) ReturnFalse; if(!cProj.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; if(!SumAndNormalize(inputs.getOutput(), cProj.getOutput(), AttentionOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cFF1.FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.FeedForward(GetPointer(cFF1))) ReturnFalse; if(!SumAndNormalize(AttentionOut.getOutput(), cFF2.getOutput(), SAttenOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cCAtten.FeedForward(GetPointer(SAttenOut))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOT(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iUnits, iHeads}; uint local_work_size[3] = {iDimension, 1, 1}; setBuffer(def_k_DOTFeedForward, def_k_dot_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_score, iScoreBuffer) setBuffer(def_k_DOTFeedForward, def_k_dot_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_out, MHAttentionOut.getOutputIndex()) ResetLastError(); kernelExecuteLoc(def_k_DOTFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOTInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iDimension, iHeads}; setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_scores, iScoreBuffer) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb_g, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_gradient, MHAttentionOut.getGradientIndex()) ResetLastError(); kernelExecute(def_k_DOTInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cCAtten.CalcHiddenGradients((CObject *)GetPointer(SAttenOut))) ReturnFalse; if(!cFF2.CalcHiddenGradients((CObject *)GetPointer(cFF1))) ReturnFalse; if(!cFF1.CalcHiddenGradients((CObject *)GetPointer(AttentionOut))) ReturnFalse; if(!SumAndNormalize(AttentionOut.getGradient(), SAttenOut.getGradient(), cProj.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProj.CalcHiddenGradients((CObject *)GetPointer(MHAttentionOut))) ReturnFalse; if(!DOTInsideGradients()) ReturnFalse; //--- if(iPrevWindowSize != iWindowSize) { if(!cQKV.CalcHiddenGradients((CObject *)GetPointer(cProjInput))) ReturnFalse; if(!SumAndNormalize(cProjInput.getGradient(), cProj.getGradient(), cProjInput.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProjInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; } else { if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cProj.getGradient(), prevLayer.getGradient(), iWindowSize, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDOTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjInput.SetOpenCL(OpenCL); cQKV.SetOpenCL(OpenCL); iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); cRelativePositionsBias.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); cProj.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL);; cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); SAttenOut.SetOpenCL(OpenCL); cCAtten.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrevWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- ResetLastError(); if(iWindowSize != iPrevWindowSize) if(!cProjInput.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cQKV.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cRelativePositionsBias.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!MHAttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cProj.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!AttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF1.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF2.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!SAttenOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cCAtten.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindowSize = (uint)FileReadInteger(file_handle); iPrevWindowSize = (uint)FileReadInteger(file_handle); iDimension = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); //--- if(iWindowSize != iPrevWindowSize) if(!LoadInsideLayer(file_handle, GetPointer(cProjInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cRelativePositionsBias))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cProj))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(AttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF1))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF2))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(SAttenOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cCAtten))) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(iWindowSize != iPrevWindowSize) { if(!cProjInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(GetPointer(cProjInput))) ReturnFalse; } else { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; } //--- if(!cProj.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!cFF1.UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.UpdateInputWeights(GetPointer(cFF1))) ReturnFalse; if(!cCAtten.UpdateInputWeights(GetPointer(SAttenOut))) ReturnFalse; if(!updateRelativePositionsBias()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronDOTOCL *Source = source; if(iWindowSize != iPrevWindowSize) { if(!cProjInput.WeightsUpdate(GetPointer(Source.cProjInput), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } else { if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } //--- if(!cProj.WeightsUpdate(GetPointer(Source.cProj), tau)) ReturnFalse; if(!cFF1.WeightsUpdate(GetPointer(Source.cFF1), tau)) ReturnFalse; if(!cFF2.WeightsUpdate(GetPointer(Source.cFF2), tau)) ReturnFalse; if(!cCAtten.WeightsUpdate(GetPointer(Source.cCAtten), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateRelativePositionsBias(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {cRelativePositionsBias.Neurons()}; setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_gradient, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_m, cRelativePositionsBias.getFirstMomentumIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_v, cRelativePositionsBias.getSecondMomentumIndex()) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b1, b1) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b2, b2) ResetLastError(); kernelExecute(def_k_RPBUpdateAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!cRelativePositionsBias.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint heads, uint units_count, uint input_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- activation = None; //--- if(!cF.Init(0, 0, OpenCL, 3 * window, window, 8, fmax((int)input_units - 2, 1), optimization_type, batch)) ReturnFalse; cF.SetActivationFunction(None); if(!cNormV.Init(8, 1, OpenCL, fmax((int)input_units - 2, 1) * 8, batch, optimization_type)) ReturnFalse; cNormV.SetActivationFunction(None); if(!cWv.Init(units_count * window_out, 2, OpenCL, 8, optimization_type, batch)) ReturnFalse; cWv.SetActivationFunction(SIGMOID); if(!cQd.Init(0, 4, OpenCL, units_count * window_out, optimization_type, batch)) ReturnFalse; cQd.SetActivationFunction(None); if(!cDQd.Init(0, 5, OpenCL, window_out, 3, heads, units_count, 3, optimization_type, batch)) ReturnFalse; cDQd.SetActivationFunction(None); //--- if(Output != cDQd.getOutput()) { Output.BufferFree(); DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { Gradient.BufferFree(); DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- if(!cF.FeedForward(NeuronOCL)) ReturnFalse; if(!cNormV.FeedForward(GetPointer(cF))) ReturnFalse; if(!cWv.FeedForward(GetPointer(cNormV))) ReturnFalse; if(!cQd.FeedForward(GetPointer(cWv))) ReturnFalse; if(!cDQd.FeedForward(GetPointer(cQd))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cDQd.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cWv.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cWv))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cF))) ReturnFalse; if(!cF.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cDQd.UpdateInputWeights(GetPointer(cQd))) ReturnFalse; if(!cQd.UpdateInputWeights(GetPointer(cWv))) ReturnFalse; if(!cWv.UpdateInputWeights(GetPointer(cNormV))) ReturnFalse; if(!cNormV.UpdateInputWeights(GetPointer(cF))) ReturnFalse; if(!cF.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cF.Save(file_handle)) ReturnFalse; if(!cNormV.Save(file_handle)) ReturnFalse; if(!cWv.Save(file_handle)) ReturnFalse; if(!cQd.Save(file_handle)) ReturnFalse; if(!cDQd.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cF))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNormV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cWv))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQd))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cDQd))) ReturnFalse; //--- if(Output != cDQd.getOutput()) { DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cF.WeightsUpdate(source, tau)) ReturnFalse; if(!cNormV.WeightsUpdate(source, tau)) ReturnFalse; if(!cWv.WeightsUpdate(source, tau)) ReturnFalse; if(!cQd.WeightsUpdate(source, tau)) ReturnFalse; if(!cDQd.WeightsUpdate(source, tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFAQOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cF.SetOpenCL(OpenCL); cNormV.SetOpenCL(OpenCL); cWv.SetOpenCL(OpenCL); cQd.SetOpenCL(OpenCL); cDQd.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iWindow_K = fmax(window_k, 1); iUnits_K = fmax(units_k, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iWindow_K, iWindow_K, 2 * iWindowKey * iHeads, iUnits_K, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); if(!Transpose.Init(0, 0, OpenCL, iUnits, iWindow, optimization_type, batch)) ReturnFalse; Transpose.SetActivationFunction(None); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[0].SetActivationFunction(LReLU); if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[1].SetActivationFunction(None); //--- Gradient.BufferFree(); DeleteObj(Gradient); Gradient = FF[1].getGradient(); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits_K/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits_K, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!MHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!Context) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, Context.Total(), optimization, iBatch)) ReturnFalse; } //--- if(Context.GetIndex() >= 0) { CBufferFloat *inside = cContext.getOutput(); if(inside.GetIndex() != Context.GetIndex()) inside.BufferSet(Context.GetIndex()); } else { CBufferFloat *inside = cContext.getOutput(); inside.AssignArray(Context); if(!inside.BufferWrite()) ReturnFalse; } //--- return feedForward(NeuronOCL, cContext); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!KV_Embedding.FeedForward(Context)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits_K) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!Q_Embedding.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL* prevLayer, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!SecondInput) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, SecondInput.Total(), optimization, iBatch)) ReturnFalse; } cContext.SetActivationFunction(SecondActivation); //--- if(SecondGradient.GetIndex() >= 0) { CBufferFloat *inside = cContext.getGradient(); // SecondGradient принадлежит вызывающему коду. // cContext подключает его только на время текущего backward. if(!cContext.SetGradient(SecondGradient, false)) ReturnFalse; const bool calculated = calcInputGradients(prevLayer, cContext); // Восстанавливаем собственный градиент cContext до уничтожения // временного SecondGradient. const bool restored = cContext.SetGradient(inside, false); if(!restored || !calculated) ReturnFalse; } else { if(!calcInputGradients(prevLayer, cContext)) ReturnFalse; if(cContext.getGradient().GetData(SecondGradient) <= 0) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL* prevLayer, CNeuronBaseOCL* Context) { if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Context.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!Context) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, Context.Total(), optimization, iBatch)) ReturnFalse; } //--- if(Context.GetIndex() >= 0) { CBufferFloat *inside = cContext.getOutput(); if(inside.GetIndex() != Context.GetIndex()) inside.BufferSet(Context.GetIndex()); } else { CBufferFloat *inside = cContext.getOutput(); inside.AssignArray(Context); if(!inside.BufferWrite()) ReturnFalse; DeleteObj(Context); Context = inside; } //--- return updateInputWeights(NeuronOCL, cContext); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CNeuronBaseOCL* Context) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(Context)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Save(const int file_handle) { if(!CNeuronMH2AttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow_K) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits_K) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Load(const int file_handle) { if(!CNeuronMH2AttentionOCL::Load(file_handle)) ReturnFalse; iWindow_K = (uint)FileReadInteger(file_handle); iUnits_K = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCrossAttention::SetOpenCL(COpenCLMy *obj) { if(OpenCL == obj) return; COpenCLMy *previous = OpenCL; if(!!previous && ScoreIndex >= 0) previous.BufferFree(ScoreIndex); ScoreIndex = INVALID_HANDLE; CNeuronBaseOCL::SetOpenCL(obj); if(!OpenCL) return; Q_Embedding.SetOpenCL(OpenCL); KV_Embedding.SetOpenCL(OpenCL); if(Transpose.Neurons() > 0) Transpose.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); W0.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL); FF[0].SetOpenCL(OpenCL); FF[1].SetOpenCL(OpenCL); if(!!cContext) cContext.SetOpenCL(OpenCL); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!cQKV.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * 3 * iHeads, iUnits, optimization, iBatch)) ReturnFalse; if(!cSoftMax.Init(0, 1, OpenCL, iWindowKey * 3 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(3 * iHeads * iUnits); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!cMHAttentionOut.Init(0, 2, OpenCL, iWindowKey * 2 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cW0.Init(0, 3, OpenCL, 2 * iWindowKey * iHeads, 2 * iWindowKey * iHeads, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cAttentionOut.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Init(0, 5 + i, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; if(!cFF[i].Init(0, 7 + i, OpenCL, (i == 0 ? iWindow : 4 * iWindow), (i == 0 ? iWindow : 4 * iWindow), (i == 1 ? iWindow : 4 * iWindow), iUnits, optimization, iBatch)) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cSoftMax.FeedForward(GetPointer(cQKV))) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cMHAttentionOut))) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), iWindow, true)) ReturnFalse; if(!cGraphConv[0].FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].FeedForward(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].FeedForward(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].FeedForward(GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormalize(cAttentionOut.getOutput(), cFF[1].getOutput(), Output, iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cFF[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cGraphConv[1]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormalize(cAttentionOut.getGradient(), Gradient, cW0.getGradient(), iWindow, false)) ReturnFalse; if(!cW0.CalcHiddenGradients((CObject *)GetPointer(cMHAttentionOut))) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!cSoftMax.CalcHiddenGradients((CObject *)GetPointer(cQKV))) ReturnFalse; if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cMHAttentionOut))) ReturnFalse; if(!cGraphConv[0].UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].UpdateInputWeights(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].UpdateInputWeights(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].UpdateInputWeights(GetPointer(cFF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronGTE *Source = source; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].WeightsUpdate(GetPointer(Source.cGraphConv[i]), tau)) ReturnFalse; if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_GTEFeedForward, def_k_gteff_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEFeedForward, def_k_gteff_score, ScoreIndex) setBuffer(def_k_GTEFeedForward, def_k_gteff_out, cAttentionOut.getOutputIndex()) setArgument(def_k_GTEFeedForward, def_k_gteff_dimension, (int)iWindowKey) kernelExecuteLoc(def_k_GTEFeedForward, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_scores, ScoreIndex) setBuffer(def_k_GTEInsideGradients, def_k_gteig_gradient, cAttentionOut.getGradientIndex()) kernelExecute(def_k_GTEFeedForward, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGTE::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- if(!!OpenCL) if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) { cGraphConv[i].SetOpenCL(OpenCL); cFF[i].SetOpenCL(OpenCL); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Save(file_handle)) ReturnFalse; if(!cFF[i].Save(file_handle)) ReturnFalse; } //--- if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cSoftMax))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cMHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cAttentionOut))) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, GetPointer(cGraphConv[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; } //--- iHeads = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGTE::TrainMode(bool flag) { //--- CNeuronBaseOCL::TrainMode(flag); cQKV.TrainMode(flag); cSoftMax.TrainMode(flag); cMHAttentionOut.TrainMode(flag); cW0.TrainMode(flag); cAttentionOut.TrainMode(flag); for(int i = 0; i < 2; i++) { cGraphConv[i].TrainMode(flag); cFF[i].TrainMode(flag); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!cInput.Init(0, 0, OpenCL, window, window, 32, count, optimization, iBatch)) ReturnFalse; if(!cNorm.Init(0, 1, OpenCL, 32 * count, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(LReLU); //--- if(!cResidual[0].Init(0, 2, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[1].Init(0, 3, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[2].Init(0, 4, OpenCL, 32, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[3].Init(0, 5, OpenCL, 64, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[4].Init(0, 6, OpenCL, 64, 128, count, optimization, iBatch)) ReturnFalse; if(!cResidual[5].Init(0, 7, OpenCL, 128, 128, count, optimization, iBatch)) ReturnFalse; //--- if(!cOutput.Init(0, 8, OpenCL, 128, 128, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cInput.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].FeedForward(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].FeedForward(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.FeedForward(GetPointer(cResidual[5]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cOutput.CalcHiddenGradients((CObject *)GetPointer(cResidual[5]))) ReturnFalse; for(int i = 5; i > 0; i--) if(!cResidual[i].CalcHiddenGradients((CObject *)GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cResidual[0].CalcHiddenGradients((CObject *)GetPointer(cNorm))) ReturnFalse; if(!cNorm.CalcHiddenGradients((CObject *)GetPointer(cInput))) ReturnFalse; if(!cInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].UpdateInputWeights(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].UpdateInputWeights(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.UpdateInputWeights(GetPointer(cResidual[5]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CCCMREncoder *Source = source; if(!cInput.WeightsUpdate(GetPointer(Source.cInput), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(GetPointer(Source.cNorm), tau)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].WeightsUpdate(GetPointer(Source.cResidual[i]), tau)) ReturnFalse; if(!cOutput.WeightsUpdate(GetPointer(Source.cOutput), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].Save(file_handle)) ReturnFalse; if(!cOutput.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNorm))) ReturnFalse; for(int i = 0; i < 6; i++) if(!LoadInsideLayer(file_handle, GetPointer(cResidual[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cOutput))) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCCMREncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInput.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cOutput.SetOpenCL(OpenCL); for(int i = 0; i < 6; i++) cResidual[i].SetOpenCL(OpenCL); //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCCMREncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cInput.TrainMode(flag); cNorm.TrainMode(flag); cOutput.TrainMode(flag); for(int i = 0; i < 6; i++) cResidual[i].TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window1, uint window2, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronXCiTOCL::Init(numOutputs, myIndex, open_cl, window1, lpi_window, heads, units_count, layers, optimization_type, batch)) ReturnFalse; //--- Cross XCA iWindow2 = fmax(window2, 1); uint num = iWindowKey * iHeads * iUnits; //Size of V tensor uint v_weights = (iWindow2 + 1) * iWindowKey * iHeads; //Size of weights' matrix of V tensor //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize V tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cValue.Add(temp)) ReturnFalse; //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize V weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(v_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < v_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(v_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Motion) { if(!NeuronOCL || !Motion) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *v = cValue.At(i * 2); if(IsStopped() || !ConvolutionForward(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), Motion, v, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !Concat(qkv, v, cConcat.At(2 * i), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(cConcat.At(2 * i), temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::calcInputGradients(CNeuronBaseOCL* prevLayer, CNeuronBaseOCL* Motion) { if(!prevLayer || !Motion) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(cConcat.At(i * 2), cConcat.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; if(IsStopped() || !DeConcat(QKV_Tensors.At(i * 2 + 1), cValue.At(i * 2 + 1), cConcat.At(i * 2 + 1), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) out_grad = temp; if(i == iLayers - 1) { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), Motion.getGradient(), iWindow, iWindowKey * iHeads, None)) ReturnFalse; } else { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), GetPointer(TempBuffer), iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(TempBuffer), Motion.getGradient(), Motion.getGradient(), iWindow2, false)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Motion) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 2 * iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), cValue.At(l * 2 + 1), inputs, (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronCrossXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.cV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCrossXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronXCiTOCL::SetOpenCL(obj); cConcat.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cV_Weights.SetOpenCL(OpenCL); TempBuffer.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Save(const int file_handle) { if(!CNeuronXCiTOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, (int)iWindow2, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cValue.Save(file_handle) || !cV_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Load(const int file_handle) { if(!CNeuronXCiTOCL::Load(file_handle)) ReturnFalse; //--- Loading constant iWindow2 = (uint)FileReadInteger(file_handle); //--- Loading objects if(!cValue.Load(file_handle) || !cV_Weights.Load(file_handle)) ReturnFalse; //--- uint num = iWindowKey * iHeads * iUnits; //Size of V tensor //--- CBufferFloat *temp = NULL; for(uint i = 0; i < iLayers; i++) { for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!FeatureExtractor.Init(0, 0, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!PrevFeatures.Init(0, 1, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(!Motion.Init(0, 2, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); //--- if(!Temp.Init(0, 3, OpenCL, window * count, optimization, iBatch)) ReturnFalse; if(!LocalContext.Init(0, 4, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!GlobalContext.Init(0, 5, OpenCL, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!MotionContext.Init(0, 6, OpenCL, 16, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!RecurentUnit.Init(0, 7, OpenCL, 16 * count, optimization, iBatch) || !RecurentUnit.SetInputs(16 * count)) ReturnFalse; if(!UpScale.Init(0, 8, OpenCL, 16, 16, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Delta Features if(!SumAndNormalize(FeatureExtractor.getOutput(), FeatureExtractor.getOutput(), PrevFeatures.getOutput(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!FeatureExtractor.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormalize(FeatureExtractor.getOutput(), PrevFeatures.getOutput(), Motion.getOutput(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- Context if(Temp.getOutputIndex() != NeuronOCL.getOutputIndex()) { CBufferFloat* temp = Temp.getOutput(); temp.BufferSet(NeuronOCL.getOutputIndex()); Temp.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); } if(!LocalContext.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalContext.FeedForward(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.FeedForward(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; //--- Flow if(!RecurentUnit.FeedForward(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.FeedForward(GetPointer(RecurentUnit))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!UpScale.CalcHiddenGradients((CObject *)GetPointer(RecurentUnit))) ReturnFalse; if(!RecurentUnit.CalcHiddenGradients((CObject *)GetPointer(MotionContext))) ReturnFalse; //--- if(!MotionContext.calcInputGradients(GetPointer(GlobalContext), GetPointer(Motion))) ReturnFalse; if(!GlobalContext.CalcHiddenGradients((CObject *)GetPointer(LocalContext))) ReturnFalse; if(!LocalContext.CalcHiddenGradients((CObject *)GetPointer(Temp))) ReturnFalse; //--- if(!FeatureExtractor.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), Temp.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!FeatureExtractor.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!LocalContext.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!GlobalContext.UpdateInputWeights(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.UpdateInputWeights(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; if(!RecurentUnit.UpdateInputWeights(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.UpdateInputWeights(GetPointer(RecurentUnit))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronCCMROCL *Source = source; if(!FeatureExtractor.WeightsUpdate(GetPointer(Source.FeatureExtractor), tau)) ReturnFalse; if(!LocalContext.WeightsUpdate(GetPointer(Source.LocalContext), tau)) ReturnFalse; if(!GlobalContext.WeightsUpdate(GetPointer(Source.GlobalContext), tau)) ReturnFalse; if(!MotionContext.WeightsUpdate(GetPointer(Source.MotionContext), tau)) ReturnFalse; if(!RecurentUnit.WeightsUpdate(GetPointer(Source.RecurentUnit), tau)) ReturnFalse; if(!UpScale.WeightsUpdate(GetPointer(Source.UpScale), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Clear(void) { if(!RecurentUnit.Clear()) ReturnFalse; //--- CBufferFloat *temp = FeatureExtractor.getOutput(); temp.BufferInit(temp.Total(), 0); if(!temp.BufferWrite()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!FeatureExtractor.Save(file_handle)) ReturnFalse; if(!PrevFeatures.Save(file_handle)) ReturnFalse; if(!Motion.Save(file_handle)) ReturnFalse; if(!Temp.Save(file_handle)) ReturnFalse; if(!LocalContext.Save(file_handle)) ReturnFalse; if(!GlobalContext.Save(file_handle)) ReturnFalse; if(!MotionContext.Save(file_handle)) ReturnFalse; if(!RecurentUnit.Save(file_handle)) ReturnFalse; if(!UpScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(FeatureExtractor))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(PrevFeatures))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Motion))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Temp))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(LocalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(GlobalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MotionContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(RecurentUnit))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(UpScale))) ReturnFalse; //--- if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCCMROCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); FeatureExtractor.SetOpenCL(OpenCL); PrevFeatures.SetOpenCL(OpenCL); Motion.SetOpenCL(OpenCL); Temp.SetOpenCL(OpenCL); LocalContext.SetOpenCL(OpenCL); GlobalContext.SetOpenCL(OpenCL); MotionContext.SetOpenCL(OpenCL); RecurentUnit.SetOpenCL(OpenCL); UpScale.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCCMROCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); FeatureExtractor.TrainMode(bTrain); PrevFeatures.TrainMode(bTrain); Motion.TrainMode(bTrain); Temp.TrainMode(bTrain); LocalContext.TrainMode(bTrain); GlobalContext.TrainMode(bTrain); MotionContext.TrainMode(bTrain); RecurentUnit.TrainMode(bTrain); UpScale.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint variables, uint lenth, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * variables * lenth, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iVariables = variables; iLenth = lenth; //--- uint mult = 2; uint weights = (iDimension + 2) * iDimension * iVariables; //--- if(ArrayResize(iBuffersK, 18) < 18) ReturnFalse; if(ArrayResize(iInputsK, 18) < 18) ReturnFalse; if(ArrayResize(iMeadl, 12) < 12) ReturnFalse; CBufferFloat *temp = NULL; //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iInputsK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Meadl Output and Gradient buffers iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- Initilize Weights for(int i = 0; i < 2; i++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(weights)) ReturnFalse; float k = (float)(1 / sqrt(iDimension + 2)); for(uint w = 0; w < weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; //--- for(uint d = 0; d < 2; d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; } } //--- Constants //--- Alpha { float temp_ar[] = {0, 0.2f, 0.3f, 0.8f, 8.0f / 9, 1, 1}; if(!cAlpha.AssignArray(temp_ar)) ReturnFalse; if(!cAlpha.BufferCreate(OpenCL)) ReturnFalse; } //--- Beta K1 { float temp_ar[] = {0, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K2 { float temp_ar[] = {0.2f, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K3 { float temp_ar[] = {3.0f / 40, 9.0f / 40, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K4 { float temp_ar[] = {44.0f / 44, -56.0f / 15, 32.0f / 9, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K5 { float temp_ar[] = {19372.0f / 6561, -25360 / 2187.0f, 64448 / 6561.0f, -212.0f / 729, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K6 { float temp_ar[] = {9017 / 3168.0f, -355 / 33.0f, 46732 / 5247.0f, 49.0f / 176, -5103.0f / 18656, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Yt+1 { float temp_ar[] = {35.0f / 384, 0, 500.0f / 1113, 125.0f / 192, -2187.0f / 6784, 11.0f / 84}; if(!cSolution.AssignArray(temp_ar)) ReturnFalse; if(!cSolution.BufferCreate(OpenCL)) ReturnFalse; } //--- if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputK(CBufferFloat* inputs, int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- if(k == 0) { if(iInputsK[k] != inputs.GetIndex()) { OpenCL.BufferFree(iInputsK[k]); iInputsK[k] = inputs.GetIndex(); } return true; } //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k]) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat *)cBeta.At(k)).BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBuffer(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iInputsK[k]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iMeadl[k * 2]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, float(cAlpha.At(k))) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(LReLU)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iBuffersK[k]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, cAlpha.At(k)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(None)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutput(CBufferFloat* inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, Output.GetIndex()) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- if(!SumAndNormalize(Output, inputs, Output, iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { for(int k = 0; k < 6; k++) { if(!CalculateInputK(NeuronOCL.getOutput(), k)) ReturnFalse; if(!CalculateKBuffer(k)) ReturnFalse; } //--- return CalculateOutput(NeuronOCL.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutputGradient(CBufferFloat* inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[6]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[7]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[8]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[9]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[10]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[11]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, Gradient.GetIndex()) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!Gradient.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBufferGradient(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iMeadl[k * 2 + 1]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iBuffersK[k + 6]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(LReLU)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iInputsK[k]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iInputsK[k + 12]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iMeadl[k * 2 + 1]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(None)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputKGradient(CBufferFloat* inputs, int k) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[12]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[13]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[14]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[15]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[16]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[17]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k + 6]) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat *)cBeta.At(k)).BufferRead()) ReturnFalse; #endif //--- for(int i = k - 1; i >= 0; i--) { float mult = 1.0f / (i == (k - 1) ? 6 - k : 1); uint global_work_offset[1] = {0}; uint global_work_size[1] = {iLenth * iVariables}; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, iBuffersK[k + 6]) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, iBuffersK[k + 12]) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, iBuffersK[k + 6]) setArgument(def_k_MatrixSum, def_k_sum_dimension, iDimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, mult) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!inputs.BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!CalculateOutputGradient(prevLayer.getGradient())) ReturnFalse; for(int k = 5; k >= 0; k--) { if(!CalculateKBufferGradient(k)) ReturnFalse; if(!CalculateInputKGradient(GetPointer(cTemp), k)) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getOutput(), iDimension, false, 0, 0, 0, 1.0f / (k == 0 ? 6 : 1))) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension + 2, iDimension, iVariables}; ResetLastError(); setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iInputsK[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iMeadl[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iInputsK[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iMeadl[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iInputsK[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iMeadl[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iInputsK[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iMeadl[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iInputsK[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iMeadl[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iInputsK[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iMeadl[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(1)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(2)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(0)).BufferRead()) ReturnFalse; #endif //--- setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iMeadl[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iBuffersK[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iMeadl[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iBuffersK[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iMeadl[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iBuffersK[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iMeadl[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iBuffersK[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iMeadl[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iBuffersK[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iMeadl[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iBuffersK[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(4)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(5)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CBufferFloat*)cWeights.At(3)).BufferRead()) ReturnFalse; #endif //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWeights.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iVariables), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iLenth), INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cWeights.Load(file_handle)) ReturnFalse; cWeights.SetOpenCL(OpenCL); //--- iDimension = (int)FileReadInteger(file_handle); iVariables = (int)FileReadInteger(file_handle); iLenth = (int)FileReadInteger(file_handle); //--- CBufferFloat *temp = NULL; for(uint i = 0; i < 18; i++) { OpenCL.BufferFree(iBuffersK[i]); OpenCL.BufferFree(iInputsK[i]); //--- iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Output and Gradient buffers OpenCL.BufferFree(iMeadl[i]); iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- cTemp.BufferFree(); if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronNODEOCL::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- CNeuronBaseOCL::SetOpenCL(obj); cWeights.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); cAlpha.BufferCreate(OpenCL); cBeta.SetOpenCL(OpenCL); cSolution.BufferCreate(OpenCL); //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(i > 11) continue; iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); } //--- } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint variables, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * variables * units_count, optimization_type, batch)) ReturnFalse; if(!cQKV.Init(0, 0, OpenCL, window, window, 3 * window_key * heads, variables * units_count, optimization, iBatch)) ReturnFalse; //--- iWindow = int(fmax(window, 1)); iDimension = int(fmax(window_key, 1)); iHeads = int(fmax(heads, 1)); iVariables = int(fmax(variables, 1)); iCount = int(fmax(units_count, 1)); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cW0.Init(0, 3, OpenCL, iDimension * iHeads, iDimension * iHeads, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 3; i++) if(!cNODE[i].Init(0, 4 + i, OpenCL, iWindow, iVariables, iCount, optimization, iBatch)) ReturnFalse; //--- if(!cFF[0].Init(0, 7, OpenCL, iWindow, iWindow, 4 * iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cFF[1].Init(0, 8, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- Time Derivative { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_TimeDerivative, def_k_tdqkv, cQKV.getOutputIndex()) setBuffer(def_k_TimeDerivative, def_k_tddqkv, cdQKV.getOutputIndex()) setArgument(def_k_TimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_TimeDerivative, global_work_offset, global_work_size) #ifdef _DEBUG if(!cdQKV.getOutput().BufferRead()) ReturnFalse; #endif } //--- MH Attention Out { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iCount, iVariables}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_FeedForwardContAtt, def_k_caqkv, cQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cascore, iScore) setBuffer(def_k_FeedForwardContAtt, def_k_caout, cAttentionOut.getOutputIndex()) setArgument(def_k_FeedForwardContAtt, def_k_cadimension, int(iDimension)) setArgument(def_k_FeedForwardContAtt, def_k_caheads, int(iHeads)) kernelExecuteLoc(def_k_FeedForwardContAtt, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Generate Query, Key, Value if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //--- MH Continuas Attention if(!attentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cW0.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } if(!SumAndNormalize(prev.getOutput(), cW0.getOutput(), prev.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } if(!SumAndNormalize(prev.getOutput(), cNODE[2].getOutput(), getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- MH Attention Out Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv, cQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv_g, cQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv_g, cdQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcascore, iScore) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaout_g, cAttentionOut.getGradientIndex()) setArgument(def_k_HiddenGradientContAtt, def_k_hgcadimension, int(iDimension)) kernelExecute(def_k_HiddenGradientContAtt, global_work_offset, global_work_size) #ifdef _DEBUG if(!cAttentionOut.getGradient().BufferRead()) ReturnFalse; #endif } //--- Time Derivative Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HGTimeDerivative, def_k_tdqkv, cQKV.getGradientIndex()) setBuffer(def_k_HGTimeDerivative, def_k_tddqkv, cdQKV.getGradientIndex()) setArgument(def_k_HGTimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_HGTimeDerivative, global_work_offset, global_work_size) #ifdef _DEBUG if(!cdQKV.getGradient().BufferRead()) ReturnFalse; #endif } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::calcInputGradients(CNeuronBaseOCL* prevLayer) { //--- Feed Forward Gradient if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cFF[1]))) ReturnFalse; if(!cNODE[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormalize(Gradient, cNODE[2].getGradient(), cNODE[2].getGradient(), iDimension, false)) ReturnFalse; //--- Neural ODE Gradient CNeuronBaseOCL *prev = cNODE[1].AsObject(); for(int i = 2; i > 0; i--) { if(!prev.CalcHiddenGradients(cNODE[i].AsObject())) ReturnFalse; prev = cNODE[i - 1].AsObject(); } if(!cW0.CalcHiddenGradients(prev.AsObject())) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), cNODE[2].getGradient(), cW0.getGradient(), iDimension, false)) ReturnFalse; //--- MH Attention Gradient if(!cAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; //--- Query, Key, Value Graddients if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!SumAndNormalize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iDimension, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronConformer *Source = source; //--- MH Attention if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- MH Attention if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) if(!cNODE[i].Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFF[i].Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; for(int i = 0; i < 3; i++) if(!LoadInsideLayer(file_handle, GetPointer(cNODE[i]))) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; //--- iWindow = FileReadInteger(file_handle); iDimension = FileReadInteger(file_handle); iHeads = FileReadInteger(file_handle); iVariables = FileReadInteger(file_handle); iCount = FileReadInteger(file_handle); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(iScore >= 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConformer::SetOpenCL(COpenCLMy * obj) { if(obj == OpenCL) return; if(iScore > 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cdQKV.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); for(int i = 0; i < 3; i++) cNODE[i].SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cFF[i].SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); //--- if(getGradientIndex() != cFF[1].getGradientIndex()) SetGradientIndex(cFF[1].getGradientIndex()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint at_layers, uint count, uint & mlp[], ENUM_OPTIMIZATION optimization_type, uint batch) { uint mlp_layers = mlp.Size(); if(mlp_layers == 0) ReturnFalse; if(ArrayResize(cLinearModel, mlp_layers + 1) != (mlp_layers + 1)) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, mlp[mlp_layers - 1] * count, optimization_type, batch)) ReturnFalse; if(!cTransformerEncoder.Init(0, 0, OpenCL, window, window_key, heads, count, at_layers, optimization, iBatch)) ReturnFalse; if(!cInput.Init(0, 1, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- uint w = window; for(uint i = 0; i < mlp_layers; i++) { if(!cLinearModel[i].Init(0, i + 2, OpenCL, w, w, mlp[i], count, optimization, iBatch)) ReturnFalse; cLinearModel[i].SetActivationFunction(LReLU); w = mlp[i]; } if(!cLinearModel[mlp_layers].Init(0, mlp_layers + 2, OpenCL, 1, 1, 1, w * count, optimization, iBatch)) ReturnFalse; cLinearModel[mlp_layers].SetActivationFunction(TANH); if(!cProjection.Init(0, mlp_layers + 3, OpenCL, window, window, w, count, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); SetActivationFunction(TANH); //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[mlp_layers].SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTransformerEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjection.FeedForward(GetPointer(cTransformerEncoder))) ReturnFalse; if(cInput.getOutputIndex() != NeuronOCL.getOutputIndex()) cInput.getOutput().BufferSet(NeuronOCL.getOutputIndex()); //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].FeedForward(neuron)) ReturnFalse; neuron = GetPointer(cLinearModel[i]); } //--- if(!SumAndNormalize(neuron.getOutput(), cProjection.getOutput(), Output, 1, false, 0, 0, 0, 0.5)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!cTransformerEncoder.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cTransformerEncoder.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *neuron = NULL; int total = (int)cLinearModel.Size() - 1; for(int i = total; i >= 0; i--) { neuron = (i > 0 ? cLinearModel[i - 1] : cInput).AsObject(); if(!neuron.CalcHiddenGradients(cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SumAndNormalize(neuron.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cTransformerEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cProjection.UpdateInputWeights(cTransformerEncoder.AsObject())) ReturnFalse; //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = cLinearModel[i].AsObject(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronClientOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cTransformerEncoder.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cInput.SetOpenCL(OpenCL); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cTransformerEncoder.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; uint total = cLinearModel.Size(); if(FileWriteInteger(file_handle, int(total), INT_VALUE) != INT_VALUE) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Load(const int file_handle) { ArrayFree(cLinearModel); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransformerEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInput.AsObject())) ReturnFalse; int total = FileReadInteger(file_handle); if(total <= 0) ReturnFalse; if(ArrayResize(cLinearModel, total) < total) ReturnFalse; for(int i = 0; i < total; i++) { cLinearModel[i].SetOpenCL(OpenCL); if(!LoadInsideLayer(file_handle, cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[total - 1].SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronClientOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cTransformerEncoder.TrainMode(bTrain); cProjection.TrainMode(bTrain); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronClientOCL *sour = source; if(!cTransformerEncoder.WeightsUpdate(sour.cTransformerEncoder.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(sour.cProjection.AsObject(), tau)) ReturnFalse; //--- uint total = cLinearModel.Size(); if(total != sour.cLinearModel.Size()) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].WeightsUpdate(sour.cLinearModel[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, 4 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronUShapeAttention *temp = new CNeuronUShapeAttention(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, 2 * units_count, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window, window, 2 * units_count, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, 2 * units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, 2 * window, 2 * window, window, units_count, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), Output, 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), Gradient, prevLayer.getGradient(), 1, false)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronUShapeAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) { cAttention[i].SetOpenCL(OpenCL); cMergeSplit[i].SetOpenCL(OpenCL); } if(!!cNeck) cNeck.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cAttention[i].Save(file_handle)) ReturnFalse; if(!cMergeSplit[i].Save(file_handle)) ReturnFalse; } if(!cNeck.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronUShapeAttention: cNeck = new CNeuronUShapeAttention(); if(!cNeck) ReturnFalse; break; case defNeuronConvOCL: cNeck = new CNeuronConvOCL(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) return true; //--- CNeuronUShapeAttention *temp = source; for(int i = 0; i < 2; i++) { if(!cAttention[i].WeightsUpdate(temp.cAttention[i].AsObject(), tau)) ReturnFalse; if(!cMergeSplit[i].WeightsUpdate(temp.cMergeSplit[i].AsObject(), tau)) ReturnFalse; } if(!cNeck.WeightsUpdate(temp.cNeck.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- uint num = 2 * 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 2 * 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tenzor uint scores = 2 * iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = 2 * iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = 2 * iWindow * iUnits; //Size of our tensore uint w0 = 2 * (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = 4 * (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionForward(CBufferFloat* weights, CBufferFloat* inputs, CBufferFloat* outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = outputs.Total() / (2 * window_out); global_work_size[1] = window_out; global_work_size[2] = 1; //--- setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_w, weights.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_o, outputs.GetIndex()) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_inputs, (int)(inputs.Total() / 2)) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_step, (int)step) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_window_in, (int)window) setArgument(def_k_FeedForwardComplexConv, def_k_ffс_window_out, (int)window_out) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_activation - 1, (int)activ) //--- kernelExecute(def_k_FeedForwardComplexConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionScore(CBufferFloat* qkv, CBufferFloat* scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_score, scores.GetIndex()) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_dimension, (int)iWindowKey) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_mask, (int)mask) kernelExecute(def_k_ComplexMHAttentionScore, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionOut(CBufferFloat* qkv, CBufferFloat* scores, CBufferFloat* out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_out, out.GetIndex()) setArgument(def_k_ComplexMHAttentionOut, def_k_mhao_dimension, (int)iWindowKey) //--- kernelExecute(def_k_ComplexMHAttentionOut, global_work_offset, global_work_size) #ifdef _DEBUG if(!scores.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutuionUpdateWeights(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* momentum1, CBufferFloat* momentum2, uint window, uint window_out, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = weights.Total() / 2; global_work_size[1] = weights.Total() / 2; if(weights.GetIndex() < 0) ReturnFalse; if(optimization == SGD) { if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_learning_rates, lr) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_momentum, alpha) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvMomentum, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif } else { if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) float lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvAdam, global_work_offset, global_work_size) #ifdef _DEBUG if(!weights.BufferRead()) ReturnFalse; #endif t++; } //--- return true;; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionInputGradients(CBufferFloat* weights, CBufferFloat* gradient, CBufferFloat* inputs, CBufferFloat* inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total() / 2; global_work_size[1] = 1; //--- setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_w, weights.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_g, gradient.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_o, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_outputs, int(Neurons() / 2) - shift_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_step, (int)step) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_in, (int)window) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_out, (int)window_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_activation, (int)activ) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_shift_out, (int)shift_out) kernelExecute(def_k_CalcHiddenGradientComplexConv, global_work_offset, global_work_size) #ifdef _DEBUG if(!inp_gradient.BufferRead()) ReturnFalse; #endif //--- return true;; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionInsideGradients(CBufferFloat* qkv, CBufferFloat* qkv_g, CBufferFloat* scores, CBufferFloat* gradient) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()) //--- kernelExecute(def_k_ComplexMHAttentionGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!qkv.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::SumAndNormalize(CBufferFloat* tensor1, CBufferFloat* tensor2, CBufferFloat* out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, out.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, 2 * dimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 2 * shift_in1) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 2 * shift_in2) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 2 * shift_out) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, multiplyer) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- if(!normilize) return true; //--- setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, dimension) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindow * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, kv, iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), kv_g, inp, GetPointer(Temp), iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindow, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); KV_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle) || !KV_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronMLMHAttentionOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionMLKV *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < ((iLayers + iLayersToOneKV - 1) / iLayersToOneKV); l++) { if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.KV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? KV_Weights.At(l * 2 + 1) : KV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = fmax(window_kv, 1); iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnitsKV; //Size of KV tensor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindowKV + 1) * iWindowKey * iHeadsKV; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnitsKV * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of our tensore uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, kv, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, SecondActivation)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(i > 0) out_grad = temp; } //--- if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(0), KV_Tensors.At(1), SecondInput, SecondGradient, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; for(int i = 1; i < KV_Tensors.Total() / 2; i++) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i * (optimization == SGD ? 2 : 3)), KV_Tensors.At(i * 2 + 1), SecondInput, GetPointer(Temp), iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindowKV, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_key, uint heads, uint heads_kv, uint units_count, uint pam_layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_in * units_count, optimization_type, batch)) ReturnFalse; //--- iWindowIn = window_in; iWindowKey = MathMax(window_key, 1); iHeads = MathMax(heads, 1); iHeadsKV = MathMax(heads_kv, 1); iCount = units_count; iPAMLayers = MathMax(pam_layers, 2); //--- caS3.Clear(); caQuery.Clear(); caKV.Clear(); caScore.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3) ReturnFalse; if(!s3.Init(0, l, OpenCL, iWindowIn, iCount, optimization, iBatch) || !caS3.Add(s3)) ReturnFalse; s3.SetActivationFunction(None); //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, iWindowKey * iHeads, iCount, optimization, iBatch) || !caQuery.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- KV conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 2 * iWindowKey * iHeadsKV, iCount, optimization, iBatch) || !caKV.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- MH Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowKey * iHeadsKV, iWindowKey * iHeadsKV, iWindowIn, iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!cFF1.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 4 * iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF1.SetActivationFunction(LReLU); if(!cFF2.Init(0, 0, OpenCL, 4 * iWindowIn, 4 * iWindowIn, iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF2.SetActivationFunction(None); if(!SetGradient(cFF2.getGradient())) ReturnFalse; //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionOut(CBufferFloat* q, CBufferFloat* kv, int scores, CBufferFloat* out, int window) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount/*Q units*/, iCount/*K units*/, iHeads}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_q, q.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_kv, kv.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_score, scores) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_out, out.GetIndex()) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_dimension, (int)iWindowKey) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_window, window) kernelExecuteLoc(def_k_MH2PyrAttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* q_g, CBufferFloat* kv, CBufferFloat* kv_g, int scores, CBufferFloat* gradient) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, q_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kv_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, gradient.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iCount) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!gradient.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; CBufferFloat *q = NULL; CBufferFloat *kv = NULL; //--- for(uint l = 0; l < iPAMLayers; l++) { //--- Mix current = caS3.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; //--- Query current = caQuery.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; q = current.getOutput(); //--- Key and Value current = caKV.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; kv = current.getOutput(); //--- PAM current = caAttentionOut.At(l); if(!current || !AttentionOut(q, kv, caScore.At(l), current.getOutput(), iPAMLayers - l)) ReturnFalse; prev = current; //--- W0 current = caW0.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; } //--- Residual current = caW0.At(iPAMLayers); if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), current.getOutput(), iWindowIn, true)) ReturnFalse; //---FeedForward if(!cFF1.FeedForward(current.AsObject()) || !cFF2.FeedForward(cFF1.AsObject()) ) ReturnFalse; //--- Residual if(!SumAndNormalize(current.getOutput(), cFF2.getOutput(), getOutput(), iWindowIn, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; CNeuronBaseOCL *q = NULL; CNeuronBaseOCL *kv = NULL; //--- FeedForward current = caW0.At(iPAMLayers); if(!current || !cFF1.CalcHiddenGradients(cFF2.AsObject()) || !current.CalcHiddenGradients(cFF1.AsObject()) ) ReturnFalse; next = current; //--- Residual current = caW0.At(iPAMLayers - 1); if(!SumAndNormalize(getGradient(), next.getGradient(), current.getGradient(), iWindowIn, false)) ReturnFalse; CBufferFloat *residual = next.getGradient(); next = current; //--- for(int l = int(iPAMLayers - 1); l >= 0; l--) { //--- W0 current = caAttentionOut.At(l); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; //--- MH Attention q = caQuery.At(l); kv = caKV.At(l); if(!q || !kv || !AttentionInsideGradients(q.getOutput(), q.getGradient(), kv.getOutput(), kv.getGradient(), caScore.At(l), current.getGradient()) ) ReturnFalse; //--- Query current = caS3.At(l); if(!current || !current.CalcHiddenGradients(q.AsObject()) || !Concat(current.getGradient(), current.getGradient(), residual, iWindowIn, 0, iCount) ) ReturnFalse; //--- Key and Value if(!current || !current.CalcHiddenGradients(kv.AsObject()) || !SumAndNormalize(current.getGradient(), residual, current.getGradient(), iWindowIn, false) ) ReturnFalse; next = current; //--- S3 current = (l == 0 ? prevLayer : caW0.At(l - 1)); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; next = current; } //--- Residual current = caW0.At(iPAMLayers - 1); if(!DeActivation(prevLayer.getOutput(), residual, current.getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), residual, prevLayer.getGradient(), iWindowIn, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 current = caS3.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Query prev = current; current = caQuery.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Key and Value current = caKV.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- W0 prev = caAttentionOut.At(l); current = caW0.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current; } //--- FeedForward prev = caW0.At(iPAMLayers); if(!cFF1.UpdateInputWeights(prev) || !cFF2.UpdateInputWeights(cFF1.AsObject()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::ArraySetOpenCL(CArrayObj * array, COpenCLMy * obj) { if(!array || array.Total() <= 0) return; //--- CNeuronBaseOCL *neuron = NULL; for(int l = 0; l < array.Total(); l++) { neuron = array.At(l); if(!neuron) continue; neuron.SetOpenCL(obj); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); ArraySetOpenCL(GetPointer(caS3), OpenCL); ArraySetOpenCL(GetPointer(caQuery), OpenCL); ArraySetOpenCL(GetPointer(caKV), OpenCL); ArraySetOpenCL(GetPointer(caAttentionOut), OpenCL); ArraySetOpenCL(GetPointer(caW0), OpenCL); cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); //--- Score for(uint l = 0; l < iPAMLayers; l++) { int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) return; if(!caScore.Add(temp)) return; } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSPyrAttentionOCL *Source = source; if(iPAMLayers != Source.iPAMLayers) ReturnFalse; CNeuronBaseOCL *s = NULL; CNeuronBaseOCL *c = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 c = caS3.At(l); s = Source.caS3.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Query c = caQuery.At(l); s = Source.caQuery.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Key and Value c = caKV.At(l); s = Source.caKV.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- W0 c = caW0.At(l); s = Source.caW0.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; } //--- FeedForward if(!cFF1.WeightsUpdate(Source.cFF1.AsObject(), tau) || !cFF2.WeightsUpdate(Source.cFF2.AsObject(), tau) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iWindowIn)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeadsKV)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPAMLayers)) < INT_VALUE) ReturnFalse; //--- Save objects CNeuronBaseOCL *neuron = NULL; for(uint l = 0; l < iPAMLayers; l++) { neuron = caS3.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caQuery.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caKV.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caW0.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; } //--- if(!cFF1.Save(file_handle) || !cFF2.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Load(const int file_handle) { //--- Clear Arrays if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); caS3.Clear(); caQuery.Clear(); caKV.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iWindowIn = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeadsKV = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iPAMLayers = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; //--- Load objects CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, l, OpenCL, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, s3) || !caS3.Add(s3) ) ReturnFalse; //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caQuery.Add(conv) ) ReturnFalse; //--- Key and Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caKV.Add(conv) ) ReturnFalse; //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caW0.Add(conv) ) ReturnFalse; } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!LoadInsideLayer(file_handle, cFF1.AsObject()) || !LoadInsideLayer(file_handle, cFF2.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables; //--- uint num_q = iWindowKey * iHeads * iUnits * iVariables; //Size of Q tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariables; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = (iWindow * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint scores = iUnits * iUnits * iHeads * iVariables; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //Size of multi-heads self-attention uint out = iWindow * iUnits * iVariables; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariables)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, k, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, v, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariables)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMVMHAttentionMLKV *Source = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), Source.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE) || !FileWriteInteger(file_handle, iVariables, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle)) ReturnFalse; if(!K_Tensors.Save(file_handle) || !K_Weights.Save(file_handle)) ReturnFalse; if(!V_Tensors.Save(file_handle) || !V_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMVMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); K_Tensors.SetOpenCL(OpenCL); K_Weights.SetOpenCL(OpenCL); V_Tensors.SetOpenCL(OpenCL); V_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, uint variables_q, uint variables_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables_q, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = window_kv; iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables_q; iVariablesKV = variables_kv; //--- uint num_q = iWindowKey * iHeads * iUnits * iVariables; //Size of Q tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariablesKV; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = (iWindowKV * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint scores = iUnits * iUnitsKV * iHeads * iVariables; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //Size of multi-heads self-attention uint out = iWindow * iUnits * iVariables; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionOut(CBufferFloat* q, CBufferFloat* kv, CBufferFloat* scores, CBufferFloat* out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariablesKV)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, k, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, v, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariablesKV, iWindowKey * iHeadsKV * iVariablesKV, iUnitsKV)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //out.BufferRead(); //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat* q, CBufferFloat* qg, CBufferFloat* kv, CBufferFloat* kvg, CBufferFloat* scores, CBufferFloat* outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariablesKV)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) #ifdef _DEBUG if(!outg.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); if(!SecondGradient.BufferInit(SecondGradient.Total(), 0) || !SecondGradient.BufferWrite()) ReturnFalse; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- Deactivation7 if(SecondActivation != None && !DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(NeuronOCL.Activation() != None && !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMVMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iVariablesKV), INT_VALUE)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); iVariablesKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronMVCrossAttentionMLKV::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = FF_Weights.At((iLayers - 1) * (optimization == SGD ? 6 : 9) + 2); return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); if(!cPatching.Init(0, 0, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; cPatching.SetActivationFunction(None); if(!cCIPosition.Init(0, 1, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCIPosition.SetActivationFunction(None); if(!cCMPosition.Init(0, 2, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCMPosition.SetActivationFunction(None); if(!cChanelIndependentAttention.Init(0, 3, OpenCL, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, variables, optimization, iBatch)) ReturnFalse; cChanelIndependentAttention.SetActivationFunction(None); if(!cChanelMixAttention.Init(0, 4, OpenCL, window * variables, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, optimization, iBatch)) ReturnFalse; cChanelMixAttention.SetActivationFunction(None); if(!cGlobalInjectionAttention.Init(0, 5, OpenCL, window, window_key, heads, window * variables, heads_kv, units_count, units_count, layers, layers_to_one_kv, variables, 1, optimization, iBatch)) ReturnFalse; cGlobalInjectionAttention.SetActivationFunction(None); if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cPatching.FeedForward(NeuronOCL)) ReturnFalse; if(!cCIPosition.FeedForward(cPatching.AsObject()) || !cCMPosition.FeedForward(cPatching.AsObject()) ) ReturnFalse; if(!cChanelIndependentAttention.FeedForward(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.FeedForward(cCMPosition.AsObject())) ReturnFalse; if(!cGlobalInjectionAttention.FeedForward(cCIPosition.AsObject(), cCMPosition.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cChanelIndependentAttention.CalcHiddenGradients(cGlobalInjectionAttention.AsObject(), cChanelMixAttention.getOutput(), cChanelMixAttention.getGradient(), (ENUM_ACTIVATION)cChanelMixAttention.Activation()) ) ReturnFalse; //--- if(!cCMPosition.CalcHiddenGradients(cChanelMixAttention.AsObject())) ReturnFalse; if(!cCIPosition.CalcHiddenGradients(cChanelIndependentAttention.AsObject())) ReturnFalse; //--- if(!cPatching.CalcHiddenGradients(cCIPosition.AsObject())) ReturnFalse; if(!SumAndNormalize(cPatching.getGradient(), cPatching.getGradient(), GetPointer(cTemp), 1, false)) ReturnFalse; if(!cPatching.CalcHiddenGradients(cCMPosition.AsObject())) ReturnFalse; if(!SumAndNormalize(cPatching.getGradient(), GetPointer(cTemp), cPatching.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cPatching.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cCIPosition.UpdateInputWeights(cPatching.AsObject()) || !cCMPosition.UpdateInputWeights(cPatching.AsObject()) ) ReturnFalse; //--- if(!cChanelIndependentAttention.UpdateInputWeights(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.UpdateInputWeights(cCMPosition.AsObject())) ReturnFalse; //--- if(!cGlobalInjectionAttention.UpdateInputWeights(cChanelIndependentAttention.AsObject(), cChanelMixAttention.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!cCIPosition.Save(file_handle)) ReturnFalse; if(!cCMPosition.Save(file_handle)) ReturnFalse; if(!cChanelIndependentAttention.Save(file_handle)) ReturnFalse; if(!cChanelMixAttention.Save(file_handle)) ReturnFalse; if(!cGlobalInjectionAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Load(const int file_handle) { cTemp.BufferFree(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCIPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCMPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelIndependentAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelMixAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGlobalInjectionAttention.AsObject())) ReturnFalse; //--- if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInjectTST::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cPatching.SetOpenCL(OpenCL); cCIPosition.SetOpenCL(OpenCL); cCMPosition.SetOpenCL(OpenCL); cChanelIndependentAttention.SetOpenCL(OpenCL); cChanelMixAttention.SetOpenCL(OpenCL); cGlobalInjectionAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronInjectTST::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = cGlobalInjectionAttention.getWeights(); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * dimension, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iWindow, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, Output.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindow, iCount, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, NeuronOCL.getGradientIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, Gradient.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, 8, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(8 * units_count_kv, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cSceneSpecificKnowledge.Init(0, 1, OpenCL, 8 * units_count_kv, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::feedForward(CNeuronBaseOCL* NeuronOCL) { if(bTrain && !cSceneSpecificKnowledge.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(NeuronOCL, cSceneSpecificKnowledge.getOutput(), cSceneSpecificKnowledge.getGradient(), (ENUM_ACTIVATION)cSceneSpecificKnowledge.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; if(!cSceneSpecificKnowledge.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cSceneSpecificKnowledge.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSceneSpecificKnowledge.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSceneSpecific::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cSceneSpecificKnowledge.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //Size of KV tensor uint q_weights = (iWindow * iHeads) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindow * iHeadsKV) * iWindowKey; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- CNeuronBaseOCL *base = NULL; CNeuronSceneSpecific *ss = NULL; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } if(i % iLayersToOneKV == 0) { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init((q_weights + kv_weights), cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init((q_weights + kv_weights), cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } else { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init(q_weights, cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(q_weights, cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax((num_q + num_kv)*iWindow, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.FeedForward(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.FeedForward(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(bTrain && !sa.FeedForward(cSceneAgnostic[i * 2])) ReturnFalse; CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); //--- weights q_weights = QKV_Weights[i * 2]; q = QKV_Tensors[i * 2]; if(i % iLayersToOneKV == 0) { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), GetPointer(Temp))) ReturnFalse; kv_weights = KV_Weights[(i / iLayersToOneKV) * 2]; kv = KV_Tensors[(i / iLayersToOneKV) * 2]; if(IsStopped() || !DeConcat(q_weights, kv_weights, GetPointer(Temp), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(IsStopped() || !MatMul(inputs, kv_weights, kv, iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; } else { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), q_weights)) ReturnFalse; } if(IsStopped() || !MatMul(inputs, q_weights, q, iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors[i * 2]; CBufferFloat *out = AO_Tensors[i * 2]; if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors[i * 6]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9)], out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormalize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors[i * 6 + 1]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors[i * 6 + 2]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormalize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors[KV_Tensors.Total() - 1]; CNeuronBaseOCL *ss = NULL, *sa = NULL; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors[(i / iLayersToOneKV) * 2 + 1]; //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], out_grad, FF_Tensors[i * 6 + 1], FF_Tensors[i * 6 + 4], 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors[i * 6 + 3]; if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], FF_Tensors[i * 6 + 4], FF_Tensors[i * 6], temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9)], out_grad, AO_Tensors[i * 2], AO_Tensors[i * 2 + 1], iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value sa = cSceneAgnostic[i * 2 + 1]; ss = cSceneSpecific[i * 2 + 1]; if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[(i / iLayersToOneKV) * 2], kv_g, S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2], GetPointer(Temp), S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; if(IsStopped() || !SumAndNormalize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !MatMulGrad(inp, temp, QKV_Weights[i * 2], QKV_Weights[i * 2 + 1], QKV_Tensors[i * 2 + 1], iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormalize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !MatMulGrad(inp, GetPointer(Temp), KV_Weights[i / iLayersToOneKV * 2], KV_Weights[i / iLayersToOneKV * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2 + 1], iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; if(IsStopped() || !SumAndNormalize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(QKV_Weights[i * 2 + 1], KV_Weights[i / iLayersToOneKV * 2 + 1], ss.getGradient(), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), ss.getGradient(), ss.Activation(), sa.Activation())) ReturnFalse; } else { if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), QKV_Weights[i * 2 + 1], ss.Activation(), sa.Activation())) ReturnFalse; } if(i > 0) out_grad = temp; } //--- CBufferFloat *inp_grad = prevLayer.getGradient(); if(!prevLayer.SetGradient(GetPointer(Temp), false)) ReturnFalse; for(int i = int(iLayers - 2); (i >= 0 && !IsStopped()); i -= 2) { ss = cSceneSpecific[i]; if(IsStopped() || !ss.CalcHiddenGradients(cSceneSpecific[i + 1])) ReturnFalse; if(IsStopped() || !prevLayer.CalcHiddenGradients(ss, NULL)) ReturnFalse; if(IsStopped() || !SumAndNormalize(prevLayer.getGradient(), inp_grad, inp_grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!prevLayer.SetGradient(inp_grad, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.UpdateInputWeights(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.UpdateInputWeights(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(!sa.UpdateInputWeights(cSceneAgnostic[i * 2])) ReturnFalse; //--- Attention out if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), FF_Tensors.At(i * 6 + 3), AO_Tensors.At(i * 2), (optimization == SGD ? FF_Weights.At(i * 6 + 3) : FF_Weights.At(i * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- Feed Forward if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), (optimization == SGD ? FF_Weights.At(i * 6 + 4) : FF_Weights.At(i * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(i * 6 + 5), FF_Tensors.At(i * 6 + 1), (optimization == SGD ? FF_Weights.At(i * 6 + 5) : FF_Weights.At(i * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cSceneSpecific.Save(file_handle)) ReturnFalse; if(!cSceneAgnostic.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Load(file_handle)) ReturnFalse; if(!cSceneSpecific.Load(file_handle)) ReturnFalse; if(!cSceneAgnostic.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHSceneConditionAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionMLKV::SetOpenCL(obj); cSceneSpecific.SetOpenCL(OpenCL); cSceneAgnostic.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; //--- Init SuperPoints for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } } //--- CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 6, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 7, OpenCL, iSPWindow, iSPWindow, 2 * iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKeyValue.Add(conv)) ReturnFalse; } //--- Mask conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 8, OpenCL, iSPWindow, iSPWindow, iUnits * iHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cMask.Add(conv)) ReturnFalse; transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, l * 14 + 9, OpenCL, iSPUnits, iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMask.Add(transp)) ReturnFalse; //--- MH Cross Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 10, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; //--- Cross Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 11, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 12, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 13, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 14, OpenCL, iWindow, iWindow, 2 * iWindowKey * iSPHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKeyValue.Add(conv)) ReturnFalse; } //--- MH Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 15, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; //--- Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 16, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 17, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- FeedForward conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 18, OpenCL, iWindow, iWindow, iWindow * 4, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 19, OpenCL, iWindow * 4, iWindow * 4, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 20, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(l == (iLayers - 1)) { if(!SetGradient(conv.getGradient())) ReturnFalse; if(!SetOutput(base.getOutput())) ReturnFalse; } } //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* kv, const int scores, CNeuronBaseOCL* out, CNeuronBaseOCL* mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f ) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHMaskAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_mask_at_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_at_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_at_score, scores) setBuffer(kernel, def_k_mask_at_out, out.getOutputIndex()) setBuffer(kernel, def_k_mask_at_mask, (!mask ? scores : mask.getOutputIndex())) setArgument(kernel, def_k_mask_at_dimension, dimension) setArgument(kernel, def_k_mask_at_heads_kv, heads_kv) setArgument(kernel, def_k_mask_at_mask_level, (!mask ? 0 : mask_level)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* kv, const int scores, CNeuronBaseOCL* out, CNeuronBaseOCL* mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHMaskAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_mask_atg_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_kv_g, kv.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_scores, scores) setBuffer(kernel, def_k_mask_atg_gradient, out.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_mask, (!mask ? scores : mask.getOutputIndex())) setBuffer(kernel, def_k_mask_atg_mask_g, (!mask ? scores : mask.getGradientIndex())) setArgument(kernel, def_k_mask_atg_kunits, units_kv) setArgument(kernel, def_k_mask_atg_heads_kv, heads_kv) setArgument(kernel, def_k_mask_atg_mask_level, (!mask ? 0 : mask_level)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!kv.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.FeedForward(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.FeedForward(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; neuron = cMask[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cMask[l * 2])) ReturnFalse; if(!AttentionOut(q, kv_cross, cScores[l * 2], cMHCrossAttentionOut[l], neuron, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.FeedForward(inputs)) ReturnFalse; } if(!AttentionOut(q, kv_self, cScores[l * 2 + 1], cMHSelfAttentionOut[l], NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 1]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.FeedForward(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cFeedForward[l * 2])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 2]; if(!inputs || !SumAndNormalize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *superpoints = cSuperPoints[cSuperPoints.Total() - 1]; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = cSPKeyValue[cSPKeyValue.Total() - 1], *kv_self = cQKeyValue[cQKeyValue.Total() - 1]; //--- if(!cTempSP.Fill(0) || !cTempSelfKV.Fill(0) || !cTempCrossKV.Fill(0)) ReturnFalse; for(int l = int(iLayers - 1); l >= 0; l--) { //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; neuron = cResidual[l * 3 + 1]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; if(!SumAndNormalize(((CNeuronBaseOCL*)cResidual[l * 3 + 2]).getGradient(), neuron.getGradient(), ((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention neuron = cMHSelfAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cSelfAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 3]; if(((l + 1) % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !cTempSelfKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_self, cScores[l * 2 + 1], neuron, NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(kv_self.getGradient(), GetPointer(cTempSelfKV), kv_self.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(kv_self.getGradient(), GetPointer(cTempSelfKV), GetPointer(cTempSelfKV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } inputs = cResidual[l * 3]; if(!inputs || !inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if((l % iLayersSP) == 0) { CBufferFloat *temp = inputs.getGradient(); if(!inputs.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!inputs.CalcHiddenGradients(kv_self, NULL)) ReturnFalse; if(!SumAndNormalize(temp, GetPointer(cTempQ), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!inputs.SetGradient(temp, false)) ReturnFalse; } if(!SumAndNormalize(((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross Attention neuron = cMHCrossAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cCrossAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 2]; if(((l + 1) % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !cTempCrossKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_cross, cScores[l * 2], neuron, cMask[l * 2 + 1], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; inputs = (l == 0 ? cQuery[1] : cResidual[l * 3 - 1]); if(!inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if(!SumAndNormalize(inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), inputs.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((l % iLayersSP) == 0) { if(!superpoints.CalcHiddenGradients(kv_cross, NULL)) ReturnFalse; if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cMask[l * 2 + 1]) || !DeActivation(neuron.getOutput(), neuron.getGradient(), neuron.getGradient(), neuron.Activation())) ReturnFalse; if(!superpoints.CalcHiddenGradients(neuron, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), superpoints.getGradient(), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- for(int l = cSuperPoints.Total() - 2; l >= 0; l--) { superpoints = cSuperPoints[l]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[l + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(superpoints, NULL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.UpdateInputWeights(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.UpdateInputWeights(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.UpdateInputWeights(inputs)) ReturnFalse; } neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.UpdateInputWeights(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPFormer::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL);; cQuery.SetOpenCL(OpenCL); cSPKeyValue.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cQKeyValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::CreateBuffers(void) { for(uint l = 0; l < iLayers; l++) { //--- Cross Attention int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); vector temp = vector::Zeros(s); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; if(!OpenCL.BufferFromVector(s, temp, CL_MEM_READ_WRITE)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iSPWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempSelfKV.BufferInit(2 * iWindowKey * iUnits * iHeads, 0) || !cTempSelfKV.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossKV.BufferInit(2 * iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossKV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cSPKeyValue.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cQKeyValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cSPKeyValue.Load(file_handle)) ReturnFalse; if(!cMask.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cQKeyValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(uint i = 0; i < iLayers; i++) { neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; } //--- if(Output != neuron.getOutput()) if(!SetOutput(neuron.getOutput())) ReturnFalse; //--- if(!CreateBuffers()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; buf = base.getWeights(); if(!buf || !buf.BufferInit(buf.Total(), 0) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; CNeuronLearnabledPE *pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 2, OpenCL, base.Neurons(), optimization, iBatch)) ReturnFalse; if(!cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 3, OpenCL, pe.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetOutput(pe.GetPE())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 4; for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } layer_id++; } CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMAFT::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Score int s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross Attention s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int pos_bias, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const bool use_pos_bias) { if(!OpenCL || !q || !k || !v || !out || scores < 0 || (use_pos_bias && pos_bias < 0)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, (use_pos_bias ? pos_bias : scores)) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, int(use_pos_bias)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!v.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::CalcPositionBias(CBufferFloat* pos_q, CBufferFloat* pos_k, const int pos_bias, const int units, const int units_kv, const int dimension) { if(!OpenCL || !pos_q || !pos_k || pos_bias < 0) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {units/*Q units*/, units_kv}; int kernel = def_k_CalcPositionBias; //--- ResetLastError(); setBuffer(kernel, def_k_cpb_data1, pos_q.GetIndex()) setBuffer(kernel, def_k_cpb_data2, pos_k.GetIndex()) setBuffer(kernel, def_k_cpb_result, pos_bias) setArgument(kernel, def_k_cpb_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG vector temp; if(!OpenCL.BufferToVector(pos_bias, temp)) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.FeedForward(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 2], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; value = cQPosition[l * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), inputs.getOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Calc Position bias if(!CalcPositionBias(value.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 2 + 1], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormalize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cCrossAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 4]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2 + 1], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 3]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(l == 0) base = cQuery[2]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), GetPointer(cTempQ), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), GetPointer(cTempQ), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormalize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- Qeury query = cQuery[1]; if(!query || !query.CalcHiddenGradients(cQuery[2])) ReturnFalse; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[2]).AddPEGradient(base.getGradient()))) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } //--- From Value superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.UpdateInputWeights(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cQPosition.Save(file_handle)) ReturnFalse; if(!cQKey.Save(file_handle)) ReturnFalse; if(!cQValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSPKey.Save(file_handle)) ReturnFalse; if(!cSPValue.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cQPosition.Load(file_handle)) ReturnFalse; if(!cQKey.Load(file_handle)) ReturnFalse; if(!cQValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSPKey.Load(file_handle)) ReturnFalse; if(!cSPValue.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; for(uint i = 0; i < iLayers; i++) { CNeuronBaseOCL *neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; neuron = cQPosition[i * 2 + 1]; if(!neuron || !(((CNeuronBaseOCL*)cQPosition[(i + 1) * 2]).SetGradient(neuron.getGradient())) ) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[cQPosition.Total() - 1]).SetGradientIndex(getGradientIndex()))) ReturnFalse; if(Type() == defNeuronMAFT) { if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[2]).GetPE()))) ReturnFalse; } //--- if(!CreateBuffers()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint ref_size, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- CNeuronBaseOCL *base = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronConvOCL *conv = NULL; CNeuronLearnabledPE *pe = NULL; //--- Init Querys cQuery.Clear(); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 0, OpenCL, iSPUnits, iSPWindow, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iSPUnits, iSPUnits, iUnits, 1, iSPWindow, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 2, OpenCL, iSPWindow, iUnits, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 3, OpenCL, iSPWindow, iSPWindow, iWindow, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch) || !cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 5, OpenCL, pe.Neurons(), optimization, iBatch) || !base.SetOutput(pe.GetPE()) || !cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 6; cSuperPoints.Clear(); for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual || !residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch) || !cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); } layer_id++; } conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch) || !cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Reference cReference.Clear(); base = new CNeuronBaseOCL(); if(!base || !base.Init(iWindow * iUnits, layer_id, OpenCL, ref_size, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; base.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, base.Neurons(), optimization, iBatch) || !cReference.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers cQKey.Clear(); cQValue.Clear(); cSPKey.Clear(); cSPValue.Clear(); cSelfAttentionOut.Clear(); cCrossAttentionOut.Clear(); cMHCrossAttentionOut.Clear(); cMHSelfAttentionOut.Clear(); cMHRefAttentionOut.Clear(); cRefAttentionOut.Clear(); cRefKey.Clear(); cRefValue.Clear(); cResidual.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cResidual.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Reference Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHRefAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cRefAttentionOut.Add(conv)) ReturnFalse; layer_id++; if(!conv.SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[cSelfAttentionOut.Total() - 1]).getGradient(), true)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Score int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Reference Cross-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).FeedForward(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; if(!SecondInput) ReturnFalse; if(reference.getOutput() != SecondInput) if(!reference.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).FeedForward(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Calc Position bias cross = cQPosition[l * 2]; if(!cross || !CalcPositionBias(cross.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 3], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormalize(cross.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 1], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.FeedForward(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.FeedForward(reference)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 2], cMHRefAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.FeedForward(cMHRefAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormalize(cross.getOutput(), self.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormalize(inputs.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormalize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormalize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormalize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGRES::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } //--- cScores.Clear(); cPositionBias.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cReference.SetOpenCL(OpenCL); cRefKey.SetOpenCL(OpenCL); cRefValue.SetOpenCL(OpenCL); cMHRefAttentionOut.SetOpenCL(OpenCL); cRefAttentionOut.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Reference Cross-Attention base = cMHRefAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cRefAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 7]; key = cRefKey[l]; value = cRefValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 3 + 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; value = cCrossAttentionOut[l]; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cSelfAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 6]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2 + 1], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormalize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormalize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 3 + 5]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormalize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } if(l == 0) base = cQuery[4]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual if(!SumAndNormalize(base.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = base; } //--- Qeury query = cQuery[3]; if(!query || !query.CalcHiddenGradients(cQuery[4])) ReturnFalse; base = cQPosition[0]; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[4]).AddPEGradient(base.getGradient()))) ReturnFalse; if(!DiversityLoss(query, iUnits, iWindow, true)) ReturnFalse; for(int i = 2; i >= 0; i--) { query = cQuery[i]; if(!query || !query.CalcHiddenGradients(cQuery[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(query, NULL)) ReturnFalse; CBufferFloat *inputs_gr = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(query.getGradient(), false)) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; //--- From Value CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormalize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; if(!DiversityLoss(superpoints, iSPUnits, iSPWindow, true)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(inputs_gr, false)) ReturnFalse; //--- Reference base = cReference[0]; if(base.getGradient() != SecondGradient) { if(!base.SetGradient(SecondGradient)) ReturnFalse; base.SetActivationFunction(SecondActivation); } base = cReference[2]; if(!base || !base.CalcHiddenGradients(cRefKey[0])) ReturnFalse; inputs_gr = base.getGradient(); if(!base.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[0])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; for(uint i = 1; i < iLayers; i++) { if(!base.CalcHiddenGradients(cRefKey[i])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[i])) ReturnFalse; if(!SumAndNormalize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!base.SetGradient(inputs_gr, false)) ReturnFalse; base = cReference[1]; if(!base.CalcHiddenGradients(cReference[2])) ReturnFalse; if(!DiversityLoss(base, iUnits, iWindow, true)) ReturnFalse; base = cReference[0]; if(!base.CalcHiddenGradients(cReference[1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).UpdateInputWeights(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).UpdateInputWeights(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.UpdateInputWeights(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.UpdateInputWeights(reference)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.UpdateInputWeights(cMHRefAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Save(const int file_handle) { if(!CNeuronMAFT::Save(file_handle)) ReturnFalse; //--- if(!cReference.Save(file_handle)) ReturnFalse; if(!cRefKey.Save(file_handle)) ReturnFalse; if(!cRefValue.Save(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Save(file_handle)) ReturnFalse; if(!cRefAttentionOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Load(const int file_handle) { if(!CNeuronMAFT::Load(file_handle)) ReturnFalse; //--- if(!cReference.Load(file_handle)) ReturnFalse; if(!cRefKey.Load(file_handle)) ReturnFalse; if(!cRefValue.Load(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Load(file_handle)) ReturnFalse; if(!cRefAttentionOut.Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < iLayers; i++) { if(!(((CNeuronBaseOCL*)cRefAttentionOut[i]).SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[i]).getGradient(), true))) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[4]).GetPE()))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_kv, uint heads_kv, uint units_count_kv, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronGEGWA *temp = new CNeuronGEGWA(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, (units_count + 1) / 2, window_kv, heads_kv, units_count_kv, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronMLCrossAttentionMLKV *temp = new CNeuronMLCrossAttentionMLKV(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, window_kv, heads_kv, (units_count + 1) / 2, units_count_kv, layers, 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(!cResidual.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cCrossAttention.Init(0, 6, OpenCL, window, window_key, heads, window_kv, heads_kv, units_count, units_count_kv, layers, 1, optimization, iBatch)) ReturnFalse; if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, false)) ReturnFalse; if(!cCrossAttention.FeedForward(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::calcInputGradients(CNeuronBaseOCL* prevLayer, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; if(!cResidual.CalcHiddenGradients(cCrossAttention.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(bAddNeckGradient) { CBufferFloat *temp = cNeck.getGradient(); if(!cNeck.SetGradient(cMergeSplit[0].getGradient(), false)) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormalize(cNeck.getGradient(), temp, temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cNeck.SetGradient(temp, false)) ReturnFalse; } else if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject(), SecondInput, GetPointer(cTemp), SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, GetPointer(cTemp), SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), GetPointer(cTemp), cMergeSplit[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; if(!cCrossAttention.UpdateInputWeights(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Save(const int file_handle) { if(!CNeuronUShapeAttention::Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronGEGWA: cNeck = new CNeuronGEGWA(); if(!cNeck) ReturnFalse; break; case defNeuronMLCrossAttentionMLKV: cNeck = new CNeuronMLCrossAttentionMLKV(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cResidual.Init(0, 0, OpenCL, cCrossAttention.Neurons(), optimization, iBatch)) ReturnFalse; if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGEGWA::SetOpenCL(COpenCLMy * obj) { CNeuronUShapeAttention::SetOpenCL(obj); cResidual.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL* CNeuronGEGWA::GetInsideLayer(const int layer) const { if(layer < 0) return NULL; //--- if(layer == 0) return cNeck; //--- if(!cNeck || cNeck.Type() != Type()) return NULL; //--- CNeuronGEGWA* temp = cNeck; return temp.GetInsideLayer(layer - 1); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, window, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(window * units_count, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cPrimitives.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(bTrain && !cPrimitives.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(cPrimitives.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!DiversityLoss(cPrimitives.AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!cPrimitives.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cPrimitives.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrimitives.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLPC::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cPrimitives.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionOut(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, scores) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!out.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionInsideGradients(CNeuronBaseOCL* q, CNeuronBaseOCL* k, CNeuronBaseOCL* v, const int scores, CNeuronBaseOCL* out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!v.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint prim_window, uint window_key, uint prim_units, uint prim_heads, uint cont_window, uint cont_units, uint cont_heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, cont_window * cont_units, optimization_type, batch)) ReturnFalse; //--- iPrimWindow = prim_window; iPrimUnits = prim_units; iPrimHeads = prim_heads; iContWindow = cont_window; iContUnits = cont_units; iContHeads = cont_heads; iWindowKey = window_key; //--- cQuery.Clear(); cKey.Clear(); cValue.Clear(); cMHAttentionOut.Clear(); cAttentionOut.Clear(); cResidual.Clear(); cFeedForward.Clear(); //--- CNeuronBaseOCL *neuron = NULL; CNeuronConvOCL *conv = NULL; //--- Primitives Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 0, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 2, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 4, OpenCL, iPrimHeads * iWindowKey, iPrimHeads * iWindowKey, iPrimWindow, iPrimUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 6, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 7, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 8, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 10, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 12, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 13, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 14, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 16, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 18, OpenCL, iContWindow, iContWindow, 4 * iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 19, OpenCL, 4 * iContWindow, 4 * iContWindow, iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; //--- if(!SetGradient(conv.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::feedForward(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Primitives) || !k.FeedForward(Primitives) || !v.FeedForward(Primitives) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[0]) ) ReturnFalse; v = cResidual[0]; if(!v || !SumAndNormalize(Primitives.getOutput(), neuron.getOutput(), v.getOutput(), iPrimWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Context) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[1]) ) ReturnFalse; v = cResidual[1]; if(!v || !SumAndNormalize(Context.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(neuron) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.FeedForward(cMHAttentionOut[2]) ) ReturnFalse; v = cResidual[2]; if(!v || !SumAndNormalize(q.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.FeedForward(neuron) || !k.FeedForward(q) || !SumAndNormalize(neuron.getOutput(), k.getOutput(), Output, iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::calcInputGradients(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { if(!Primitives || !Context) ReturnFalse; //--- CNeuronBaseOCL *neuron = cResidual[2], *q = cFeedForward[1], *k = cAttentionOut[2], *v = NULL, *residual = GetPointer(this); //--- Feed Forward if(!neuron || !q || !k || !q.CalcHiddenGradients(cFeedForward[1]) || !neuron.CalcHiddenGradients(q, NULL) || !SumAndNormalize(neuron.getGradient(), residual.getGradient(), k.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = k; //--- Context Self-Attention neuron = cMHAttentionOut[2]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iContUnits, iContHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[1]; if(!neuron || !neuron.CalcHiddenGradients(q, NULL) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), q.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = q; if(!neuron.CalcHiddenGradients(k, NULL) || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(v, NULL) || !SumAndNormalize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- Cross-Attention neuron = cMHAttentionOut[1]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[0]; if(!Context.CalcHiddenGradients(q, NULL) || !DeActivation(Context.getOutput(), residual.getGradient(), residual.getGradient(), Context.Activation()) || !SumAndNormalize(Context.getGradient(), residual.getGradient(), Context.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = cAttentionOut[0]; if(!neuron.CalcHiddenGradients(k, NULL)) ReturnFalse; CBufferFloat *temp = neuron.getGradient(); if(!neuron.SetGradient(residual.getGradient(), false)) ReturnFalse; if(!neuron.CalcHiddenGradients(v, NULL) || !SumAndNormalize(temp, neuron.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; //--- Primitives Self-Attention neuron = cMHAttentionOut[0]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[0]; k = cKey[0]; v = cValue[0]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; if(!Primitives.CalcHiddenGradients(q, NULL) || !DeActivation(Primitives.getOutput(), residual.getGradient(), residual.getGradient(), Primitives.Activation()) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(k, NULL) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(v, NULL) || !SumAndNormalize(Primitives.getGradient(), residual.getGradient(), Primitives.getGradient(), iPrimWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::updateInputWeights(CNeuronBaseOCL* Primitives, CNeuronBaseOCL* Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Primitives) || !k.UpdateInputWeights(Primitives) || !v.UpdateInputWeights(Primitives) ) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[0]) ) ReturnFalse; neuron = cResidual[0]; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Context) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[1]) ) ReturnFalse; neuron = cResidual[1]; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.UpdateInputWeights(cMHAttentionOut[2]) ) ReturnFalse; neuron = cResidual[2]; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(q) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::CreateBuffers(void) { //--- Primitives Self-Attention int s = int(iPrimUnits * iPrimUnits * iPrimHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross-Attention s = int(iContUnits * iPrimUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Content Self-Attention s = int(iContUnits * iContUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOCM::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } cScores.Clear(); CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save objects if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iPrimWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load objects if(!cQuery.Load(file_handle)) ReturnFalse; if(!cKey.Load(file_handle)) ReturnFalse; if(!cValue.Load(file_handle)) ReturnFalse; if(!cAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- Load constants iPrimWindow = (uint)FileReadInteger(file_handle); iPrimUnits = (uint)FileReadInteger(file_handle); iPrimHeads = (uint)FileReadInteger(file_handle); iContWindow = (uint)FileReadInteger(file_handle); iContUnits = (uint)FileReadInteger(file_handle); iContHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- CNeuronBaseOCL *neuron = NULL; cMHAttentionOut.Clear(); cResidual.Clear(); //--- Primitives Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, iPrimWindow * iPrimUnits, optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- neuron = cFeedForward[1]; if(!SetGradient(neuron.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint content_size, uint content_units, uint primitive_units, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * content_units, optimization_type, batch)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.Init(0, 0, OpenCL, window, window_key, heads, units_count, window, heads, (content_units + 3), 2, layers, optimization, iBatch)) ReturnFalse; cGEGWA.AddNeckGradient(true); //--- Content Encoder cContentEncoder.Clear(); cContentEncoder.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * content_units, 1, OpenCL, content_size, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 2, OpenCL, window * content_units, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, window * (content_units + 3), optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; //--- Background cBackGround.Clear(); cBackGround.SetOpenCL(OpenCL); neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * 3, 4, OpenCL, content_size, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, window * 3, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.Init(0, 6, OpenCL, window, window_key, heads, heads, primitive_units, content_units, 2, 1, optimization, iBatch)) ReturnFalse; //--- Decoder cDecoder.Clear(); cDecoder.SetOpenCL(OpenCL); CNeuronOCM *ocm = new CNeuronOCM(); if(!ocm || !ocm.Init(0, 7, OpenCL, window, window_key, units_count, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; for(uint i = 0; i < layers; i++) { neuron = cGEGWA.GetInsideLayer(i); ocm = new CNeuronOCM(); if(!ocm || !neuron || !ocm.Init(0, i + 8, OpenCL, window, window_key, neuron.Neurons() / window, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; } //--- Object Cluster Module if(!cOCM.Init(0, layers + 8, OpenCL, window, window_key, primitive_units, heads, window, content_units, heads, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput(), true) || !SetGradient(cOCM.getGradient(), true) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!SecondInput) ReturnFalse; //--- Context Encoder CNeuronBaseOCL *context = cContentEncoder[0]; if(context.getOutput() != SecondInput) { if(!context.SetOutput(SecondInput, true)) ReturnFalse; } int content_total = cContentEncoder.Total(); for(int i = 1; i < content_total - 1; i++) { context = cContentEncoder[i]; if(!context || !context.FeedForward(cContentEncoder[i - 1]) ) ReturnFalse; } //--- Background Encoder CNeuronBaseOCL *background = NULL; if(bTrain) { for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.FeedForward(cBackGround[i - 1]) ) ReturnFalse; } } else { background = cBackGround[cBackGround.Total() - 1]; if(!background) ReturnFalse; } CNeuronBaseOCL *neuron = cContentEncoder[content_total - 1]; if(!neuron || !Concat(context.getOutput(), background.getOutput(), neuron.getOutput(), context.Neurons(), background.Neurons(), 1)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.FeedForward(NeuronOCL, neuron.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.FeedForward(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.feedForward(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.feedForward(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.feedForward(decoder, context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *neuron = cContentEncoder[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) { if(!neuron.SetGradient(SecondGradient)) ReturnFalse; neuron.SetActivationFunction(SecondActivation); } //--- Object Cluster Module CNeuronBaseOCL *context = cContentEncoder[cContentEncoder.Total() - 2]; if(!cOCM.calcInputGradients(cDecoder[cDecoder.Total() - 1], context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = NULL; for(int i = cDecoder.Total() - 1; i > 0; i--) { decoder = cDecoder[i]; if(!decoder.calcInputGradients(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } decoder = cDecoder[0]; if(!decoder.calcInputGradients(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; //--- Linguistic Primitive Construction CBufferFloat *context_grad = context.getGradient(); if(!context.SetGradient(PrevOutput, false)) ReturnFalse; if(!context.CalcHiddenGradients(cLPC.AsObject()) || !SumAndNormalize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) ) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention neuron = cContentEncoder[cContentEncoder.Total() - 1]; if(!neuron || !NeuronOCL.CalcHiddenGradients((CObject*)GetPointer(cGEGWA), neuron.getOutput(), neuron.getGradient(), (ENUM_ACTIVATION)neuron.Activation())) ReturnFalse; if(!DiversityLoss(neuron, cOCM.GetContextWindow(), neuron.Neurons() / cOCM.GetContextWindow(), true)) ReturnFalse; CNeuronBaseOCL *background = cBackGround[cBackGround.Total() - 1]; if(!background || !DeConcat(context.getGradient(), background.getGradient(), neuron.getGradient(), context.Neurons(), background.Neurons(), 1) || !DeActivation(context.getOutput(), context.getGradient(), context.getGradient(), context.Activation()) || !SumAndNormalize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) || !context.SetGradient(context_grad, false) ) ReturnFalse; //--- Context Encoder for(int i = cContentEncoder.Total() - 3; i >= 0; i--) { context = cContentEncoder[i]; if(!context || !context.CalcHiddenGradients(cContentEncoder[i + 1]) ) ReturnFalse; } //--- Background if(!DeActivation(background.getOutput(), background.getGradient(), background.getGradient(), background.Activation())) ReturnFalse; for(int i = cBackGround.Total() - 2; i > 0; i--) { background = cBackGround[i]; if(!background || !background.CalcHiddenGradients(cBackGround[i + 1]) ) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { //--- Context Encoder CNeuronBaseOCL *context = NULL; for(int i = 1; i < cContentEncoder.Total() - 1; i++) { context = cContentEncoder[i]; if(!context || !context.UpdateInputWeights(cContentEncoder[i - 1], NULL) ) ReturnFalse; } //--- Background CNeuronBaseOCL *background = NULL; for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.UpdateInputWeights(cBackGround[i - 1]) ) ReturnFalse; } //--- Geometry-Enhaced Group-Word Attention background = cContentEncoder[cContentEncoder.Total() - 1]; if(!cGEGWA.UpdateInputWeights(NeuronOCL, background.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.UpdateInputWeights(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.updateInputWeights(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.updateInputWeights(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.updateInputWeights(decoder, context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRefMask::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cGEGWA.SetOpenCL(OpenCL); cContentEncoder.SetOpenCL(OpenCL); cBackGround.SetOpenCL(OpenCL); cLPC.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); cOCM.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cGEGWA.Save(file_handle)) ReturnFalse; if(!cContentEncoder.Save(file_handle)) ReturnFalse; if(!cBackGround.Save(file_handle)) ReturnFalse; if(!cLPC.Save(file_handle)) ReturnFalse; if(!cDecoder.Save(file_handle)) ReturnFalse; if(!cOCM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGEGWA.AsObject())) ReturnFalse; if(!cContentEncoder.Load(file_handle)) ReturnFalse; if(!cBackGround.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLPC.AsObject())) ReturnFalse; if(!cDecoder.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOCM.AsObject())) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput()) || !SetGradient(cOCM.getGradient()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(GELU); idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnits, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iUnits, optimization, iBatch) || !cScale.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 2 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 2 * iWindow, 2 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MHRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionGradient(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; int kernel = def_k_MHRelativeAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_ratg_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_ratg_q_g, cQuery.getGradientIndex()) setBuffer(kernel, def_k_ratg_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_ratg_k_g, cKey.getGradientIndex()) setBuffer(kernel, def_k_ratg_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_ratg_v_g, cValue.getGradientIndex()) setBuffer(kernel, def_k_ratg_scores, iScore) setBuffer(kernel, def_k_ratg_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bk_g, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bv_g, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gc_g, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gp_g, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gradient, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradientIndex()) setArgument(kernel, def_k_ratg_kunits, int(cKey.Neurons() / (iHeads * iWindowKey))) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(NeuronOCL) || !cValue.FeedForward(NeuronOCL) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; if(!MatMul(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cScale[0]).getOutput(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = cScale.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).CalcHiddenGradients(cScale[i + 1])) ReturnFalse; if(!MatMulGrad(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getGradient(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradient(), ((CNeuronBaseOCL*)cScale[0]).getGradient(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).CalcHiddenGradients(cMHAttentionPooling[i + 1])) ReturnFalse; if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(NeuronOCL) || !cValue.UpdateInputWeights(NeuronOCL) ) ReturnFalse; for(int i = cBKey.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = cBValue.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = cGlobalContentBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; for(int i = cScale.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeSelfAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau) || !cKey.WeightsUpdate(Source.cKey.AsObject(), tau) || !cValue.WeightsUpdate(Source.cValue.AsObject(), tau) ) ReturnFalse; if(!cBKey.WeightsUpdate(Source.cBKey.AsObject(), tau)) ReturnFalse; if(!cBValue.WeightsUpdate(Source.cBValue.AsObject(), tau)) ReturnFalse; if(!cGlobalContentBias.WeightsUpdate(Source.cGlobalContentBias.AsObject(), tau)) ReturnFalse; if(!cGlobalPositionalBias.WeightsUpdate(Source.cGlobalPositionalBias.AsObject(), tau)) ReturnFalse; if(!cMHAttentionPooling.WeightsUpdate(Source.cMHAttentionPooling.AsObject(), tau)) ReturnFalse; if(!cScale.WeightsUpdate(Source.cScale.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRelativeSelfAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnits * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(iUnits * iUnits, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cBKey.Save(file_handle)) ReturnFalse; if(!cBValue.Save(file_handle)) ReturnFalse; if(!cGlobalContentBias.Save(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Save(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; SetOpenCL(OpenCL); //--- if(!cBKey.Load(file_handle)) ReturnFalse; if(!cBValue.Load(file_handle)) ReturnFalse; if(!cGlobalContentBias.Load(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Load(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Load(file_handle)) ReturnFalse; if(!cScale.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cScale[cScale.Total() - 1]; if(!SetGradient(neuron.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CResidualConv(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, window, units_count, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *neuron = cLayers[0]; if(!neuron || !neuron.FeedForward(NeuronOCL)) ReturnFalse; for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron || !neuron.FeedForward(cLayers[i - 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(neuron.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1])) ReturnFalse; } neuron = cLayers[0]; if(!neuron.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRMAT *Source = source; if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle) || !cLayers.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle) || !cLayers.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cLayers[-1]; SetOutput(neuron.getOutput(), true); SetGradient(neuron.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRMAT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cLayers.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindow = window; iUnits = units_count; iHeads = heads; //--- cNeurons.Clear(); cNeurons.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow * iHeads, iWindow * iHeads, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cNeurons.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::feedForward(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.FeedForward(prev) ) ReturnFalse; prev = current; } //--- if(!MatMul(current.getOutput(), NeuronOCL.getOutput(), Output, 1, iHeads, iWindow, iUnits)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *current = cNeurons[cNeurons.Total() - 1]; if(!MatMulGrad(current.getOutput(), current.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, 1, iHeads, iWindow, iUnits) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation()) ) ReturnFalse; for(int i = cNeurons.Total() - 2; i > 0; i--) { current = cNeurons[i]; if(!current.CalcHiddenGradients(cNeurons[i + 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current;; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHAttentionPooling* Source = source; //--- if(!cNeurons.WeightsUpdate(Source.cNeurons.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cNeurons.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); cNeurons.SetOpenCL(OpenCL); //--- if(!cNeurons.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHAttentionPooling::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cNeurons.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMotifs::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL::SetActivationFunction(value); cMotifs.SetActivationFunction(activation); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { uint inputs = (units_count * step + (window - step)) * dimension; uint motifs = units_count * dimension; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs + motifs, optimization_type, batch)) ReturnFalse; if(!cMotifs.Init(0, 0, OpenCL, dimension * window, dimension * step, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Activation() != activation) SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); if(!cMotifs.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cMotifs.getOutput(), Output, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cMotifs.getGradient(), Gradient, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cMotifs.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cMotifs.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMotifs.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMotifs::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cMotifs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; //--- uint units1 = (iUnits + 1) / 2; uint units2 = (iUnits + 2) / 3; uint units3 = (iUnits + 3) / 4; uint wide = MathMax(MathMax(iUnits, units1 * 2), MathMax(units2 * 3, units3 * 4)); //--- int idx = 0; if(!cWideInputs.Init(0, idx, OpenCL, wide * iWindow, optimization, iBatch)) ReturnFalse; CBufferFloat *temp = cWideInputs.getOutput(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- idx++; if(!cAttentions[0].Init(0, idx, OpenCL, iWindow, window_key, iUnits, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[1].Init(0, idx, OpenCL, 2 * iWindow, window_key, units1, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[2].Init(0, idx, OpenCL, 3 * iWindow, window_key, units2, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[3].Init(0, idx, OpenCL, 4 * iWindow, window_key, units3, heads, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cConcatAttentions.Init(0, idx, OpenCL, 4 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cPooling.Init(0, idx, OpenCL, iWindow, iUnits, 4, optimization, iBatch)) ReturnFalse; //--- SetActivationFunction(None); if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { //--- Attention if(!cAttentions[0].FeedForward(NeuronOCL)) ReturnFalse; //--- Copy inputs if(!Concat(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cWideInputs.getOutput(), iWindow, 0, iUnits)) ReturnFalse; if(cWideInputs.Activation() != NeuronOCL.Activation()) cWideInputs.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].FeedForward(cWideInputs.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!Concat(cAttentions[0].getOutput(), cAttentions[1].getOutput(), cAttentions[2].getOutput(), cAttentions[3].getOutput(), cConcatAttentions.getOutput(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention pooling if(!cPooling.FeedForward(cConcatAttentions.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Attention pooling if(!cConcatAttentions.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!DeConcat(cAttentions[0].getGradient(), cAttentions[1].getGradient(), cAttentions[2].getGradient(), cAttentions[3].getGradient(), cConcatAttentions.getGradient(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention if(!NeuronOCL.CalcHiddenGradients(cAttentions[0].AsObject())) ReturnFalse; //--- Multi scale attentions for(uint i = 1; i < cAttentions.Size(); i++) { if(!cWideInputs.CalcHiddenGradients(cAttentions[i].AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cWideInputs.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { //--- Attention if(!cAttentions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].UpdateInputWeights(cWideInputs.AsObject())) ReturnFalse; //--- Attention pooling if(!cPooling.UpdateInputWeights(cConcatAttentions.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cWideInputs.SetOpenCL(OpenCL); for(int i = 0; i < 4; i++) cAttentions[i].SetOpenCL(OpenCL); cConcatAttentions.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cWideInputs.Save(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!cAttentions[i].Save(file_handle)) ReturnFalse; if(!cConcatAttentions.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cWideInputs.AsObject())) ReturnFalse; for(int i = 0; i < 4; i++) if(!LoadInsideLayer(file_handle, cAttentions[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatAttentions.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMolformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint motif_window, uint motif_step, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronMotifs *motif = new CNeuronMotifs(); uint motif_units = units_count - MathMax(motif_window - motif_step, 0); motif_units = (motif_units + motif_step - 1) / motif_step; if(!motif || !motif.Init(0, idx, OpenCL, window, motif_window, motif_step, motif_units, optimization, iBatch) || !cLayers.Add(motif) ) ReturnFalse; //--- idx++; CNeuronMultiScaleAttention *msat = NULL; CResidualConv *ff = NULL; uint units_total = units_count + motif_units; for(uint i = 0; i < layers; i++) { //--- Attention msat = new CNeuronMultiScaleAttention(); if(!msat || !msat.Init(0, idx, OpenCL, window, window_key, units_total, heads, optimization, iBatch) || !cLayers.Add(msat) ) ReturnFalse; idx++; //--- FeedForward ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units_total, optimization, iBatch) || !cLayers.Add(ff) ) ReturnFalse; idx++; } //--- Out CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, units_total, window, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, units_total, units_total, units_count, window, 1, optimization, iBatch) || !cLayers.Add(conv) ) ReturnFalse; idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; //--- if(!SetOutput(transp.getOutput()) || !SetGradient(transp.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iUnitsKV = units_kv; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnitsKV, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling(); if(!pooling || !pooling.Init(0, idx, OpenCL, iWindowKey, iUnits, iHeads, optimization, iBatch) || !cMHAttentionPooling.Add(pooling) ) ReturnFalse; //--- idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(GELU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- Key-Value inputs projection cKVProjection.Clear(); cKVProjection.SetOpenCL(OpenCL); idx++; neuron = new CNeuronBaseOCL; if(!neuron || !neuron.Init(0, idx, OpenCL, window_kv * iUnitsKV, optimization, iBatch) || !cKVProjection.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, window_kv, window_kv, iWindow, iUnitsKV, 1, optimization, iBatch) || !cKVProjection.Add(conv) ) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRelativeCrossAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cKVProjection.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnitsKV * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(MathMax(iUnits, iWindow) * iUnitsKV, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron || !SecondInput) ReturnFalse; if(neuron.getOutput() != SecondInput) if(!neuron.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.FeedForward(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(neuron) || !cValue.FeedForward(neuron) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(neuron.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cScale[0]).FeedForward(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) if(!neuron.SetGradient(SecondGradient)) ReturnFalse; if(neuron.Activation() != SecondActivation) neuron.SetActivationFunction(SecondActivation); //--- if(!DiversityLoss(AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- CNeuronBaseOCL* next = cScale[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cScale.Total() - 2; i >= 0; i--) { curr = cScale[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } for(int i = cMHAttentionPooling.Total() - 1; i >= 0; i--) { curr = cMHAttentionPooling[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; neuron = cKVProjection[cKVProjection.Total() - 1]; if(!neuron || !MatMulGrad(neuron.getOutput(), neuron.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) || !DiversityLoss(NeuronOCL, iUnits, iWindow, true) ) ReturnFalse; temp = neuron.getGradient(); if(!neuron.SetGradient(GetPointer(cTemp), false) || !neuron.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormalize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; for(int i = cKVProjection.Total() - 2; i >= 0; i--) { neuron = cKVProjection[i]; if(!neuron || !neuron.CalcHiddenGradients(cKVProjection[i + 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *neuron = NULL; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.UpdateInputWeights(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(neuron) || !cValue.UpdateInputWeights(neuron) ) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cScale[0]).UpdateInputWeights(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronRelativeSelfAttention::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeCrossAttention *Source = source; if(!cKVProjection.WeightsUpdate(GetPointer(Source.cKVProjection), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Save(const int file_handle) { if(!CNeuronRelativeSelfAttention::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnitsKV) < INT_VALUE) ReturnFalse; if(!cKVProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Load(const int file_handle) { if(!CNeuronRelativeSelfAttention::Load(file_handle)) ReturnFalse; iUnitsKV = (uint)FileReadInteger(file_handle); SetOpenCL(OpenCL); if(!cKVProjection.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units_count < 3) ReturnFalse; //--- cLayers.Clear(); //--- int bars_to_paattern = (units_count > 10 ? 3 : 2); CNeuronConvOCL *conv = new CNeuronConvOCL(); int idx = 0; int units = (int)units_count - bars_to_paattern + 1; if(!conv || !conv.Init(0, idx, open_cl, bars_to_paattern * window, window, window, units, 1, optimization_type, batch) || !cLayers.Add(conv) ) ReturnFalse; conv.SetActivationFunction(SIGMOID); idx++; units = units - bars_to_paattern + 1; CNeuronMotifs *motifs = new CNeuronMotifs(); if(!motifs || !motifs.Init(0, idx, open_cl, window, bars_to_paattern, 1, units, optimization_type, batch) || !cLayers.Add(motifs) ) ReturnFalse; motifs.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, motifs.Neurons(), optimization_type, batch)) ReturnFalse; cLayers.SetOpenCL(OpenCL); //--- CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *ff = NULL; units = int(motifs.Neurons() / window); for(uint i = 0; i < layers; i++) { idx++; attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, idx, OpenCL, window, window_key, units, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * properties, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = NULL; CNeuronRelativeSelfAttention *self_attention = NULL; CNeuronRelativeCrossAttention *cross_attention = NULL; CResidualConv *ff = NULL; //--- int idx = 0; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * properties, idx, OpenCL, 1, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; CBufferFloat *temp = neuron.getOutput(); if(!temp.Fill(1)) ReturnFalse; idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, window * properties, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; //--- for(uint i = 0; i < layers; i++) { idx++; self_attention = new CNeuronRelativeSelfAttention(); if(!self_attention || !self_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, optimization, iBatch) || !cLayers.Add(self_attention) ) { DeleteObj(self_attention); ReturnFalse; } idx++; cross_attention = new CNeuronRelativeCrossAttention(); if(!cross_attention || !cross_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, window, units_count, optimization, iBatch) || !cLayers.Add(cross_attention) ) { DeleteObj(cross_attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, properties, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- if(bTrain) { neuron = cLayers[1]; if(!neuron || !neuron.FeedForward(cLayers[0])) ReturnFalse; } for(int i = 2; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.FeedForward(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(cTemp.GetIndex() < 0 || cTemp.Total() < NeuronOCL.Neurons()) { cTemp.BufferFree(); if(!cTemp.BufferInit(NeuronOCL.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; } if(!NeuronOCL.getGradient() || !NeuronOCL.getGradient().Fill(0)) ReturnFalse;; //--- CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i > 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1], NeuronOCL.getOutput(), GetPointer(cTemp), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(neuron.Type() == defNeuronRelativeCrossAttention) if(!SumAndNormalize(NeuronOCL.getGradient(), GetPointer(cTemp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- int idx = 0; if(!cAtomEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; idx++; if(!cMotifEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; //--- cMotifProjection.Clear(); cMotifProjection.SetOpenCL(OpenCL); int motifs = int(cMotifEncoder.Neurons() / window); idx++; CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, motifs, window, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, motifs, motifs, units_count, 1, window, optimization, iBatch) || !cMotifProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cPropertyDecoder.Init(0, idx, OpenCL, window, window_key, properties, motifs, heads, layers, optimization, iBatch)) ReturnFalse; //--- cPropertyProjection.Clear(); cPropertyProjection.SetOpenCL(OpenCL); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, properties, window, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, properties, properties, units_count, 1, window, optimization, iBatch) || !cPropertyProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cConcatenate.Init(0, idx, OpenCL, 3 * window * units_count, optimization, iBatch)) ReturnFalse; idx++; if(!cPooling.Init(0, idx, OpenCL, window, units_count, 3, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput(), true) || !SetGradient(cPooling.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cAtomEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.FeedForward(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Concatenate uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); prev = cMotifProjection[cMotifProjection.Total() - 1]; if(!Concat(cAtomEncoder.getOutput(), prev.getOutput(), current.getOutput(), cConcatenate.getOutput(), window, window, window, units)) ReturnFalse; //--- Out if(!cPooling.FeedForward(cConcatenate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); CNeuronBaseOCL *motifs = cMotifProjection[cMotifProjection.Total() - 1]; CNeuronBaseOCL *prop = cPropertyProjection[cPropertyProjection.Total() - 1]; if(!motifs || !prop || !DeConcat(cAtomEncoder.getGradient(), motifs.getGradient(), prop.getGradient(), cConcatenate.getGradient(), window, window, window, units)) ReturnFalse; //--- if(cAtomEncoder.Activation() != None) if(!DeActivation(cAtomEncoder.getOutput(), cAtomEncoder.getGradient(), cAtomEncoder.getGradient(), cAtomEncoder.Activation())) ReturnFalse; if(motifs.Activation() != None) if(!DeActivation(motifs.getOutput(), motifs.getGradient(), motifs.getGradient(), motifs.Activation())) ReturnFalse; if(prop.Activation() != None) if(!DeActivation(prop.getOutput(), prop.getGradient(), prop.getGradient(), prop.Activation())) ReturnFalse; //--- if(!motifs.calcAlignmentGradient(cAtomEncoder.AsObject(), true)) ReturnFalse; for(int i = cMotifProjection.Total() - 2; i >= 0; i--) { motifs = cMotifProjection[i]; if(!motifs || !motifs.CalcHiddenGradients(cMotifProjection[i + 1])) ReturnFalse; } //--- for(int i = cPropertyProjection.Total() - 2; i >= 0; i--) { prop = cPropertyProjection[i]; if(!prop || !prop.CalcHiddenGradients(cPropertyProjection[i + 1])) ReturnFalse; } //--- if(!cPropertyDecoder.CalcHiddenGradients(cPropertyProjection[0]) || !cMotifEncoder.CalcHiddenGradients(cPropertyDecoder.AsObject())) ReturnFalse; CBufferFloat *temp = cMotifEncoder.getGradient(); if(!cMotifEncoder.SetGradient(cConcatenate.getGradient(), false) || !cMotifEncoder.CalcHiddenGradients(cMotifProjection[0]) || !SumAndNormalize(temp, cMotifEncoder.getGradient(), temp, window, false, 0, 0, 0, 1) || !cMotifEncoder.SetGradient(temp, false) || !NeuronOCL.CalcHiddenGradients(cMotifEncoder.AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cConcatenate.getGradient(), false) || !NeuronOCL.CalcHiddenGradients(cAtomEncoder.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, window, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cAtomEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.UpdateInputWeights(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Out if(!cPooling.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAMCT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cAtomEncoder.SetOpenCL(OpenCL); cMotifEncoder.SetOpenCL(OpenCL); cMotifProjection.SetOpenCL(OpenCL); cPropertyDecoder.SetOpenCL(OpenCL); cPropertyProjection.SetOpenCL(OpenCL); cConcatenate.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAtomEncoder.Save(file_handle)) ReturnFalse; if(!cMotifEncoder.Save(file_handle)) ReturnFalse; if(!cMotifProjection.Save(file_handle)) ReturnFalse; if(!cPropertyDecoder.Save(file_handle)) ReturnFalse; if(!cPropertyProjection.Save(file_handle)) ReturnFalse; if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cAtomEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMotifEncoder.AsObject())) ReturnFalse; if(!cMotifProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPropertyDecoder.AsObject())) ReturnFalse; if(!cPropertyProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothing(const CNeuronBaseOCL* neuron, const CNeuronBaseOCL* smoothing) { if(!OpenCL || !neuron || !neuron.getOutput() || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothing; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getOutputIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getOutputIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!smoothing.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothingGradient(const CNeuronBaseOCL* neuron, const CNeuronBaseOCL* smoothing) { if(!OpenCL || !neuron || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothingGradient; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getGradientIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getGradientIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!neuron.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint smoothing, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units_count, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iSmoothing = smoothing; iUnits = units_count; //--- if(!cFeatureSmoothing.Init(0, 0, OpenCL, (iSmoothing + 1)*iUnits * iDimension, optimization, iBatch)) ReturnFalse; cFeatureSmoothing.SetActivationFunction(None); if(!cTranspose.Init(0, 1, OpenCL, (iSmoothing + 1)*iUnits, iDimension, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); if(!cDistance.Init(0, 2, OpenCL, (iSmoothing + 1)*iUnits, optimization, iBatch)) ReturnFalse; cDistance.SetActivationFunction(None); if(!cAdaptation.Init(0, 3, OpenCL, cDistance.Neurons(), optimization, iBatch)) ReturnFalse; cAdaptation.SetActivationFunction(None); cAdaptation.SetHeads(iUnits); //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!FeatureSmoothing(NeuronOCL, cFeatureSmoothing.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cFeatureSmoothing.AsObject())) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cAdaptation.FeedForward(cDistance.AsObject())) ReturnFalse; if(!MatMul(cAdaptation.getOutput(), cFeatureSmoothing.getOutput(), Output, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cAdaptation.getOutput(), cAdaptation.getGradient(), cFeatureSmoothing.getOutput(), cFeatureSmoothing.getPrevOutput(), Gradient, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; if(!cDistance.CalcHiddenGradients(cAdaptation.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getGradient(), cDistance.getGradient(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cFeatureSmoothing.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormalize(cFeatureSmoothing.getGradient(), cFeatureSmoothing.getPrevOutput(), cFeatureSmoothing.getGradient(), iDimension, false, 0, 0, 0, 1) ) ReturnFalse; if(!FeatureSmoothingGradient(NeuronOCL, cFeatureSmoothing.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cFeatureSmoothing.getPrevOutput(), NeuronOCL.getGradient(), iDimension, false, 0, 0, 0, 1) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronNAFS::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cFeatureSmoothing.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cAdaptation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSmoothing)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; //--- Save objects if(!cFeatureSmoothing.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cAdaptation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSmoothing = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); //--- Load objects if(!LoadInsideLayer(file_handle, cFeatureSmoothing.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAdaptation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRMAT::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL *neuron = cLayers[-1]; if(!neuron) return; neuron.SetActivationFunction(value); CNeuronBaseOCL::SetActivationFunction((ENUM_ACTIVATION)neuron.Activation()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, count * window, variables, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, variables, count * window, optimization, iBatch)) ReturnFalse; if(!cTransposeRCD.Init(0, 1, OpenCL, count, window, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeRCD.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cTransposeRCD.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronTransposeOCL::calcInputGradients(cTransposeRCD.AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cTransposeRCD.AsObject())) ReturnFalse; //--- return prevLayer.CalcHiddenGradients(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cTransposeRCD.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeRCD.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTransposeVRCOCL::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cTransposeRCD.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronProofOCL::Init(numOutputs, myIndex, open_cl, window, step, units_count * window_out * variables, ADAM, batch)) ReturnFalse; //--- iWindowOut = window_out; iVariables = variables; iHeads = MathMax(MathMin(heads, window), 1); //--- const int window_h = int((iWindow + heads - 1) / heads); const int count = (int)((window_h + 1) * iWindowOut * iVariables); if(!WeightsConv) { WeightsConv = new CBufferFloat(); if(!WeightsConv) ReturnFalse; } if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(window_h + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!FirstMomentumConv) { FirstMomentumConv = new CBufferFloat(); if(!FirstMomentumConv) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SecondMomentumConv) { SecondMomentumConv = new CBufferFloat(); if(!SecondMomentumConv) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(DeltaWeightsConv); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::feedForward(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3]; global_work_size[0] = Output.Total() / (iWindowOut * iVariables); global_work_size[1] = iHeads; global_work_size[2] = iVariables; //--- ResetLastError(); const int kernel = def_k_FeedForwardMHConv; setBuffer(kernel, def_k_ffc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffс_window_out, (int)iWindowOut) setArgument(kernel, def_k_ffc_activation, (int)activation) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iVariables; global_work_size[1] = iVariables; ResetLastError(); const int kernel = def_k_CalcHiddenGradientMHConv; setBuffer(kernel, def_k_chgc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_g, Gradient.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_chgc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgc_step, (int)iStep) setArgument(kernel, def_k_chgc_window_in, (int)iWindow) setArgument(kernel, def_k_chgc_window_out, (int)iWindowOut) setArgument(kernel, def_k_chgc_activation, (int)NeuronOCL.Activation()) setArgument(kernel, def_k_chgc_shift_out, (int)0) setArgument(kernel, def_k_chgc_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { WeightsConv.Total() }; float lt = (float)(lr * MathSqrt(1.0 - MathPow((double)b2, (double)t)) / (1.0 - MathPow((double)b1, (double)t))); ResetLastError(); const int kernel = def_k_UpdateWeightsMHConvAdam; setBuffer(kernel, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwca_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwca_l, lt) setArgument(kernel, def_k_uwca_b1, b1) setArgument(kernel, def_k_uwca_b2, b2) setArgument(kernel, def_k_uwca_window_in, (int)iWindow) setArgument(kernel, def_k_uwca_window_out, (int)iWindowOut) setArgument(kernel, def_k_uwca_step, (int)iStep) setArgument(kernel, def_k_uwca_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!FileWriteInteger(file_handle, (int)iHeads)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- if(!acConvolutions[0].Init(0, 0, OpenCL, window, window, window_out, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[0].SetActivationFunction(GELU); //--- if(!acConvolutions[1].Init(0, 1, OpenCL, window_out, window_out, window, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[1].SetActivationFunction(None); //--- if(!SetGradient(acConvolutions[1].getGradient(), true)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::feedForward(CNeuronBaseOCL* NeuronOCL) { CObject *prev = NeuronOCL; for(uint i = 0; i < acConvolutions.Size(); i++) { if(!acConvolutions[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(acConvolutions[i]); } //--- if(!SumAndNormalize(NeuronOCL.getOutput(), acConvolutions[acConvolutions.Size() - 1].getOutput(), Output, acConvolutions[0].GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = (int)acConvolutions.Size() - 2; i >= 0; i--) { if(!acConvolutions[i].CalcHiddenGradients(acConvolutions[i + 1].AsObject())) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(acConvolutions[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), Gradient, NeuronOCL.Activation()) || !SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { for(int i = (int)acConvolutions.Size() - 1; i > 0; i--) { if(!acConvolutions[i].UpdateInputWeights(acConvolutions[i - 1].AsObject())) ReturnFalse; } if(!acConvolutions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMHFeedForward* Source = source; if(acConvolutions.Size() != Source.acConvolutions.Size()) ReturnFalse; for(int i = 0; i < (int)acConvolutions.Size(); i++) if(!acConvolutions[i].WeightsUpdate(Source.acConvolutions[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!acConvolutions[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!LoadInsideLayer(file_handle, acConvolutions[i].AsObject())) ReturnFalse; //--- if(!SetGradient(acConvolutions[acConvolutions.Size() - 1].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFeedForward::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < acConvolutions.Size(); i++) acConvolutions[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronMHFeedForward *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronRelativeCrossAttention *cross = NULL; CNeuronMHFeedForward *conv = NULL; bool use_self = units_count > 0; int layer = 0; for(uint i = 0; i < layers; i++) { if(use_self) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, layer, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } layer++; } cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, layer, OpenCL, window, window_key, units_count, heads, window_cross, units_cross, optimization, iBatch) || !cLayers.Add(cross) ) { DeleteObj(cross); ReturnFalse; } layer++; conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, layer, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } layer++; } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CObject *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cLayers.Total(); i++) { current = cLayers[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(PrevOutput.Total() != SecondGradient.Total()) { PrevOutput.BufferFree(); if(!PrevOutput.BufferInit(SecondGradient.Total(), 0) || !PrevOutput.BufferCreate(OpenCL)) ReturnFalse; } if(!SecondGradient.Fill(0) || !PrevOutput.Fill(0)) ReturnFalse; //--- CNeuronBaseOCL *next = cLayers[-1]; CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { current = cLayers[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, PrevOutput, SecondGradient, 1, true, 0, 0, 0, 1)) ReturnFalse; next = current; } //--- if(!NeuronOCL.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormalize(SecondGradient, PrevOutput, SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { current = cLayers[i]; if(!current || !current.UpdateInputWeights(cLayers[i - 1], SecondInput)) ReturnFalse; } //--- if(!((CNeuronBaseOCL*)cLayers[0]).UpdateInputWeights(NeuronOCL, SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDilatedCasualConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint dimension, uint units_count, uint variables, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 1, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); uint units = units_count; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint i = 0; i < layers; i++) { s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, i * 2, OpenCL, dimension, dimension * units * variables, optimization, iBatch) || !cLayers.Add(s3)) { DeleteObj(s3); ReturnFalse; } s3.SetActivationFunction(None); //--- conv = new CNeuronConvOCL(); units = MathMax((units - window + step) / step, 1); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window * dimension, step * dimension, dimension, units, variables, optimization, iBatch) || !cLayers.Add(conv)) { if(!!conv) DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(GELU); } //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, OpenCL, conv.Neurons(), optimization_type, batch)) ReturnFalse; //--- if(!SetGradient(conv.getGradient(), true) || !SetOutput(conv.getOutput(), true)) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeCrossAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint history_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMultiScaleRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, window_key, history_size, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cSelfAttention.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, optimization, iBatch)) ReturnFalse; index++; if(!cTransposeSA.Init(0, index, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cConvolution.Init(0, index, OpenCL, iUnits, iUnits, iWindowKey, 1, iWindow, optimization, iBatch)) ReturnFalse; index++; uint windows[] = { iWindowKey * iWindow }; if(!cHistory.Init(0, index, OpenCL, iUnitsKV, iWindowKey, windows)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeSA.FeedForward(cSelfAttention.AsObject())) ReturnFalse; if(!cConvolution.FeedForward(cTransposeSA.AsObject())) ReturnFalse; if(!cHistory.FeedForward(cConvolution.AsObject())) ReturnFalse; //--- return CNeuronMultiScaleRelativeCrossAttention::feedForward(cSelfAttention.AsObject(), cHistory.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMultiScaleRelativeCrossAttention::calcInputGradients(cSelfAttention.AsObject(), cHistory.getOutput(), cHistory.getGradient(), (ENUM_ACTIVATION)cHistory.Activation())) ReturnFalse; if(!cConvolution.CalcHiddenGradients(cHistory.AsObject())) ReturnFalse; if(!cTransposeSA.CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; CBufferFloat *temp = cSelfAttention.getGradient(); if(!cSelfAttention.SetGradient(cTransposeSA.getPrevOutput(), false) || !cSelfAttention.CalcHiddenGradients(cTransposeSA.AsObject()) || !SumAndNormalize(temp, cSelfAttention.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cSelfAttention.SetGradient(temp, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSelfAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!CNeuronMultiScaleRelativeCrossAttention::updateInputWeights(cSelfAttention.AsObject(), cHistory.getOutput())) ReturnFalse; if(!cHistory.UpdateInputWeights(cConvolution.AsObject())) ReturnFalse; if(!cConvolution.UpdateInputWeights(cTransposeSA.AsObject())) ReturnFalse; if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Save(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; if(!cTransposeSA.Save(file_handle)) ReturnFalse; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cHistory.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Load(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeSA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistory.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Clear(void) { if(!CNeuronMultiScaleRelativeCrossAttention::Clear()) ReturnFalse; if(!cSelfAttention.Clear()) ReturnFalse; if(!cTransposeSA.Clear()) ReturnFalse; if(!cConvolution.Clear()) ReturnFalse; if(!cHistory.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRecursiveAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMultiScaleRelativeCrossAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); cTransposeSA.SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cHistory.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iVariables = variables; //--- int index = 0; if(!cQuery.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(SIGMOID); index++; if(!cKey.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(TANH); index++; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction(TANH); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL)) ReturnFalse; if(!cKey.FeedForward(NeuronOCL) || !cKeyT.FeedForward(cKey.AsObject())) ReturnFalse; if(!MatMul(cKeyT.getOutput(), NeuronOCL.getOutput(), cKeyValue.getOutput(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!MatMul(cQuery.getOutput(), cKeyValue.getOutput(), cAttentionOut.getOutput(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttentionOut.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cQuery.getOutput(), cQuery.getGradient(), cKeyValue.getOutput(), cKeyValue.getGradient(), cAttentionOut.getGradient(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!MatMulGrad(cKeyT.getOutput(), cKeyT.getGradient(), NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cKeyValue.getGradient(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!SumAndNormalize(Gradient, cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(cKeyT.Activation() != None) if(!DeActivation(cKeyT.getOutput(), cKeyT.getGradient(), cKeyT.getGradient(), cKeyT.Activation())) ReturnFalse; if(cQuery.Activation() != None) if(!DeActivation(cQuery.getOutput(), cQuery.getGradient(), cQuery.getGradient(), cQuery.Activation())) ReturnFalse; //--- if(!cKey.CalcHiddenGradients(cKeyT.AsObject()) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormalize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKey.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronLinerAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau)) ReturnFalse; if(!cKey.WeightsUpdate(Source.cKey.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (int)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; //--- int index = 2; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction((ENUM_ACTIVATION)cKey.Activation()); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLinerAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cKeyT.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(window_key, 1); iUnits = units_count; iHeads = MathMax(heads, 1); iVariables = variables; //--- uint index = 0; if(!cMask.Init(0, index, OpenCL, 2 * iWindow, 2 * iWindow, iVariables * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cMask.SetActivationFunction(SIGMOID); CBufferFloat *temp = cMask.GetWeightsConv(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- index++; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(None); index++; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cPooling.Init(0, index, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(LReLU); index++; if(!cFeedForward[1].Init(0, index, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables/*Q units*/, iVariables/*K units*/, iHeads * iUnits/*Heads*/}; uint local_work_size[3] = {1, iVariables, 1}; ResetLastError(); int kernel = def_k_MaskAttentionComplex; setBuffer(kernel, def_k_maskattcom_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcom_out, cMHAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_masks, cMask.getOutputIndex()) setArgument(kernel, def_k_maskattcom_dimension, (int)iWindowKey) setArgument(kernel, def_k_maskattcom_heads_kv, (int)(iHeads * iUnits)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables, iWindowKey, iHeads * iUnits}; ResetLastError(); int kernel = def_k_MaskAttentionGradientsComplex; setBuffer(kernel, def_k_maskattcomgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcomgr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_mask_g, cMask.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_maskattcomgr_kunits, (int)iVariables) setArgument(kernel, def_k_maskattcomgr_heads_kv, (int)(iHeads * iUnits)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cQ.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::SumAndNormalize(CBufferFloat* tensor1, CBufferFloat* tensor2, CBufferFloat* out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; int kernel = def_k_MatrixSum; setBuffer(kernel, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_sum_matrix_out, out.GetIndex()) setArgument(kernel, def_k_sum_dimension, 2 * dimension) setArgument(kernel, def_k_sum_shift_in1, 2 * shift_in1) setArgument(kernel, def_k_sum_shift_in2, 2 * shift_in2) setArgument(kernel, def_k_sum_shift_out, 2 * shift_out) setArgument(kernel, def_k_sum_multiplyer, multiplyer) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- if(!normilize) return true; //--- kernel = def_k_ComplexNormalize; setBuffer(kernel, def_k_cn_inputs, out.GetIndex()) setBuffer(kernel, def_k_cn_outputs, out.GetIndex()) setBuffer(kernel, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(kernel, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(kernel, def_k_cn_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!out.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; //cMask.getOutput().BufferRead(); if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //cQKV.getOutput().BufferRead(); if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; //cPooling.getOutput().BufferRead(); if(!SumAndNormalize(NeuronOCL.getOutput(), cPooling.getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; //cFeedForward[0].getOutput().BufferRead(); if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; //cFeedForward[1].getOutput().BufferRead(); if(!SumAndNormalize(cResidual.getOutput(), cFeedForward[1].getOutput(), getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; //--- if(!DeActivation(cPooling.getOutput(), cPooling.getGradient(), cResidual.getGradient(), cPooling.Activation()) || !DeActivation(cPooling.getOutput(), cPooling.getPrevOutput(), Gradient, cPooling.Activation()) || !SumAndNormalize(cPooling.getGradient(), cPooling.getPrevOutput(), cPooling.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; //--- if(!DeActivation(cQKV.getOutput(), cQKV.getPrevOutput(), cQKV.getGradient(), cQKV.Activation()) || !prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getPrevOutput(), cResidual.getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getGradient(), Gradient, prevLayer.Activation()) || !SumAndNormalize(cResidual.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(cMask.getOutput(), cMask.getGradient(), cMask.getGradient(), cMask.Activation()) || !prevLayer.CalcHiddenGradients(cMask.AsObject()) || !SumAndNormalize(prevLayer.getGradient(), cResidual.getPrevOutput(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronComplexMVMHMaskAttention *Source = source; if(!cMask.WeightsUpdate(GetPointer(Source.cMask), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(GetPointer(Source.cPooling), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFeedForward[i].WeightsUpdate(GetPointer(Source.cFeedForward[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!cFeedForward[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!LoadInsideLayer(file_handle, cFeedForward[i].AsObject())) ReturnFalse; //--- uint index = 2; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronComplexMVMHMaskAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); for(uint i = 0; i < cFeedForward.Size(); i++) cFeedForward[i].SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, window, window, units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(5, window_key); iHeads = MathMax(1, heads); iUnits = units_count; iTopKQuerys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); iRandomKeys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(TANH); index++; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cRandomK.Init(0, index, OpenCL, iHeads * MathMax(iRandomKeys, iTopKQuerys), optimization, iBatch)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, iTopKQuerys * iHeads * iWindowKey, optimization, iBatch)) ReturnFalse; index++; if(!cPooling.Init(0, index, OpenCL, iHeads * iWindowKey, iHeads * iWindowKey, iWindow, iTopKQuerys, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(TANH); index++; if(!cTranspose[0].Init(0, index, OpenCL, iTopKQuerys, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cScaling.Init(0, index, OpenCL, iTopKQuerys, iTopKQuerys, iUnits, iWindow, optimization, iBatch)) ReturnFalse; cScaling.SetActivationFunction(None); index++; if(!cTranspose[1].Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::RandomKeys(CBufferFloat* indexes, int random, int units, int heads) { if(!indexes || random > units || indexes.Total() < (random * heads) ) ReturnFalse; //--- matrix ind = matrix::Zeros(random, heads); if(random == units) { for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = (float)r; } } else { double step = double(units) / random; for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = float(int((r + MathRand() / 32767.0) * step)); } } if(!indexes.AssignArray(ind) || !indexes.BufferWrite()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::QueryImportance(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iRandomKeys, iHeads}; uint local_work_size[3] = {1, iRandomKeys, 1}; uint kernel = def_k_ProbAttentionQeuryImp; setBuffer(kernel, def_k_probat_querys, cQ.getOutputIndex()) setBuffer(kernel, def_k_probat_keys_values, cKV.getOutputIndex()) setBuffer(kernel, def_k_probat_index_keys, cRandomK.getOutputIndex()) setBuffer(kernel, def_k_probat_querys_imp, cQ.getPrevOutIndex()) setArgument(kernel, def_k_probat_dimension, int(iWindowKey)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQ.getPrevOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::TopKIndexes(void) { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iHeads}; uint kernel = def_k_TopKImportanceToIndex; setBuffer(kernel, def_k_imptoind_importance, cQ.getPrevOutIndex()) setBuffer(kernel, def_k_imptoind_indexes, cRandomK.getPrevOutIndex()) setArgument(kernel, def_k_imptoind_tok_k, int(iTopKQuerys)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cRandomK.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionOut(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; uint kernel = def_k_QIndexAttention; setBuffer(kernel, def_k_indatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indatt_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indatt_scores, ibScore) setBuffer(kernel, def_k_indatt_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_indatt_dimension, int(iWindowKey)) setArgument(kernel, def_k_indatt_heads_kv, int(iHeads)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionInsideGradients(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint kernel = def_k_QIndexAttentionGradients; setBuffer(kernel, def_k_indattgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indattgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_indattgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indattgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_indattgr_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indattgr_scores, ibScore) setBuffer(kernel, def_k_indattgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_indattgr_kunits, int(iUnits)) setArgument(kernel, def_k_indattgr_heads_kv, int(iHeads)) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cKV.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(!RandomKeys(cRandomK.getOutput(), iRandomKeys, iUnits, iHeads)) ReturnFalse; if(!QueryImportance() || !TopKIndexes()) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!cTranspose[0].FeedForward(cPooling.AsObject())) ReturnFalse; if(!cScaling.FeedForward(cTranspose[0].AsObject())) ReturnFalse; if(!cTranspose[1].FeedForward(cScaling.AsObject())) ReturnFalse; if(!SumAndNormalize(cTranspose[1].getOutput(), NeuronOCL.getOutput(), cTranspose[1].getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return CResidualConv::feedForward(cTranspose[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::calcInputGradients(CNeuronBaseOCL* prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cQ.getGradient().Fill(0)) ReturnFalse; //--- if(!CResidualConv::calcInputGradients(cTranspose[1].AsObject())) ReturnFalse; //--- if(!cScaling.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!cTranspose[0].CalcHiddenGradients(cScaling.AsObject())) ReturnFalse; if(!cPooling.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(cQKV.Activation() != None) if(!DeActivation(cQKV.getOutput(), cQKV.getGradient(), cQKV.getGradient(), cQKV.Activation())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cTranspose[1].getGradient(), cTranspose[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(cTranspose[1].getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::updateInputWeights(CNeuronBaseOCL* NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cScaling.UpdateInputWeights(cTranspose[0].AsObject())) ReturnFalse; //--- return CResidualConv::updateInputWeights(cTranspose[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iTopKQuerys)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iRandomKeys)) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cRandomK.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; if(!cTranspose[0].Save(file_handle)) ReturnFalse; if(!cTranspose[1].Save(file_handle)) ReturnFalse; if(!cScaling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Load(const int file_handle) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- if(!CResidualConv::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iTopKQuerys = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iRandomKeys = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRandomK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMHAttentionOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScaling.AsObject())) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHProbAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- CResidualConv::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cRandomK.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); cTranspose[0].SetOpenCL(OpenCL); cTranspose[1].SetOpenCL(OpenCL); cScaling.SetOpenCL(OpenCL); //--- if(!!OpenCL) ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::WeightsUpdate(CNeuronBaseOCL* source, float tau) { if(!CResidualConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHProbAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(Source.cPooling.AsObject(), tau)) ReturnFalse; if(!cScaling.WeightsUpdate(Source.cScaling.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint windows_total, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint win = window * windows_total + MathMax(windows_total, 1) - 1; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, win, win, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; iWindowsTotal = windows_total; iWindow = window; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::feedForward(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput) { if(!OpenCL || !Output || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowOut, iVariables}; const int kernel = def_k_FeedForwardMaskMultWinConv; setBuffer(kernel, def_k_ffmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_o, getOutputIndex()) setArgument(kernel, def_k_ffmmwc_activation, Activation()) setArgument(kernel, def_k_ffmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_ffmmwc_windows_total, (int)iWindowsTotal) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::calcInputGradients(CNeuronBaseOCL* NeuronOCL, CBufferFloat* SecondInput, CBufferFloat* SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput || !NeuronOCL.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowsTotal, iVariables}; const int kernel = def_k_CalcHiddenGradientMaskMultWinConv; setBuffer(kernel, def_k_chgmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_chgmmwc_masks_g, SecondGradient.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_chgmmwc_activation, NeuronOCL.Activation()) setArgument(kernel, def_k_chgmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_chgmmwc_window_out, (int)iWindowOut) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::updateInputWeights(CNeuronBaseOCL* NeuronOCL, CBufferFloat* second) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !second) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {(iWindow + 1)*iWindowsTotal, iWindowOut, iVariables}; const int kernel = def_k_UpdateWeightsMaskMultWinConvAdam; setBuffer(kernel, def_k_uwmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmmwc_masks, second.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_v, SecondMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_uwmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_windows_total, (int)iWindowsTotal) setArgument(kernel, def_k_uwmmwc_l, lr) setArgument(kernel, def_k_uwmmwc_b1, b1) setArgument(kernel, def_k_uwmmwc_b2, b2) //--- kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!WeightsConv.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowsTotal) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindowsTotal = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window % 2 > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iVariables = variables; //--- matrix pe = matrix::Zeros(count, iWindow); vector position = vector::Ones(count); position = position.CumSum() - 1; for(uint i = 0; i < iWindow / 2; i++) { vector temp = position / MathPow(10000.0f, 2.0f * i / window); pe.Col(MathCos(temp), i * 2); pe.Col(MathSin(temp), i * 2 + 1); } PositionEncoder.BufferFree(); if(!PositionEncoder.AssignArray(pe)) ReturnFalse; SetActivationFunction(None); //--- return PositionEncoder.BufferCreate(open_cl); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::feedForward(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_RoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::calcInputGradients(CNeuronBaseOCL* NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_CalcHiddenGradRoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Save(const int file_handle) { if(!CNeuronPositionEncoder::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Load(const int file_handle) { if(!CNeuronPositionEncoder::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = uint(FileReadInteger(file_handle)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * period * variables, optimization_type, batch)) ReturnFalse; CNeuronBaseOCL::SetActivationFunction(None); //--- iCount = units_count; iVariables = variables; iPeriod = period; //--- if(!cAttention.Init(0, 0, OpenCL, iPeriod * iVariables, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); //--- if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, MathMin(iPeriod, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atn_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMin(iCount, iPeriod), MathMin(iPeriod * iCount, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atng_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cAttention.FeedForward()) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; } //--- return AttentNorm(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!AttentNormGrad(NeuronOCL)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return cAttention.UpdateInputWeights(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAttentNorm* Source = source; return cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPeriod)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iPeriod = (uint)FileReadInteger(file_handle); //--- if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count, variables, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { 1, MathMin(iWindow, uint(OpenCL.GetMaxLocalSize(1))), iCount}; uint local_work_size[3] = { 1, global_work_size[1], 1}; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, cNorm.getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iWindow, iCount)) setArgument(kernel, def_k_atn_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMax(iCount, iWindow), MathMin(MathMax(iWindow, iCount), uint(OpenCL.GetMaxLocalSize(1))), iCount }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, cNorm.getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iCount, iWindow)) setArgument(kernel, def_k_atng_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!NeuronOCL.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cAttention.getOutput(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; if(!AttentNorm(cTranspose.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!AttentNormGrad(cTranspose.AsObject())) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getPrevOutput(), cAttention.getGradient(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!SumAndNormalize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), getPrevOutput(), NeuronOCL.getGradient(), iCount, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint period, uint timeframe, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, heads, units_count, window, units_count, optimization_type, batch)) ReturnFalse; //--- FF[0].SetActivationFunction(GELU); //--- iTimeframe = MathMax(1, timeframe); if(!cParams.Init(0, 0, OpenCL, window * units_count, period, optimization, iBatch)) ReturnFalse; if(!cParams.Zeros()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!SecondInput) ReturnFalse; //--- int pos = int(SecondInput[0]); pos = (pos / int(iTimeframe)) % cParams.GetPeriod(); if(!cParams.SetPosition(pos) || !cParams.FeedForward()) ReturnFalse; //--- if(!Q_Embedding.FeedForward(cParams.AsObject())) ReturnFalse; //--- if(!KV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormalize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormalize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; //--- if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormalize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!prevLayer.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!cParams.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow_K, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!cParams.UpdateInputWeights()) ReturnFalse; if(!Q_Embedding.UpdateInputWeights(cParams.AsObject())) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronCrossAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTQMHA* Source = source; if(!cParams.WeightsUpdate(Source.cParams.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Save(const int file_handle) { if(!CNeuronCrossAttention::Save(file_handle)) ReturnFalse; //--- if(!cParams.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iTimeframe)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Load(const int file_handle) { if(!CNeuronCrossAttention::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iTimeframe = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTQMHA::SetOpenCL(COpenCLMy *obj) { CNeuronCrossAttention::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!sparse >= 1 || sparse < 0) ReturnFalse; fSparse = sparse; //--- if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, heads, heads, 1, units * m_units, 1, optimization_type, batch)) ReturnFalse; int index = 0; if(!cNeighbors.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; CBufferFloat* temp = cNeighbors.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cRamdomCandidates.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cProjection[0].Init(index, 0, OpenCL, window, window, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(SoftPlus); index++; if(!cProjection[1].Init(index, 0, OpenCL, 2 * heads, 2 * heads, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(TANH); index++; if(!cScores.Init(0, index, OpenCL, units * m_units * heads, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SignificantNeighborsSampling(CNeuronBaseOCL *NeuronOCL) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint window = cProjection[0].GetWindow(); //--- if(!NeuronOCL || NeuronOCL.Neurons() != (units * window)) ReturnFalse; if(!cNeighbors.SwapOutputs()) ReturnFalse; CBufferFloat *temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, float(units))) ReturnFalse; //--- uint global_work_offset[2] = { 0 }; uint global_work_size[2] = { units, m_units }; uint local_work_size[2] = { 1, m_units }; //--- uint kernel = def_k_SignificantNeighborsSampling; setBuffer(kernel, def_k_sns_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_sns_random_cands, cRamdomCandidates.getOutputIndex()) setBuffer(kernel, def_k_sns_candidates, cNeighbors.getPrevOutIndex()) setBuffer(kernel, def_k_sns_neighbors, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_sns_dimension, window) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cNeighbors.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScores(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScores; setBuffer(kernel, def_k_smhs_data, cProjection[1].getOutputIndex()) setBuffer(kernel, def_k_smhs_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_indexes, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_smhs_sparse, fSparse) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cNeighbors.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScoresGrad(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScoresGrad; setBuffer(kernel, def_k_smhs_gr_data_gr, cProjection[1].getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_gr_scores_gr, cScores.getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_indexes, cNeighbors.getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cProjection[1].getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!SignificantNeighborsSampling(NeuronOCL)) ReturnFalse; CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].FeedForward(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!SparseMHScores()) ReturnFalse; if(!CNeuronConvOCL::feedForward(cScores.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cScores.AsObject())) ReturnFalse; if(!SparseMHScoresGrad()) ReturnFalse; //--- int total = (int)cProjection.Size(); CNeuronBaseOCL* inputs = NULL; for(int i = total - 1; i >= 0; i--) { inputs = (i > 0 ? cProjection[i - 1].AsObject() : prevLayer); if(!inputs.CalcHiddenGradients(cProjection[i].AsObject())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].UpdateInputWeights(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!CNeuronConvOCL::updateInputWeights(cScores.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSNSMHAttention* Source = source; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].WeightsUpdate(Source.cProjection[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteFloat(file_handle, fSparse) < sizeof(float)) ReturnFalse; //--- if(!cNeighbors.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; fSparse = FileReadFloat(file_handle); //--- if(!LoadInsideLayer(file_handle, cNeighbors.AsObject())) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!LoadInsideLayer(file_handle, cProjection[i].AsObject())) ReturnFalse; //--- int index = 1; if(!cRamdomCandidates.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; index += int(cProjection.Size()); if(!cScores.Init(0, index, OpenCL, Neurons() * iWindow, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSNSMHAttention::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cNeighbors.SetOpenCL(OpenCL); cRamdomCandidates.SetOpenCL(OpenCL); for(uint i = 0; i < cProjection.Size(); i++) cProjection[i].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint dimension_k, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHFeedForward::Init(numOutputs, myIndex, open_cl, window, 2 * window, units, 1, heads, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cMask.Init(0, index, OpenCL, units, window, heads, m_units, sparse, optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, window, window, 2 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, window, window, 4 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, 2 * dimension_k * heads, 2 * dimension_k * heads, window, units, 1, optimization, iBatch)) ReturnFalse; cW0.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttention(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttention; setBuffer(kernel, def_k_glatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_out, cMHAttention.getOutputIndex()) setArgument(kernel, def_k_glatt_dimension, dimension_k) setArgument(kernel, def_k_glatt_total_kv, units) setArgument(kernel, def_k_glatt_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttention.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttentionGrad(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttentionGrad; setBuffer(kernel, def_k_glatt_gr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_q_gr, cQ.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_kv_gr, cKV.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask_gr, cMask.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_gr_out_gr, cMHAttention.getGradientIndex()) setArgument(kernel, def_k_glatt_gr_dimension, dimension_k) setArgument(kernel, def_k_glatt_gr_total_q, units) setArgument(kernel, def_k_glatt_gr_total_kv, units) setArgument(kernel, def_k_glatt_gr_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQ.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; if(!cQ.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalLocalAttention()) ReturnFalse; if(!cW0.FeedForward(cMHAttention.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cResidual.getOutput(), cW0.GetFilters(), true, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return CNeuronMHFeedForward::feedForward(cResidual.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMHFeedForward::calcInputGradients(cResidual.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cResidual.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttention.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!GlobalLocalAttentionGrad()) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQ.AsObject())) ReturnFalse; if(!DeActivation(cResidual.getOutput(), cResidual.getGradient(), cResidual.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cKV.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMask.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQ.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return CNeuronMHFeedForward::updateInputWeights(cResidual.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGlobalLocalAttention* Source = source; if(!cMask.WeightsUpdate(Source.cMask.AsObject(), tau)) ReturnFalse; if(!cQ.WeightsUpdate(Source.cQ.AsObject(), tau)) ReturnFalse; if(!cKV.WeightsUpdate(Source.cKV.AsObject(), tau)) ReturnFalse; if(!cW0.WeightsUpdate(Source.cW0.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Save(const int file_handle) { if(!CNeuronMHFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cMask.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Load(const int file_handle) { if(!CNeuronMHFeedForward::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- int index = 3; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index += 2; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGlobalLocalAttention::SetOpenCL(COpenCLMy *obj) { CNeuronMHFeedForward::SetOpenCL(obj); //--- cMask.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint heads, uint dimension_k, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronMLMHAttentionOCL *att = NULL; CNeuronGlobalLocalAttention *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, layers, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, att_layers, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, variables, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Concatenate if(!Concat(cTimeModule[-1].getOutput(), cMixModule[-1].getOutput(), cSpatialModule[-1].getOutput(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- return CNeuronMHAttentionPooling::feedForward(cConcatResults.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcatResults.AsObject())) ReturnFalse; //--- DeConcatenate if(!cTimeModule[-1] || !cMixModule[-1] || !cSpatialModule[-1] || !DeConcat(cTimeModule[-1].getGradient(), cMixModule[-1].getGradient(), cSpatialModule[-1].getGradient(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; //--- Spatial Module for(int i = cSpatialModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cSpatialModule[i - 1] : NeuronOCL); next = cSpatialModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Mix Module for(int i = cMixModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cMixModule[i - 1] : cProjectionT[-1]); next = cMixModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Time Module CBufferFloat *temp = current.getGradient(); if(!current.SetGradient(current.getPrevOutput(), false)) ReturnFalse; for(int i = cTimeModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cTimeModule[i - 1] : cProjectionT[-1]); next = cTimeModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormalize(temp, current.getGradient(), temp, 1, false, 0, 0, 0, 1) || !current.SetGradient(temp, false)) ReturnFalse; //--- Time projection temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; for(int i = cProjectionT.Total() - 1; i >= 0; i--) { current = (i > 0 ? cProjectionT[i - 1] : NeuronOCL); next = cProjectionT[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, SecondGradient, SecondActivation)) ReturnFalse; } if(!SumAndNormalize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronMHAttentionPooling::updateInputWeights(cConcatResults.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronExtralonger *Source = source; if(!cProjectionT.WeightsUpdate(Source.cProjectionT.AsObject(), tau)) ReturnFalse; if(!cTimeModule.WeightsUpdate(Source.cTimeModule.AsObject(), tau)) ReturnFalse; if(!cMixModule.WeightsUpdate(Source.cMixModule.AsObject(), tau)) ReturnFalse; if(!cSpatialModule.WeightsUpdate(Source.cSpatialModule.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cProjectionT.Save(file_handle)) ReturnFalse; if(!cTimeModule.Save(file_handle)) ReturnFalse; if(!cSpatialModule.Save(file_handle)) ReturnFalse; if(!cMixModule.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!cProjectionT.Load(file_handle)) ReturnFalse; if(!cTimeModule.Load(file_handle)) ReturnFalse; if(!cSpatialModule.Load(file_handle)) ReturnFalse; if(!cMixModule.Load(file_handle)) ReturnFalse; //--- int index = cProjectionT.Total() + cTimeModule.Total() + cSpatialModule.Total() + cMixModule.Total(); if(!cConcatResults.Init(0, index, OpenCL, iHeads * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronExtralonger::SetOpenCL(COpenCLMy *obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cProjectionT.SetOpenCL(OpenCL); cTimeModule.SetOpenCL(OpenCL); cSpatialModule.SetOpenCL(OpenCL); cMixModule.SetOpenCL(OpenCL); cConcatResults.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units * units, optimization, iBatch)) ReturnFalse; cScores.SetHeads(units); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!MatMul(cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 2; if(!cScores.Init(0, index, OpenCL, cGraphs.Neurons(), optimization, iBatch)) ReturnFalse; cScores.SetHeads(cValue.GetUnits()); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units, units, sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!SparseMatMul(cScores.GetIndexes(), cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormalize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!SparseMatMulGrad(cScores.GetIndexes(), cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSparseGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScores.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 3; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralongerGraph::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint experts, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronGraphAttention *att = NULL; CNeuronGlobLocGraphAtt *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); for(uint i = 0; i < layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; for(uint i = 0; i < att_layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, experts, sparse, emb_dimension, uint(sparse * (dimension + emb_dimension)), optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, variables, dimension + emb_dimension, experts, sparse, emb_dimension, m_units, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cState.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint dimension, uint dimension_k, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConv::Init(numOutputs, myIndex, open_cl, heads * dimension_k, heads * dimension_k, dimension, variables, 1, optimization_type, batch)) ReturnFalse; //--- iDimensionK = dimension_k; iHeads = heads; iVariables = variables; //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, dimension, dimension, 3 * iHeads * iDimensionK, iVariables, 1, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(SIGMOID); index++; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cGate.Init(0, index, OpenCL, dimension, dimension, iDimensionK * iHeads, iVariables, 1, optimization, iBatch)) ReturnFalse; cGate.SetActivationFunction(SIGMOID); index++; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cGate.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!DeConcat(cQ.getOutput(), cK.getOutput(), cV.getOutput(), cQKV.getOutput(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; //--- if(!MatMul(cK.getOutput(), cV.getOutput(), cKV.getOutput(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!cState.SwapOutputs()) ReturnFalse; if(!DiagMatMul(cGate.getOutput(), cState.getPrevOutput(), cState.getOutput(), iDimensionK, iDimensionK, iHeads * iVariables, cState.Activation())) ReturnFalse; if(!SumAndNormalize(cState.getOutput(), cKV.getOutput(), cState.getOutput(), iDimensionK, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cQ.getOutput(), cState.getOutput(), cMHAttention.getOutput(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!CNeuronSpikeConv::feedForward(cMHAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConv::calcInputGradients(cMHAttention.AsObject())) ReturnFalse; //--- if(!MatMulGrad(cQ.getOutput(), cQ.getGradient(), cState.getOutput(), cState.getGradient(), cMHAttention.getGradient(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!DeActivation(cKV.getOutput(), cKV.getGradient(), cState.getGradient(), cKV.Activation())) ReturnFalse; if(!DiagMatMulGrad(cGate.getOutput(), cGate.getGradient(), cState.getPrevOutput(), cKV.getPrevOutput(), cState.getGradient(), iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cGate) if(!MatMulGrad(cK.getOutput(), cK.getGradient(), cV.getOutput(), cV.getGradient(), cKV.getGradient(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!Concat(cQ.getGradient(), cK.getGradient(), cV.getGradient(), cQKV.getGradient(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cQKV) //--- if(!NeuronOCL.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGate.AsObject())) ReturnFalse; if(!SumAndNormalize(PrevOutput, temp, temp, GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGate.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConv::updateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGateLineAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cGate.WeightsUpdate(Source.cGate.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Save(const int file_handle) { if(!CNeuronSpikeConv::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cGate.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iDimensionK) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Load(const int file_handle) { if(!CNeuronSpikeConv::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGate.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iDimensionK = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- int index = 1; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index += 2; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); cState.Clear(); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGateLineAttention::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConv::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cGate.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cV.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cState.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window, 2 * window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFields() * cKV_Embedding.GetFields() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Attention(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFields(); uint total_k = cKV_Embedding.GetFields(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, total_k, iHeads}; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFAT; setBuffer(kernel, def_k_mhfat_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_mhfat_dimension, dimension) setArgument(kernel, def_k_mhfat_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::AttentionGradients(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFields(); uint total_k = cKV_Embedding.GetFields(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, (uint)MathMin(MathMax(total_k, MathMax(total_q, dimension)), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFATGrad; setBuffer(kernel, def_k_mhfat_gr_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_q_gr, cQ_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_kv_gr, cKV_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_scale_gr, cScale.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_gr_gradients, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_mhfat_gr_dimension, dimension) setArgument(kernel, def_k_mhfat_gr_total_k, total_k) setArgument(kernel, def_k_mhfat_gr_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cKV_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cQ_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cScale.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iHeads), INT_VALUE) < INT_VALUE) ReturnFalse; if(!cQ_Embedding.Save(file_handle)) ReturnFalse; if(!cKV_Embedding.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(!LoadInsideLayer(file_handle, cQ_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; uint index = 3; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index += 2; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHFAT* Source = source; dWeightsUpdate(cQ_Embedding, Source, tau); dWeightsUpdate(cKV_Embedding, Source, tau); dWeightsUpdate(cScale, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFAT::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL && ibScoreIndex >= 0) { OpenCL.BufferFree(ibScoreIndex); ibScoreIndex = INVALID_HANDLE; } //--- CNeuronMSRes::SetOpenCL(obj); //--- cQ_Embedding.SetOpenCL(OpenCL); cKV_Embedding.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFAT::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); //--- cQ_Embedding.TrainMode(bTrain); cKV_Embedding.TrainMode(bTrain); cScale.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint window_cross, uint fields_cross, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window_cross, 2 * window_key * iHeads, fields_cross, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFields() * cKV_Embedding.GetFields() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- index++; if(!cContext.Init(0, index, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFields(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(cContext.getOutput() != Context) if(!cContext.SetOutput(Context, true)) ReturnFalse; //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(cContext.AsObject())) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondGradient) ReturnFalse; if(cContext.getGradient() != SecondGradient) if(!cContext.SetGradient(SecondGradient, true)) ReturnFalse; cContext.SetActivationFunction(SecondActivation); //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cContext.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Load(const int file_handle) { if(!CNeuronMHFAT::Load(file_handle)) ReturnFalse; if(!cContext.Init(0, 6, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFields(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHCrossFAT::SetOpenCL(COpenCLMy *obj) { CNeuronMHFAT::SetOpenCL(obj); cContext.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint embed_size, uint units_in, uint groups, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, embed_size * topK * groups, optimization_type, batch)) ReturnFalse; activation = None; //--- iEmbedingSize = embed_size; iTopK = topK; iUnits = units_in; iGroups = groups; //--- cDNN.Clear(); cW.Clear(); cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); //--- CNeuronFieldAwareConv* conv = NULL; CFieldAwareParams* params = NULL; CNeuronSparseSoftMax* softmax = NULL; uint index = 0; //--- DNN conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, window_in, 2 * embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * embed_size, embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); //--- W index++; params = new CFieldAwareParams(); if(!params || !params.Init(0, index, OpenCL, units_in, groups, (embed_size + 3) / 4, candidates, (topK + 2) / 3, optimization, iBatch) || !cW.Add(params)) DeleteObjAndFalse(params); params.SetActivationFunction(None); index++; softmax = new CNeuronSparseSoftMax(); if(!softmax || !softmax.Init(0, index, OpenCL, groups, units_in, topK, optimization, iBatch) || !cW.Add(softmax)) DeleteObjAndFalse(softmax); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).FeedForward()) ReturnFalse; } else if(!curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; if(!SparseConcatenate(softmax.GetIndexes(), softmax.getOutput(), cDNN[-1].getOutput(), Output, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cW[-1] || !cDNN[-1] || cW[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cW[-1]; if(!SparseConcatenateGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cDNN[-1].getOutput(), cDNN[-1].getGradient(), Gradient, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; Deactivation(cDNN[-1]); //--- CNeuronBaseOCL* next = cDNN[-1]; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = cDNN.Total() - 2; i >= 0; i--) { curr = cDNN[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- W next = cW[-1]; for(int i = cW.Total() - 2; i >= 0; i--) { curr = cW[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).UpdateInputWeights()) ReturnFalse; } else if(!curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iEmbedingSize); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, iUnits); dFileWriteUInt(file_handle, iGroups); //--- if(!cDNN.Save(file_handle)) ReturnFalse; if(!cW.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iEmbedingSize); dFileReadUInt(file_handle, iTopK); dFileReadUInt(file_handle, iUnits); dFileReadUInt(file_handle, iGroups); //--- if(!cDNN.Load(file_handle)) ReturnFalse; if(!cW.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAutoToken* Source = source; dWeightsUpdate(cDNN, Source, tau); dWeightsUpdate(cW, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cDNN.ClearStates()) ReturnFalse; if(!cW.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAutoToken::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAutoToken::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cDNN.TrainMode(bTrain); cW.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension < heads || dimension % heads > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * dimension, optimization_type, batch)) ReturnFalse; activation = None; if(!cParams.Init(0, 0, OpenCL, units * units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- if(bTrain) if(!cParams.FeedForward()) ReturnFalse; //--- uint heads = cParams.GetFields(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMul(cParams.getOutput(), NeuronOCL.getOutput(), Output, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), Output, Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint heads = cParams.GetFields(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMulGrad(cParams.getOutput(), cParams.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; dWeightsUpdate(cParams, ((CNeuronMultiMixAttention*)source), tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cAttention.Init(0, index, OpenCL, dimension, units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint expert_dimension = dimension / heads; uint hidden_dimension = MathMax((expert_dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, expert_dimension, hidden_dimension, windows, steps, heads, units, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint dimension = cAttention.GetDimension(); if(!SumAndNormalize(cSharedExpert[-1].getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cAttention.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(cAttention.getOutput(), PrevOutput, Gradient, cAttention.Activation())) ReturnFalse; uint dimension = cAttention.GetDimension(); if(!cAttention.CalcHiddenGradients(cSharedExpert[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormalize(cAttention.getGradient(), PrevOutput, cAttention.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cAttention.Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMTmixAttBlock* Source = source; dWeightsUpdate(cAttention, Source, tau); dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cAttention.SetOpenCL(OpenCL); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cAttention.TrainMode(bTrain); cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint heads, bool mask_future, uint q_units, uint kv_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * heads * q_units, optimization_type, batch)) ReturnFalse; activation = None; //--- if(kv_units < 1) ReturnFalse; //--- iQUnits = q_units; iKVUnits = kv_units; iHeads = heads; iDimension = dimension; bMask = mask_future; //--- if(!cLogSumExp.BufferInit(q_units * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iKVUnits, iDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttention; setBuffer(kernel, def_k_mhflat_q, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhflat_kv, Context.GetIndex()) setBuffer(kernel, def_k_mhflat_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflat_out, getOutputIndex()) setArgument(kernel, def_k_mhflat_dimension, iDimension) setArgument(kernel, def_k_mhflat_total_kv, iKVUnits) setArgument(kernel, def_k_mhflat_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !prevLayer.getOutput() || !prevLayer.getGradient() || !SecondInput || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iKVUnits), iDimension, iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttentionGrad; setBuffer(kernel, def_k_mhflatg_q, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhflatg_q_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhflatg_kv, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhflatg_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflatg_out, getOutputIndex()) setBuffer(kernel, def_k_mhflatg_out_gr, getGradientIndex()) setArgument(kernel, def_k_mhflatg_dimension, iDimension) setArgument(kernel, def_k_mhflatg_total_q, iQUnits) setArgument(kernel, def_k_mhflatg_total_kv, iKVUnits) setArgument(kernel, def_k_mhflatg_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iKVUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iKVUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, bMask); //--- //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCrossMHFlashAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronBaseOCL::SetOpenCL(obj); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint hidden_dimension = MathMax((dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension, hidden_dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, dimension, hidden_dimension, windows, steps, units, 1, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- CNeuronFieldAwareConv* conv = cSharedExpert[-1]; uint dimension = conv.GetFilters(); if(!SumAndNormalize(conv.getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormalize(Output, NeuronOCL.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; CNeuronFieldAwareConv* conv = cSharedExpert[0]; uint dimension = conv.GetWindow(); if(!NeuronOCL.CalcHiddenGradients(conv) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormalize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMixFeedForward* Source = source; dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; //--- if(!CNeuronMixFeedForward::Init(numOutputs, myIndex, open_cl, dimensions[count - 1], units_out, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; CNeuronMultiWindowsConvOCL* mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(None); index++; CNeuronFieldPatternEmbedding* emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(SIGMOID); //--- index++; if(!cSequenceLast.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cSequenceLast.SetActivationFunction(None); index++; if(!cKVSequenceLast.Init(0, index, OpenCL, embed_size, 2 * embed_size * heads, units_s, candidates, topK, optimization, iBatch)) ReturnFalse; cKVSequenceLast.SetActivationFunction(SIGMOID); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cStackKVSequence.Init(0, index, OpenCL, stack_size, 2 * embed_size * heads, units_s, optimization, iBatch)) ReturnFalse; index++; //--- uint units_ns = windows.Size() - units_s; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, (units_ns + stack_size)*embed_size, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; //--- CNeuronAutoToken* select = NULL; CNeuronSparseMoEConv* query = NULL; CCrossMHFlashAttention* attention = NULL; CNeuronSpikeConv* W0 = NULL; CNeuronMTmixAttBlock* ff = NULL; uint units_in = units_ns + stack_size; for(uint i = 0; i < layers; i++) { select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, embed_size, units_in, (layers - i)*units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; units_in = (layers - i) * units_out; query = new CNeuronSparseMoEConv(); if(!query || !query.Init(0, index, OpenCL, embed_size * topK, embed_size * heads, units_in, candidates, topK, optimization, iBatch) || !cFlow.Add(query)) DeleteObjAndFalse(query); query.SetActivationFunction(SIGMOID); index++; attention = new CCrossMHFlashAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, heads, false, units_in, stack_size, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; W0 = new CNeuronSpikeConv(); if(!W0 || !W0.Init(0, index, OpenCL, embed_size * heads, embed_size * heads, embed_size, units_in, 1, optimization, iBatch) || !cFlow.Add(W0)) DeleteObjAndFalse(W0); index++; ff = new CNeuronMTmixAttBlock(); if(!ff || !ff.Init(0, index, OpenCL, embed_size, units_in, heads, embed_size, candidates, topK, optimization, iBatch) || !cFlow.Add(ff)) DeleteObjAndFalse(ff); index++; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!DeConcat(cSequenceLast.getOutput(), prev.getPrevOutput(), prev.getOutput(), cSequenceLast.Neurons(), prev.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.FeedForward(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(curr.getPrevOutput(), cStackSequence.getOutput(), prev.getOutput(), prev.Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMixFeedForward::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CBufferFloat* grad = cStackKVSequence.getGradient(); if(!grad.Fill(0)) ReturnFalse; CBufferFloat* out = cStackKVSequence.getOutput(); CBufferFloat* temp = cStackKVSequence.getPrevOutput(); ENUM_ACTIVATION act = (ENUM_ACTIVATION)cStackKVSequence.Activation(); CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(cFlow[i + 1], out, temp, act)) ReturnFalse; if(cFlow[i + 1].Type() == defCrossMHFlashAttention) if(!SumAndNormalize(temp, grad, grad, 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- curr = cPrepare[-1]; if(!curr || !DeConcat(curr.getPrevOutput(), cStackSequence.getGradient(), cFlow[0].getGradient(), cFlow[0].Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.CalcHiddenGradients(cStackKVSequence.AsObject())) ReturnFalse; if(!cSequenceLast.CalcHiddenGradients(cKVSequenceLast.AsObject())) ReturnFalse; temp = cSequenceLast.getGradient(); if(!cSequenceLast.SetGradient(cSequenceLast.getPrevOutput(), false) || !cSequenceLast.CalcHiddenGradients(cStackSequence.AsObject()) || !SumAndNormalize(temp, cSequenceLast.getGradient(), temp, 1, false, 0, 0, 0, 1) || !cSequenceLast.SetGradient(temp, false)) ReturnFalse; if(!Concat(cSequenceLast.getGradient(), curr.getPrevOutput(), curr.getGradient(), cSequenceLast.Neurons(), curr.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; Deactivation(curr); for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(cPrepare[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cKVSequenceLast.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.UpdateInputWeights(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMixFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronOneTrans* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cKVSequenceLast, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cStackKVSequence, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Save(const int file_handle) { //--- if(!CNeuronMixFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cSequenceLast.Save(file_handle)) ReturnFalse; if(!cKVSequenceLast.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cStackKVSequence.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Load(const int file_handle) { //--- if(!CNeuronMixFeedForward::Load(file_handle)) ReturnFalse; //--- if(!cPrepare.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKVSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackKVSequence.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Clear(void) { //--- if(!CNeuronMixFeedForward::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; if(!cSequenceLast.Clear()) ReturnFalse; if(!cKVSequenceLast.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cStackKVSequence.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOneTrans::SetOpenCL(COpenCLMy *obj) { CNeuronMixFeedForward::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cSequenceLast.SetOpenCL(OpenCL); cKVSequenceLast.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cStackKVSequence.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOneTrans::TrainMode(bool flag) { CNeuronMixFeedForward::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cFlow.TrainMode(bTrain); cSequenceLast.TrainMode(bTrain); cKVSequenceLast.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cStackKVSequence.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Init(numOutputs, myIndex, open_cl, dimension_q, units_q, heads, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iQUnits = units_q; iXUnits = unit_x; iHeads = heads; iXDimension = dimension_x; bMask = false; //--- cPrepareQ.Clear(); cW0.Clear(); cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); //--- Query uint index = 0; uint head_size = (dimension_q + heads - 1) / heads; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_q, dimension_q, heads * head_size, units_q, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, head_size, head_size, dimension_x, units_q * heads, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); //--- W0 index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, dimension_x * units_q * heads, optimization, iBatch) || !cW0.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_x, dimension_x, head_size, units_q * heads, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, heads * head_size, heads * head_size, dimension_q, units_q, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); //--- if(!cLogSumExp.BufferInit(units_q * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !cW0[0] || !cW0[0].getOutput() || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iXUnits, iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCA; setBuffer(kernel, def_k_mhstca_query, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhstca_X, Context.GetIndex()) setBuffer(kernel, def_k_mhstca_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_output, cW0[0].getOutputIndex()) setArgument(kernel, def_k_mhstca_dimension, iXDimension) setArgument(kernel, def_k_mhstca_total_X, iXUnits) setArgument(kernel, def_k_mhstca_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) dActivation(cW0[0]); #ifdef _DEBUG if(!cW0[0].getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !cW0[0] || !cW0[0].getOutput() || !prevLayer.getOutput() || !SecondInput || !cW0[0].getGradient() || !prevLayer.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iXUnits), (uint)MathMin( MathMax( MathMax(iQUnits * iHeads, iXUnits), iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCAGrad; setBuffer(kernel, def_k_mhstca_gr_query, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_query_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhstca_gr_X, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_X_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_output, cW0[0].getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_output_gr, cW0[0].getGradientIndex()) setArgument(kernel, def_k_mhstca_gr_dimension, iXDimension) setArgument(kernel, def_k_mhstca_gr_total_q, iQUnits) setArgument(kernel, def_k_mhstca_gr_total_X, iXUnits) setArgument(kernel, def_k_mhstca_gr_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cW0[0].getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!AttentionOut(prev, Context)) ReturnFalse; prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), prev.getOutput(), prev.Neurons() / iQUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMTmixAttBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronMTmixAttBlock::calcInputGradients(cW0[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cW0.Total() - 2; i >= 0; i--) { curr = cW0[i]; if(!curr || !curr.CalcHiddenGradients(cW0[i + 1])) ReturnFalse; } //--- if(!AttentionInsideGradients(cPrepareQ[-1], SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- for(int i = cPrepareQ.Total() - 2; i >= 0; i--) { curr = cPrepareQ[i]; if(!curr || !curr.CalcHiddenGradients(cPrepareQ[i + 1])) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(curr)) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), cW0[-1].getPrevOutput(), cW0[-1].getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cW0[-1].getPrevOutput(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(prevLayer.getGradient(), cW0[-1].getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronMTmixAttBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Save(const int file_handle) { if(!CNeuronMTmixAttBlock::Save(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iXUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iXDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Load(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Load(file_handle)) ReturnFalse; if(!cW0.Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iXUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iXDimension); uint mask = 0; dFileReadUInt(file_handle, mask); bMask = bool(mask); //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMTmixAttBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHTHCrossAttention* Source = source; dWeightsUpdate(cPrepareQ, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronMTmixAttBlock::SetOpenCL(obj); //--- cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::TrainMode(bool flag) { CNeuronMTmixAttBlock::TrainMode(flag); //--- cPrepareQ.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; uint dimension_out = dimensions[count - 1]; uint units_ns = count - units_s - 1; uint bottleneck = (embed_size + topK - 1) / topK; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck * topK, bottleneck * topK, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Prepare arrays cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvOCL* mwc = NULL; CNeuronFieldPatternEmbedding* emb = NULL; CNeuronAutoToken* select = NULL; CNeuronSwiGLUOCL* swiglu = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMHTHCrossAttention* attention = NULL; //--- Prepare inputs uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(SIGMOID); index++; emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(TANH); //--- Sequence/NonSequence index++; if(!cLastSequence.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cLastSequence.SetActivationFunction(None); index++; if(!cLastNonSequence.Init(0, index, OpenCL, units_ns * embed_size, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cScenarios.Init(0, index, OpenCL, embed_size, scenarios, bottleneck, candidates, topK, optimization_type, batch)) ReturnFalse; cScenarios.SetActivationFunction(TANH); //--- Flow index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (scenarios + units_ns), optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, scenarios + units_ns, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); for(uint l = 0; l < layers; l++) { index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut()*topK, swiglu.GetWindowOut()*topK, embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; attention = new CNeuronMHTHCrossAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, units_out, heads, embed_size, stack_size, bottleneck, candidates, topK, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, embed_size, embed_size, bottleneck, units_out, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); if(l < layers - 1) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (l + 2) * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); } } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * layers * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFields(); uint units_out = GetUnits(); //--- Sequence/NonSequence if(!DeConcat(cLastSequence.getOutput(), cLastNonSequence.getOutput(), prev.getOutput(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; if(!cStackSequence.FeedForward(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.FeedForward()) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(cLastNonSequence.getOutput(), cScenarios.getOutput(), prev.getOutput(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!stack_querys || !prev || !Concat(prev.getOutput(), stack_querys.getOutput(), curr.getOutput(), embedding_size, curr.Neurons() / units_out - embedding_size, units_out)) ReturnFalse; stack_querys = curr; } else if(!curr.FeedForward(prev, cStackSequence.getOutput())) ReturnFalse; if(!stack_querys && curr.Type() == defNeuronMHTHCrossAttention) stack_querys = prev; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) return false; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFields(); uint units_out = GetUnits(); CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Flow for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr) ReturnFalse; if(next.Type() == defNeuronBaseOCL) { if(!DeConcat(curr.getGradient(), next.getPrevOutput(), next.getGradient(), embedding_size, next.Neurons() / (units_ns * scenarios) - embedding_size, units_ns * scenarios)) ReturnFalse; Deactivation(curr); } else if(!curr.CalcHiddenGradients(next, cStackSequence.getOutput(), (!stack_querys ? cStackSequence.getGradient() : cStackSequence.getPrevOutput()), (ENUM_ACTIVATION)cStackSequence.Activation())) ReturnFalse; if(curr.Type() == defNeuronMHTHCrossAttention && !!stack_querys) { if(!SumAndNormalize(cStackSequence.getGradient(), cStackSequence.getPrevOutput(), cStackSequence.getGradient(), embedding_size, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0 && curr.Type() == defNeuronBaseOCL) { if(!!stack_querys) { if(!!stack_querys && !SumAndNormalize(curr.getGradient(), stack_querys.getPrevOutput(), curr.getGradient(), embedding_size, false, 0, 0, 0, 0.5f)) ReturnFalse; } stack_querys = curr; } next = curr; } //--- Sequence/NonSequence if(!DeConcat(cLastNonSequence.getGradient(), cScenarios.getGradient(), cFlow[0].getGradient(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; Deactivation(cLastNonSequence); Deactivation(cScenarios); if(!cLastSequence.CalcHiddenGradients(cStackSequence.AsObject())) ReturnFalse; next = cPrepare[-1]; if(!next || !Concat(cLastSequence.getGradient(), cLastNonSequence.getGradient(), next.getGradient(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; //--- Inputs for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Sequence/NonSequence if(!cStackSequence.UpdateInputWeights(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.UpdateInputWeights()) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastSequence.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSTCA* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastSequence, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cLastSequence.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cScenarios.Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTCA::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cLastSequence.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTCA::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cLastSequence.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint scenarios, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, (window + 3) / 4, (window + 3) / 4, window_out, 1, scenarios, optimization_type, batch)) ReturnFalse; if(!cInputProjection.Init(0, 0, OpenCL, window, window, iWindow, GetUnits(), iVariables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cInputProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronConvOCL::feedForward(cInputProjection.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronConvOCL::calcInputGradients(cInputProjection.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cInputProjection.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cInputProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronConvOCL::updateInputWeights(cInputProjection.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cInputProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cInputProjection.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronWeightGenerator::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronWeightGenerator* Source = source; dWeightsUpdate(cInputProjection, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronWeightGenerator::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cInputProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronWeightGenerator::TrainMode(bool flag) { CNeuronConvOCL::TrainMode(flag); cInputProjection.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint querys, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeRCDOCL::Init(numOutputs, myIndex, open_cl, scenarios, querys, window_out, optimization_type, batch)) ReturnFalse; activation = None; //--- iWindowIn = window; iWindowOut = window_out; iInsideDimension = (iWindowOut + 3) / 4; iQuerys = querys; iScenarios = scenarios; //--- uint index = 0; if(!cShared.Init(0, index, OpenCL, iWindowIn, iInsideDimension, iQuerys, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cShared.SetActivationFunction(TANH); index++; if(!cScenarios.Init(0, index, OpenCL, iInsideDimension, iScenarios, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(TANH); index++; if(!cQueryVsScenarios.Init(0, index, OpenCL, 2 * iInsideDimension * iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cQueryVsScenarios.SetActivationFunction(None); index++; if(!cWQscenarios.Init(0, index, OpenCL, 2 * iInsideDimension, iWindowIn * iInsideDimension, iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cWQscenarios.SetActivationFunction(None); index++; if(!cGeneratedQuery.Init(0, index, OpenCL, iInsideDimension * iQuerys * iScenarios, optimization, iBatch)) ReturnFalse; cGeneratedQuery.SetActivationFunction(TANH); index++; if(!cQScInToScQIn.Init(0, index, OpenCL, iQuerys, iScenarios, iInsideDimension, optimization, iBatch)) ReturnFalse; cQScInToScQIn.SetActivationFunction((ENUM_ACTIVATION)cGeneratedQuery.Activation()); index++; if(!cWKshared.Init(0, index, OpenCL, iWindowOut, iInsideDimension, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cWKshared.SetActivationFunction(None); index++; if(!cWKscenarios.Init(0, index, OpenCL, iInsideDimension, iInsideDimension * iWindowOut, iScenarios, optimization, iBatch)) ReturnFalse; cWKscenarios.SetActivationFunction(SIGMOID); index++; if(!cWKscT.Init(0, index, OpenCL, iScenarios, cWKshared.Neurons(), optimization, iBatch)) ReturnFalse; cWKscT.SetActivationFunction((ENUM_ACTIVATION)cWKscenarios.Activation()); index++; if(!cWKT.Init(0, index, OpenCL, cWKscT.Neurons(), optimization, iBatch)) ReturnFalse; cWKT.SetActivationFunction(None); index++; if(!cWK.Init(0, index, OpenCL, cWKshared.Neurons(), iScenarios, optimization, iBatch)) ReturnFalse; cWK.SetActivationFunction((ENUM_ACTIVATION)cWKT.Activation()); index++; if(!cQueryMod.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cQueryMod.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Shared Query if(!cShared.FeedForward(NeuronOCL)) ReturnFalse; //--- Generator if(bTrain) { if(!cScenarios.FeedForward()) ReturnFalse; if(!cWKshared.FeedForward()) ReturnFalse; } //--- Private Query if(!ConcatVecMatrix(cShared.getOutput(), cScenarios.getOutput(), cQueryVsScenarios.getOutput(), iInsideDimension, iScenarios, iInsideDimension, iQuerys, false)) ReturnFalse; if(!cWQscenarios.FeedForward(cQueryVsScenarios.AsObject())) ReturnFalse; if(!MatMul(cWQscenarios.getOutput(), NeuronOCL.getOutput(), cGeneratedQuery.getOutput(), iInsideDimension * iScenarios, iWindowIn, 1, iQuerys, true)) ReturnFalse; dActivation(cGeneratedQuery); //--- Shared + Private if(!SumVecMatrix(cShared.getOutput(), cGeneratedQuery.getOutput(), cGeneratedQuery.getOutput(), iInsideDimension, iQuerys, 0, 0, 0, 1)) ReturnFalse; if(!Normalize(cGeneratedQuery.getOutput(), iInsideDimension)) ReturnFalse; //--- W Key if(!cWKscenarios.FeedForward(cScenarios.AsObject())) ReturnFalse; if(!cWKscT.FeedForward(cWKscenarios.AsObject())) ReturnFalse; if(!ScalarToVector(cWKshared.getOutput(), cWKscT.getOutput(), cWKT.getOutput(), iScenarios)) ReturnFalse; if(!cWK.FeedForward(cWKT.AsObject())) ReturnFalse; //--- Modify Query if(!cQScInToScQIn.FeedForward(cGeneratedQuery.AsObject())) ReturnFalse; if(!MatMul(cQScInToScQIn.getOutput(), cWK.getOutput(), cQueryMod.getOutput(), iQuerys, iInsideDimension, iWindowOut, iScenarios, true)) ReturnFalse; if(!Normalize(cQueryMod.getOutput(), iWindowOut)) ReturnFalse; if(!CNeuronTransposeRCDOCL::feedForward(cQueryMod.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Modify Query if(!CNeuronTransposeRCDOCL::calcInputGradients(cQueryMod.AsObject())) ReturnFalse; if(!MatMulGrad(cQScInToScQIn.getOutput(), cQScInToScQIn.getGradient(), cWK.getOutput(), cWK.getGradient(), cQueryMod.getGradient(), iQuerys, iInsideDimension, iWindowOut, iScenarios, true)) ReturnFalse; if(!cGeneratedQuery.CalcHiddenGradients(cQScInToScQIn.AsObject())) ReturnFalse; //--- W Key if(!cWKT.CalcHiddenGradients(cWK.AsObject())) ReturnFalse; if(!ScalarToVectorGrad(cWKshared.getOutput(), cWKshared.getGradient(), cWKscT.getOutput(), cWKscT.getGradient(), cWKT.getGradient(), iScenarios)) ReturnFalse; if(!cWKscenarios.CalcHiddenGradients(cWKscT.AsObject())) ReturnFalse; Deactivation(cWKshared); Deactivation(cWKscenarios); if(!cScenarios.CalcHiddenGradients(cWKscenarios.AsObject())) ReturnFalse; //--- Shared + Private if(!SumVecMatrixGrad(cShared.getGradient(), cGeneratedQuery.getPrevOutput(), cGeneratedQuery.getGradient(), iInsideDimension, iQuerys, 0, 0, 0, 1)) ReturnFalse; //--- Private Query Deactivation(cGeneratedQuery); if(!MatMulGrad(cWQscenarios.getOutput(), cWQscenarios.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), cGeneratedQuery.getGradient(), iInsideDimension * iScenarios, iWindowIn, 1, iQuerys, true)) ReturnFalse; if(!cQueryVsScenarios.CalcHiddenGradients(cWQscenarios.AsObject())) ReturnFalse; if(!ConcatVecMatrixGrad(cShared.getPrevOutput(), cScenarios.getPrevOutput(), cQueryVsScenarios.getGradient(), iInsideDimension, iScenarios, iInsideDimension, iQuerys, false)) ReturnFalse; if(!SumAndNormalize(cShared.getGradient(), cShared.getPrevOutput(), cShared.getGradient(), iInsideDimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cShared); if(cScenarios.Activation() != None) if(!DeActivation(cScenarios.getOutput(), cScenarios.getPrevOutput(), cScenarios.getPrevOutput(), cScenarios.Activation())) ReturnFalse; if(!SumAndNormalize(cScenarios.getGradient(), cScenarios.getPrevOutput(), cScenarios.getGradient(), iInsideDimension, false, 0, 0, 0, 1)) ReturnFalse; //--- Shared Query if(!NeuronOCL.CalcHiddenGradients(cShared.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), iWindowIn, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Shared Query if(!cShared.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cScenarios.UpdateInputWeights()) ReturnFalse; if(!cWKshared.UpdateInputWeights()) ReturnFalse; //--- Private Query if(!cWQscenarios.UpdateInputWeights(cQueryVsScenarios.AsObject())) ReturnFalse; //--- W Key if(!cWKscenarios.UpdateInputWeights(cScenarios.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::Save(const int file_handle) { if(!CNeuronTransposeRCDOCL::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iWindowIn); dFileWriteUInt(file_handle, iWindowOut); dFileWriteUInt(file_handle, iInsideDimension); dFileWriteUInt(file_handle, iQuerys); dFileWriteUInt(file_handle, iScenarios); //--- if(!cShared.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cQueryVsScenarios.Save(file_handle)) ReturnFalse; if(!cWQscenarios.Save(file_handle)) ReturnFalse; if(!cGeneratedQuery.Save(file_handle)) ReturnFalse; if(!cQScInToScQIn.Save(file_handle)) ReturnFalse; if(!cWKshared.Save(file_handle)) ReturnFalse; if(!cWKscenarios.Save(file_handle)) ReturnFalse; if(!cWKscT.Save(file_handle)) ReturnFalse; if(!cWKT.Save(file_handle)) ReturnFalse; if(!cWK.Save(file_handle)) ReturnFalse; if(!cQueryMod.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::Load(const int file_handle) { if(!CNeuronTransposeRCDOCL::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iWindowIn); dFileReadUInt(file_handle, iWindowOut); dFileReadUInt(file_handle, iInsideDimension); dFileReadUInt(file_handle, iQuerys); dFileReadUInt(file_handle, iScenarios); //--- if(!LoadInsideLayer(file_handle, cShared.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryVsScenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWQscenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGeneratedQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQScInToScQIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKshared.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKscenarios.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKscT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWKT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQueryMod.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPCGR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTransposeRCDOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronPCGR* Source = source; //--- Shared Query dWeightsUpdate(cShared, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cWKshared, Source, tau); dWeightsUpdate(cWQscenarios, Source, tau); dWeightsUpdate(cWKscenarios, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPCGR::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeRCDOCL::SetOpenCL(obj); //--- cShared.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cQueryVsScenarios.SetOpenCL(OpenCL); cWQscenarios.SetOpenCL(OpenCL); cGeneratedQuery.SetOpenCL(OpenCL); cQScInToScQIn.SetOpenCL(OpenCL); cWKshared.SetOpenCL(OpenCL); cWKscenarios.SetOpenCL(OpenCL); cWKscT.SetOpenCL(OpenCL); cWKT.SetOpenCL(OpenCL); cWK.SetOpenCL(OpenCL); cQueryMod.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPCGR::TrainMode(bool flag) { CNeuronTransposeRCDOCL::TrainMode(flag); //--- cShared.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cQueryVsScenarios.TrainMode(bTrain); cWQscenarios.TrainMode(bTrain); cGeneratedQuery.TrainMode(bTrain); cQScInToScQIn.TrainMode(bTrain); cWKshared.TrainMode(bTrain); cWKscenarios.TrainMode(bTrain); cWKscT.TrainMode(bTrain); cWKT.TrainMode(bTrain); cWK.TrainMode(bTrain); cQueryMod.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; uint dimension_out = dimensions[count - 1]; uint units_ns = count - units_s - 1; uint bottleneck = uint(0.5 * embed_size * log(topK + 1)); if(bottleneck < 8 * heads || (bottleneck % heads) != 0) bottleneck = MathMax((bottleneck + heads - 1) / heads, 4) * heads; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck * topK, bottleneck * topK, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Prepare arrays cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvOCL* mwc = NULL; CNeuronFieldPatternEmbedding* emb = NULL; CNeuronAutoToken* select = NULL; CNeuronSwiGLUOCL* swiglu = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMHTHCrossAttention* attention = NULL; //--- Prepare inputs uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(SIGMOID); index++; emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(TANH); //--- Sequence/NonSequence index++; if(!cLastSequence.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cLastSequence.SetActivationFunction(None); index++; if(!cLastNonSequence.Init(0, index, OpenCL, units_ns * embed_size, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cQuerys.Init(0, index, OpenCL, embed_size, bottleneck, units_ns, scenarios, bottleneck, candidates, topK, optimization_type, batch)) ReturnFalse; //--- Flow for(uint l = 0; l < layers; l++) { index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; attention = new CNeuronMHTHCrossAttention(); if(!attention || !attention.Init(0, index, OpenCL, bottleneck, units_ns * scenarios, heads, embed_size, stack_size, bottleneck, candidates, topK, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), bottleneck, units_ns, scenarios, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); if(l < layers - 1) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, bottleneck * (l + 2) * units_ns * scenarios, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, bottleneck, bottleneck, neuron.Neurons() / bottleneck, units_ns * scenarios, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck * topK, bottleneck * topK, bottleneck, scenarios, units_ns, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); } } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, bottleneck * layers * units_ns * scenarios, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, bottleneck, bottleneck, neuron.Neurons() / bottleneck, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cQuerys.GetCount(); uint bottleneck = cQuerys.GetDimension(); uint units_out = GetUnits(); //--- Sequence/NonSequence if(!DeConcat(cLastSequence.getOutput(), cLastNonSequence.getOutput(), prev.getOutput(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; if(!cStackSequence.FeedForward(cLastSequence.AsObject())) ReturnFalse; //--- PCRG if(!cQuerys.FeedForward(cLastNonSequence.AsObject())) ReturnFalse; //--- Flow prev = cQuerys.AsObject(); stack_querys = prev; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!stack_querys || !prev || !Concat(prev.getOutput(), stack_querys.getOutput(), curr.getOutput(), bottleneck, curr.Neurons() / (units_ns * scenarios) - bottleneck, units_ns * scenarios)) ReturnFalse; stack_querys = curr; } else if(!curr.FeedForward(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) return false; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cQuerys.GetCount(); uint bottleneck = cQuerys.GetDimension(); uint units_out = GetUnits(); //--- CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Flow for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr) ReturnFalse; if(next.Type() == defNeuronBaseOCL) { if(!DeConcat(curr.getGradient(), next.getPrevOutput(), next.getGradient(), bottleneck, next.Neurons() / (units_ns * scenarios) - bottleneck, units_ns * scenarios)) ReturnFalse; Deactivation(curr); } else if(!curr.CalcHiddenGradients(next, cStackSequence.getOutput(), (!stack_querys ? cStackSequence.getGradient() : cStackSequence.getPrevOutput()), (ENUM_ACTIVATION)cStackSequence.Activation())) ReturnFalse; if(curr.Type() == defNeuronMHTHCrossAttention && !!stack_querys) { if(!SumAndNormalize(cStackSequence.getGradient(), cStackSequence.getPrevOutput(), cStackSequence.getGradient(), embedding_size, false, 0, 0, 0, 0.5f)) ReturnFalse; } if(curr.Type() == defNeuronBaseOCL) { if(!!stack_querys && !SumAndNormalize(curr.getGradient(), stack_querys.getPrevOutput(), curr.getGradient(), bottleneck, false, 0, 0, 0, 0.5f)) ReturnFalse; stack_querys = curr; } next = curr; } //--- PCRG if(!cQuerys.CalcHiddenGradients(cFlow[0])) ReturnFalse; if(!!stack_querys) { if(!DeConcat(stack_querys.getPrevOutput(), cQuerys.getPrevOutput(), stack_querys.getGradient(), bottleneck, bottleneck, units_ns * scenarios)) ReturnFalse; if(cQuerys.Activation() != None && !DeActivation(cQuerys.getOutput(), cQuerys.getPrevOutput(), cQuerys.getPrevOutput(), cQuerys.Activation())) ReturnFalse; if(!SumAndNormalize(cQuerys.getGradient(), cQuerys.getPrevOutput(), cQuerys.getGradient(), bottleneck, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- Sequence/NonSequence if(!cLastNonSequence.CalcHiddenGradients(cQuerys.AsObject())) ReturnFalse; if(!cLastSequence.CalcHiddenGradients(cStackSequence.AsObject())) ReturnFalse; next = cPrepare[-1]; if(!next || !Concat(cLastSequence.getGradient(), cLastNonSequence.getGradient(), next.getGradient(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; //--- Inputs for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- PCRG if(!cQuerys.UpdateInputWeights(cLastNonSequence.AsObject())) ReturnFalse; //--- Flow prev = cQuerys.AsObject(); for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastSequence.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cQuerys.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQuerys.AsObject())) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronADS* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastSequence, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cQuerys, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronADS::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cLastSequence.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cQuerys.Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronADS::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cLastSequence.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cQuerys.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronADS::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cLastSequence.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cQuerys.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFieldAwareConv::Init(numOutputs, myIndex, open_cl, 2 * window, window, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; activation = None; if(!cProj.Init(0, 0, OpenCL, window, CNeuronFieldAwareConv::GetWindow(), GetFields(), embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cProj.SetActivationFunction(SoftPlus); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProj.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::feedForward(cProj.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), Output, Output, GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldAwareConv::calcInputGradients(cProj.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; CBufferFloat* temp = Gradient; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; temp = PrevOutput; } if(!SumAndNormalize(NeuronOCL.getGradient(), temp, NeuronOCL.getGradient(), GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::updateInputWeights(cProj.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Save(const int file_handle) { if(!CNeuronFieldAwareConv::Save(file_handle)) ReturnFalse; if(!cProj.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::Load(const int file_handle) { if(!CNeuronFieldAwareConv::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProj.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenFFN::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldAwareConv::WeightsUpdate(source, tau)) ReturnFalse; CNeuronPerTokenFFN* Source = source; dWeightsUpdate(cProj, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPerTokenFFN::SetOpenCL(COpenCLMy *obj) { CNeuronFieldAwareConv::SetOpenCL(obj); cProj.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPerTokenFFN::TrainMode(bool flag) { CNeuronFieldAwareConv::TrainMode(flag); cProj.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint fields, uint window_out, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFieldPatternEmbedding::Init(numOutputs, myIndex, open_cl, (window_in + 1) / 2, scenarios, window_out, candidates, topK, optimization_type, batch)) ReturnFalse; if(!cScenariosGenerator.Init(0, 0, OpenCL, window_in, (window_in + 1) / 2, fields, scenarios, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cScenariosGenerator.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronFieldPatternEmbedding::feedForward(cScenariosGenerator.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldPatternEmbedding::calcInputGradients(cScenariosGenerator.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cScenariosGenerator.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cScenariosGenerator.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldPatternEmbedding::updateInputWeights(cScenariosGenerator.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Save(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Save(file_handle)) ReturnFalse; if(!cScenariosGenerator.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Load(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenariosGenerator.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldPatternEmbedding::WeightsUpdate(source, tau)) ReturnFalse; CNeuronScenariosToken* Source = source; dWeightsUpdate(cScenariosGenerator, Source, tau) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronScenariosToken::Clear(void) { if(!CNeuronFieldPatternEmbedding::Clear()) ReturnFalse; if(!cScenariosGenerator.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronScenariosToken::SetOpenCL(COpenCLMy *obj) { CNeuronFieldPatternEmbedding::SetOpenCL(obj); cScenariosGenerator.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronScenariosToken::TrainMode(bool flag) { CNeuronFieldPatternEmbedding::TrainMode(flag); cScenariosGenerator.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(count < units) ReturnFalse; //--- if(!CNeuronFieldPatternEmbedding::Init(numOutputs, myIndex, open_cl, embed_size, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- uint index = 0; uint total_windows = 0; for(uint i = 0; i < count; i++) total_windows += dimensions[i]; if(!cNorm.Init(0, index, OpenCL, total_windows, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; if(!cProj.Init(0, index, OpenCL, dimensions, embed_size, 1, 1, optimization, iBatch)) ReturnFalse; cProj.SetActivationFunction(SIGMOID); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cProj.FeedForward(cNorm.AsObject())) ReturnFalse; if(!CNeuronFieldPatternEmbedding::feedForward(cProj.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronFieldPatternEmbedding::calcInputGradients(cProj.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cProj.UpdateInputWeights(cNorm.AsObject())) ReturnFalse; if(!CNeuronFieldPatternEmbedding::updateInputWeights(cProj.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Save(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(!cProj.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::Load(const int file_handle) { if(!CNeuronFieldPatternEmbedding::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProj.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUnifiedTokenizer::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronFieldPatternEmbedding::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronUnifiedTokenizer* Source = source; dWeightsUpdate(cNorm, Source, tau); dWeightsUpdate(cProj, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronUnifiedTokenizer::SetOpenCL(COpenCLMy *obj) { CNeuronFieldPatternEmbedding::SetOpenCL(obj); cNorm.SetOpenCL(OpenCL); cProj.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronUnifiedTokenizer::TrainMode(bool flag) { CNeuronFieldPatternEmbedding::TrainMode(flag); cNorm.TrainMode(bTrain); cProj.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { cLogSumExp.BufferFree(); //--- if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, dimension_q, units_q, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iQUnits = units_q; iXUnits = unit_x; iHeads = heads; iXDimension = dimension_x; //--- cPrepareQ.Clear(); cW0.Clear(); cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); //--- Query uint index = 0; uint head_size = (dimension_q + heads - 1) / heads; { CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension_q, heads * head_size, units_q, embed_size, candidates, topK, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, head_size, dimension_x, units_q * heads, embed_size, candidates, topK, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); } //--- W0 index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, dimension_x * units_q * heads, optimization, iBatch) || !cW0.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_x, dimension_x, head_size, units_q * heads, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, heads * head_size, heads * head_size, dimension_q, units_q, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); //--- if(!cLogSumExp.BufferInit(units_q * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !cW0[0] || !cW0[0].getOutput() || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iXUnits, iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCA; setBuffer(kernel, def_k_mhstca_query, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhstca_X, Context.GetIndex()) setBuffer(kernel, def_k_mhstca_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_output, cW0[0].getOutputIndex()) setArgument(kernel, def_k_mhstca_dimension, iXDimension) setArgument(kernel, def_k_mhstca_total_X, iXUnits) setArgument(kernel, def_k_mhstca_mask_future, int(false)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) dActivation(cW0[0]); #ifdef _DEBUG if(!cW0[0].getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !cW0[0] || !cW0[0].getOutput() || !prevLayer.getOutput() || !SecondInput || !cW0[0].getGradient() || !prevLayer.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iXUnits), (uint)MathMin( MathMax( MathMax(iQUnits * iHeads, iXUnits), iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCAGrad; setBuffer(kernel, def_k_mhstca_gr_query, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_query_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhstca_gr_X, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_X_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_output, cW0[0].getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_output_gr, cW0[0].getGradientIndex()) setArgument(kernel, def_k_mhstca_gr_dimension, iXDimension) setArgument(kernel, def_k_mhstca_gr_total_q, iQUnits) setArgument(kernel, def_k_mhstca_gr_total_X, iXUnits) setArgument(kernel, def_k_mhstca_gr_mask_future, int(false)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!AttentionOut(prev, Context)) ReturnFalse; prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!SumAndNormalize(NeuronOCL.getOutput(), prev.getOutput(), prev.getOutput(), prev.Neurons() / iQUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronPerTokenFFN::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronPerTokenFFN::calcInputGradients(cW0[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cW0.Total() - 2; i >= 0; i--) { curr = cW0[i]; if(!curr || !curr.CalcHiddenGradients(cW0[i + 1])) ReturnFalse; } //--- if(!AttentionInsideGradients(cPrepareQ[-1], SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- for(int i = cPrepareQ.Total() - 2; i >= 0; i--) { curr = cPrepareQ[i]; if(!curr || !curr.CalcHiddenGradients(cPrepareQ[i + 1])) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(curr)) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), cW0[-1].getPrevOutput(), cW0[-1].getGradient(), prevLayer.Activation()) || !SumAndNormalize(prevLayer.getGradient(), cW0[-1].getPrevOutput(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormalize(prevLayer.getGradient(), cW0[-1].getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronPerTokenFFN::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iXUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iXDimension); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); //--- if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Load(file_handle)) ReturnFalse; if(!cW0.Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iXUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iXDimension); //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainAwareAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDomainAwareAttention* Source = source; dWeightsUpdate(cPrepareQ, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDomainAwareAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronPerTokenFFN::SetOpenCL(obj); //--- cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDomainAwareAttention::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); //--- cPrepareQ.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint stack_size, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronDomainAwareAttention::Init(numOutputs, myIndex, open_cl, dimension, units, heads, dimension, stack_size * units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; if(!cStack.Init(0, 0, OpenCL, stack_size, dimension, units, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cStack.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronDomainAwareAttention::feedForward(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!CNeuronDomainAwareAttention::calcInputGradients(prevLayer, cStack.getOutput(), cStack.getGradient(), (ENUM_ACTIVATION)cStack.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!CNeuronDomainAwareAttention::updateInputWeights(NeuronOCL, cStack.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Save(const int file_handle) { if(!CNeuronDomainAwareAttention::Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Load(const int file_handle) { if(!CNeuronDomainAwareAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronDomainAwareAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFeatureSelfIteration* Source = source; dWeightsUpdate(cStack, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFeatureSelfIteration::Clear(void) { if(!CNeuronDomainAwareAttention::Clear()) ReturnFalse; if(!cStack.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFeatureSelfIteration::SetOpenCL(COpenCLMy *obj) { CNeuronDomainAwareAttention::SetOpenCL(obj); cStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFeatureSelfIteration::TrainMode(bool flag) { CNeuronDomainAwareAttention::TrainMode(flag); cStack.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint tasks, uint scenarios, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, dimension, tasks + scenarios, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cTasks.Init(0, index, OpenCL, dimension * tasks, optimization, iBatch)) ReturnFalse; cTasks.SetActivationFunction(None); index++; if(!cScenarios.Init(0, index, OpenCL, dimension * scenarios, optimization, iBatch)) ReturnFalse; cScenarios.SetActivationFunction(None); //--- cSelector.Clear(); cSelector.SetOpenCL(OpenCL); index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension, 2 * dimension, tasks, embed_size, candidates, topK, optimization, iBatch) || !cSelector.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * dimension, scenarios, tasks, embed_size, candidates, topK, optimization, iBatch) || !cSelector.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(None); index++; CNeuronSparseSoftMax* sofmax = new CNeuronSparseSoftMax(); if(!sofmax || !sofmax.Init(0, index, OpenCL, tasks, scenarios, (scenarios + 2) / 3, optimization, iBatch) || !cSelector.Add(sofmax)) DeleteObjAndFalse(sofmax); index++; if(!cScenariosToTask.Init(0, index, OpenCL, dimension * tasks, optimization, iBatch)) ReturnFalse; cScenariosToTask.SetActivationFunction(None); index++; if(!cConcat.Init(0, index, OpenCL, dimension * (tasks + scenarios), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint dimension = GetWindow(); uint tasks = cTasks.Neurons() / dimension; uint scenarios = cScenarios.Neurons() / dimension; //--- if(!DeConcat(cTasks.getOutput(), cScenarios.getOutput(), NeuronOCL.getOutput(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- Selector CNeuronBaseOCL* prev = cTasks.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSelector.Total(); i++) { curr = cSelector[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cScenarios.getOutput(), cScenariosToTask.getOutput(), tasks, softmax.DimensionOut(), scenarios, dimension)) ReturnFalse; //--- if(!SumAndNormalize(cTasks.getOutput(), cScenariosToTask.getOutput(), cScenariosToTask.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(cScenariosToTask.getOutput(), cScenarios.getOutput(), cConcat.getOutput(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- if(!CNeuronPerTokenFFN::feedForward(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint dimension = GetWindow(); uint tasks = cTasks.Neurons() / dimension; uint scenarios = cScenarios.Neurons() / dimension; //--- if(!CNeuronPerTokenFFN::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(cScenariosToTask.getGradient(), cScenarios.getPrevOutput(), cConcat.getGradient(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; //--- Selector if(cSelector[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cSelector[-1]; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cScenarios.getOutput(), cScenarios.getGradient(), cScenariosToTask.getGradient(), tasks, softmax.DimensionOut(), scenarios, dimension)) ReturnFalse; CNeuronBaseOCL* curr = NULL; for(int i = cSelector.Total() - 2; i >= 0; i--) { curr = cSelector[i]; if(!curr || !curr.CalcHiddenGradients(cSelector[i + 1])) ReturnFalse; } if(!cTasks.CalcHiddenGradients(cSelector[0])) ReturnFalse; //--- if(!SumAndNormalize(cTasks.getGradient(), cScenariosToTask.getGradient(), cTasks.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormalize(cScenarios.getGradient(), cScenarios.getPrevOutput(), cScenarios.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!Concat(cTasks.getGradient(), cScenarios.getGradient(), NeuronOCL.getGradient(), dimension * tasks, dimension * scenarios, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Selector CNeuronBaseOCL* prev = cTasks.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSelector.Total(); i++) { curr = cSelector[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronPerTokenFFN::updateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; //--- if(!cTasks.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cSelector.Save(file_handle)) ReturnFalse; if(!cScenariosToTask.Save(file_handle)) ReturnFalse; if(!cConcat.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::Load(const int file_handle) { if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTasks.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cSelector.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenariosToTask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDomainFused::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDomainFused* Source = source; dWeightsUpdate(cSelector, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDomainFused::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenFFN::SetOpenCL(obj); //--- cTasks.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cSelector.SetOpenCL(OpenCL); cScenariosToTask.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDomainFused::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); //--- cTasks.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cSelector.TrainMode(bTrain); cScenariosToTask.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPerTokenFFN::Init(numOutputs, myIndex, open_cl, window, units, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; cProj.SetActivationFunction(None); uint index = 1; if(!cGate.Init(0, index, OpenCL, window, CNeuronFieldAwareConv::GetWindow(), GetFields(), embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cGate.SetActivationFunction(SoftPlus); index++; if(!cAfterGate.Init(0, index, OpenCL, cGate.Neurons(), optimization, iBatch)) ReturnFalse; cAfterGate.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProj.FeedForward(NeuronOCL)) ReturnFalse; if(!cGate.FeedForward(NeuronOCL)) ReturnFalse; if(!ElementMult(cProj.getOutput(), cGate.getOutput(), cAfterGate.getOutput())) ReturnFalse; if(!CNeuronFieldAwareConv::feedForward(cAfterGate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronFieldAwareConv::calcInputGradients(cAfterGate.AsObject())) ReturnFalse; if(!ElementMultGrad(cProj.getOutput(), cProj.getGradient(), cGate.getOutput(), cGate.getGradient(), cAfterGate.getGradient(), cProj.Activation(), cGate.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cProj.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false) || !NeuronOCL.CalcHiddenGradients(cGate.AsObject()) || !SumAndNormalize(temp, NeuronOCL.getGradient(), temp, GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGate.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronFieldAwareConv::updateInputWeights(cAfterGate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Save(const int file_handle) { if(!CNeuronPerTokenFFN::Save(file_handle)) ReturnFalse; if(!cGate.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::Load(const int file_handle) { if(!CNeuronPerTokenFFN::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGate.AsObject())) ReturnFalse; if(!cAfterGate.Init(0, 2, OpenCL, cGate.Neurons(), optimization, iBatch)) ReturnFalse; cAfterGate.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPerTokenSwiGLU::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronPerTokenFFN::WeightsUpdate(source, tau)) ReturnFalse; CNeuronPerTokenSwiGLU* Source = source; dWeightsUpdate(cGate, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPerTokenSwiGLU::SetOpenCL(COpenCLMy *obj) { CNeuronPerTokenFFN::SetOpenCL(obj); cGate.SetOpenCL(OpenCL); cAfterGate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPerTokenSwiGLU::TrainMode(bool flag) { CNeuronPerTokenFFN::TrainMode(flag); cGate.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint blocks, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cNormYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNormYIn.SetActivationFunction(None); index++; if(!cSumXYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cSumXYIn.SetActivationFunction(None); index++; if(!cMixer.Init(0, index, OpenCL, window, units, blocks, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; index++; if(!cYOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cYOut.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Type() != Type()) { if(!NeuronOCL.getOutput() || !feedForward(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; } else { CNeuronSiameseNorm* second = NeuronOCL; if(!second.GetYOut() || !second.GetYOut().getOutput() || !feedForward(NeuronOCL, second.GetYOut().getOutput())) ReturnFalse; } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; if(!Context) return feedForward(NeuronOCL); if(Context.Total() < Neurons()) ReturnFalse; //--- uint dimension = cMixer.GetWindow(); uint units = cMixer.GetFields(); //--- Norm(Y) if(!Concat(Context, cNormYIn.getOutput(), cNormYIn.getOutput(), dimension, 0, units)) ReturnFalse; if(!Normalize(cNormYIn.getOutput(), dimension)) ReturnFalse; //--- X + Norm(Y) if(!SumAndNormalize(NeuronOCL.getOutput(), cNormYIn.getOutput(), cSumXYIn.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- Mixing if(!cMixer.FeedForward(cSumXYIn.AsObject())) ReturnFalse; //--- X if(!SumAndNormalize(NeuronOCL.getOutput(), cMixer.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Y if(!SumAndNormalize(Context, cMixer.getOutput(), cYOut.getOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; if(prevLayer.Type() == Type()) { CNeuronSiameseNorm* obj = prevLayer; CNeuronBaseOCL* second = obj.GetYOut(); if(!second.getOutput() || !second.getGradient() || !calcInputGradients(prevLayer, second.getOutput(), second.getGradient(), (ENUM_ACTIVATION)second.Activation())) ReturnFalse; } else { if(!prevLayer.getOutput() || !calcInputGradients(prevLayer, prevLayer.getOutput(), PrevOutput, (ENUM_ACTIVATION)prevLayer.Activation())) ReturnFalse; if(!SumAndNormalize(prevLayer.getOutput(), PrevOutput, prevLayer.getOutput(), cMixer.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!SecondGradient) return calcInputGradients(prevLayer); //--- if(!prevLayer || prevLayer.Neurons() < Neurons() || SecondGradient.Total() < Neurons()) ReturnFalse; //--- uint dimension = cMixer.GetWindow(); uint units = cMixer.GetFields(); //--- Mixing if(!SumAndNormalize(Gradient, cYOut.getGradient(), cMixer.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cMixer); if(!cSumXYIn.CalcHiddenGradients(cMixer.AsObject())) ReturnFalse; if(!SumAndNormalize(cSumXYIn.getGradient(), Gradient, cSumXYIn.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- X if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), cSumXYIn.getGradient(), prevLayer.Activation())) ReturnFalse; //--- Y if(!DeActivation(SecondInput, SecondGradient, cSumXYIn.getGradient(), SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Type() != Type()) { if(updateInputWeights(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; } else { CNeuronSiameseNorm* second = NeuronOCL; if(!updateInputWeights(NeuronOCL, second.GetYOut().getOutput())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!cMixer.UpdateInputWeights(cSumXYIn.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSiameseNorm* Source = source; dWeightsUpdate(cMixer, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cMixer.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSiameseNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMixer.AsObject())) ReturnFalse; //--- uint index = 0; if(!cNormYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNormYIn.SetActivationFunction(None); index++; if(!cSumXYIn.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cSumXYIn.SetActivationFunction(None); index += 2; if(!cYOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cYOut.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSiameseNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cMixer.SetOpenCL(OpenCL); cNormYIn.SetOpenCL(OpenCL); cSumXYIn.SetOpenCL(OpenCL); cYOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSiameseNorm::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cMixer.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint heads, uint dictionary_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(heads < 1 || dictionary_size < 2) ReturnFalse; if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, 2 * window, 2 * window, window, units, 1, optimization_type, batch)) ReturnFalse; //--- iDictionarySize = dictionary_size; iHeads = heads; iHiddenDimension = (window + heads - 1) / heads; //--- uint index = 0; if(!cDictionaryKeys.Init(0, index, OpenCL, (iDictionarySize * iHiddenDimension * iHeads), optimization, iBatch)) ReturnFalse; cDictionaryKeys.SetActivationFunction(None); index++; if(!cDictionaryValues.Init(0, index, OpenCL, cDictionaryKeys.Neurons(), optimization, iBatch)) ReturnFalse; cDictionaryValues.SetActivationFunction(None); index++; if(!cPrototypes.Init(0, index, OpenCL, cDictionaryKeys.Neurons(), optimization, iBatch)) ReturnFalse; cPrototypes.SetActivationFunction(None); CBufferFloat* buf = cPrototypes.getWeightsParams(); if(!buf) ReturnFalse; buf.Fill(0); index++; if(!cLevels.Init(0, index, OpenCL, units * iHeads, optimization, iBatch)) ReturnFalse; cLevels.SetActivationFunction(SIGMOID); buf = cLevels.getWeightsParams(); if(!buf) ReturnFalse; buf.Fill(-3); bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); if(bufLogSumExp <= 0) ReturnFalse; index++; if(!cQuerys.Init(0, index, OpenCL, window, window, iHeads * iHiddenDimension, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); index++; if(!cSimilarity.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cSimilarity.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, iHiddenDimension * iHeads, iHiddenDimension * iHeads, window, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cProjSimilarity.Init(0, index, OpenCL, iHiddenDimension * iHeads, iHiddenDimension * iHeads, window, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cFeedForward0.Init(0, index, OpenCL, window, window, cConv.GetWindow(), units, 1, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQuerys.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) { if(!cDictionaryKeys.FeedForward() || !cDictionaryValues.FeedForward() || !cPrototypes.FeedForward() || !cLevels.FeedForward()) ReturnFalse; } if(!MHCrossAttvsSim()) ReturnFalse; if(!cW0.FeedForward(cAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getOutput(), cW0.getOutput(), cW0.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cProjSimilarity.FeedForward(cSimilarity.AsObject())) ReturnFalse; if(!cFeedForward0.FeedForward(cW0.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cFeedForward0.AsObject())) ReturnFalse; if(!SumAndNormalize(getOutput(), cW0.getOutput(), getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFeedForward0.AsObject())) ReturnFalse; if(!cW0.CalcHiddenGradients(cFeedForward0.AsObject())) ReturnFalse; if(!SumAndNormalize(getGradient(), cW0.getGradient(), cW0.getGradient(), cW0.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!MHCrossAttvsSimGrad()) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQuerys.AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormalize(NeuronOCL.getGradient(), cW0.getGradient(), NeuronOCL.getGradient(), cQuerys.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), cW0.getPrevOutput(), cW0.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormalize(NeuronOCL.getGradient(), cW0.getPrevOutput(), NeuronOCL.getGradient(), cQuerys.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQuerys.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cDictionaryKeys.UpdateInputWeights()) ReturnFalse; if(!cDictionaryValues.UpdateInputWeights()) ReturnFalse; if(!cPrototypes.UpdateInputWeights()) ReturnFalse; if(!cLevels.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cAttentionOut.AsObject())) ReturnFalse; if(!cProjSimilarity.UpdateInputWeights(cSimilarity.AsObject())) ReturnFalse; if(!cFeedForward0.UpdateInputWeights(cW0.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cFeedForward0.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::MHCrossAttvsSim(void) { if(!OpenCL || bufLogSumExp < 0) ReturnFalse; //--- uint units = cQuerys.GetUnits(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(MathMax(iDictionarySize, iHiddenDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_MHCrossAttvsSim; setBuffer(kernel, def_k_casim_query, cQuerys.getOutputIndex()) setBuffer(kernel, def_k_casim_key, cDictionaryKeys.getOutputIndex()) setBuffer(kernel, def_k_casim_value, cDictionaryValues.getOutputIndex()) setBuffer(kernel, def_k_casim_prototype, cPrototypes.getOutputIndex()) setBuffer(kernel, def_k_casim_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_casim_logsumexp, bufLogSumExp) setBuffer(kernel, def_k_casim_output_at, cAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_casim_similarity, cSimilarity.getOutputIndex()) setArgument(kernel, def_k_casim_dimension, (int)iHiddenDimension) setArgument(kernel, def_k_casim_total_X, (int)iDictionarySize) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cAttentionOut.getOutput().BufferRead()) ReturnFalse; if(!cSimilarity.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::MHCrossAttvsSimGrad(void) { if(!OpenCL || bufLogSumExp < 0) ReturnFalse; //--- uint units = cQuerys.GetUnits(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(MathMax(MathMax(units, iDictionarySize), iHiddenDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_MHCrossAttvsSimGrad; setBuffer(kernel, def_k_casimgr_query, cQuerys.getOutputIndex()) setBuffer(kernel, def_k_casimgr_key, cDictionaryKeys.getOutputIndex()) setBuffer(kernel, def_k_casimgr_value, cDictionaryValues.getOutputIndex()) setBuffer(kernel, def_k_casimgr_prototype, cPrototypes.getOutputIndex()) setBuffer(kernel, def_k_casimgr_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_casimgr_logsumexp, bufLogSumExp) setBuffer(kernel, def_k_casimgr_output_at, cAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_casimgr_similarity, cSimilarity.getOutputIndex()) setBuffer(kernel, def_k_casimgr_grad_output_at, cAttentionOut.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_similarity, cSimilarity.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_query, cQuerys.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_key, cDictionaryKeys.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_value, cDictionaryValues.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_prototype, cPrototypes.getGradientIndex()) setBuffer(kernel, def_k_casimgr_grad_levels, cLevels.getGradientIndex()) setArgument(kernel, def_k_casimgr_dimension, (int)iHiddenDimension) setArgument(kernel, def_k_casimgr_total_X, (int)iDictionarySize) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cQuerys.getGradient().BufferRead()) ReturnFalse; if(!cDictionaryKeys.getGradient().BufferRead()) ReturnFalse; if(!cDictionaryValues.getGradient().BufferRead()) ReturnFalse; if(!cPrototypes.getGradient().BufferRead()) ReturnFalse; if(!cLevels.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- Learned dictionaries (CParams — carry trainable weights) if(!cDictionaryKeys.Save(file_handle)) ReturnFalse; if(!cDictionaryValues.Save(file_handle)) ReturnFalse; if(!cPrototypes.Save(file_handle)) ReturnFalse; if(!cLevels.Save(file_handle)) ReturnFalse; //--- Computation sub-modules with trainable weights if(!cQuerys.Save(file_handle)) ReturnFalse; //--- cAttentionOut and cSimilarity: no trainable params — re-inited on Load if(!cW0.Save(file_handle)) ReturnFalse; if(!cProjSimilarity.Save(file_handle)) ReturnFalse; if(!cFeedForward0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Load(const int file_handle) { //--- Free the transient GPU buffer BEFORE calling the parent, because the parent // may change the OpenCL pointer, making the old handle invalid if(bufLogSumExp >= 0 && !!OpenCL) OpenCL.BufferFree(bufLogSumExp); bufLogSumExp = INVALID_HANDLE; //--- if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- Learned dictionaries (CParams) if(!LoadInsideLayer(file_handle, cDictionaryKeys.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDictionaryValues.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrototypes.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLevels.AsObject())) ReturnFalse; //--- Sub-modules with trainable weights if(!LoadInsideLayer(file_handle, cQuerys.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjSimilarity.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward0.AsObject())) ReturnFalse; //--- Restore architectural parameters from loaded sub-objects // (these fields are not serialised separately) iHeads = cLevels.Neurons() / cQuerys.GetUnits(); iHiddenDimension = (iHeads > 0) ? cQuerys.GetFilters() / iHeads : 0; iDictionarySize = (iHiddenDimension > 0 && iHeads > 0) ? cDictionaryKeys.Neurons() / (iHiddenDimension * iHeads) : 0; //--- Recreate transient buffer (not persisted) if(!OpenCL) ReturnFalse; bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); if(bufLogSumExp < 0) ReturnFalse; //--- cAttentionOut and cSimilarity: no trainable params — re-initialise from dimensions uint index = 5; // position in sub-object sequence (see Init) if(!cAttentionOut.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); index++; if(!cSimilarity.Init(0, index, OpenCL, cQuerys.Neurons(), optimization, iBatch)) ReturnFalse; cSimilarity.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronDictionaryCrossAtt *Source = source; //--- Learned dictionaries dWeightsUpdate(cDictionaryKeys, Source, tau); dWeightsUpdate(cDictionaryValues, Source, tau); dWeightsUpdate(cPrototypes, Source, tau); dWeightsUpdate(cLevels, Source, tau); //--- Computation sub-modules with trainable weights // cAttentionOut and cSimilarity are excluded — no learnable parameters dWeightsUpdate(cQuerys, Source, tau); dWeightsUpdate(cW0, Source, tau); dWeightsUpdate(cProjSimilarity, Source, tau); dWeightsUpdate(cFeedForward0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDictionaryCrossAtt::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- CParams objects (cDictionaryKeys, cDictionaryValues, cPrototypes, cLevels) // are intentionally excluded: their Output buffer IS the learned parameter tensor if(!cQuerys.Clear()) ReturnFalse; if(!cAttentionOut.Clear()) ReturnFalse; if(!cSimilarity.Clear()) ReturnFalse; if(!cW0.Clear()) ReturnFalse; if(!cProjSimilarity.Clear()) ReturnFalse; if(!cFeedForward0.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDictionaryCrossAtt::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- Learned dictionaries cDictionaryKeys.TrainMode(bTrain); cDictionaryValues.TrainMode(bTrain); cPrototypes.TrainMode(bTrain); cLevels.TrainMode(bTrain); //--- Computation sub-modules cQuerys.TrainMode(bTrain); cAttentionOut.TrainMode(bTrain); cSimilarity.TrainMode(bTrain); cW0.TrainMode(bTrain); cProjSimilarity.TrainMode(bTrain); cFeedForward0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDictionaryCrossAtt::SetOpenCL(COpenCLMy *obj) { //--- Free the raw buffer on the OLD context before switching if(bufLogSumExp >= 0 && !!OpenCL) OpenCL.BufferFree(bufLogSumExp); bufLogSumExp = INVALID_HANDLE; //--- Switch OpenCL pointer in self and all sub-objects via parent CNeuronSpikeConvBlock::SetOpenCL(obj); //--- Propagate to every sub-object not covered by the parent cDictionaryKeys.SetOpenCL(OpenCL); cDictionaryValues.SetOpenCL(OpenCL); cPrototypes.SetOpenCL(OpenCL); cLevels.SetOpenCL(OpenCL); cQuerys.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cSimilarity.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cProjSimilarity.SetOpenCL(OpenCL); cFeedForward0.SetOpenCL(OpenCL); //--- Recreate transient buffer on the NEW context if(!!OpenCL && cLevels.Neurons() > 0) bufLogSumExp = OpenCL.AddBuffer(sizeof(float) * cLevels.Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+