/// \file /// \brief NeuroNet.mqh /// Library for creating Neural network for use in MQL5 experts /// \author [DNG](https://www.mql5.com/en/users/dng) /// \copyright Copyright 2019, DNG //+------------------------------------------------------------------+ ///\mainpage NeuronNet /// Library for creating Neural network for use in MQL5 experts. /// - \ref const /// - \ref enums /// - \ref ObjectTypes /// - \ref group1 /// - [Class Hierarchy](hierarchy.html) /// - [Files](files.html) //+------------------------------------------------------------------+ #property copyright "Copyright 2019, DNG" #property link "https://www.mql5.com" #property version "1.00" //+------------------------------------------------------------------+ //| Include standart libraries //+------------------------------------------------------------------+ #include #include #include #include #include //+------------------------------------------------------------------+ // Defines //+------------------------------------------------------------------+ ///\defgroup const Global constants ///@{ #define lr 1.0e-3f ///>19))^(rnd_t^(rnd_t>>8)) //--- uint rnd_x = MathRand(), rnd_y = MathRand(), rnd_z = MathRand(), rnd_w = MathRand(), rnd_t = 0; ///@} #define CreateKernel(id, name) if(!opencl.KernelCreate(id, name)) { \ PrintFormat("Error of create kernell %s: %d line %d", #name, GetLastError(), __LINE__); \ ReturnFalse; } #define setBuffer(kernel, id, buffer) if(!OpenCL.SetArgumentBuffer(kernel, id, buffer)) { \ printf("Error of set parameter kernel %s: %d; line %d", OpenCL.GetKernelName(kernel), GetLastError(), __LINE__); \ ReturnFalse; } #define setArgument(kernel, id, value) if(!OpenCL.SetArgument(kernel, id, value)) { \ printf("Error of set parameter kernel %s: %d; line %d", OpenCL.GetKernelName(kernel), GetLastError(), __LINE__); \ ReturnFalse; } #define kernelExecute(kernel,offset,global) if(!OpenCL.Execute(kernel, global.Size(), offset, global)) { \ string error; \ CLGetInfoString(OpenCL.GetContext(), CL_ERROR_DESCRIPTION, error); \ printf("Error of execution kernel %s %s:\n %s", __FUNCTION__, OpenCL.GetKernelName(kernel), error); \ string s_global=StringFormat("global = {%d",global[0]); \ for(uint i=1;i T1 pow(T2 a, T1 b) { return (T1)MathPow((T1)a, (T1)b); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ union uaLongChar { long data[3]; uchar cdata[24]; }; //+------------------------------------------------------------------+ ///\defgroup ObjectTypes Defines Object types identified ///Used to identify classes in a library ///@{ //+------------------------------------------------------------------+ ///\defgroup arr Arrays ///Used to identify array classes ///\{ #define defArrayConnects 0x7782 ///the link. ///@{ #define def_k_FeedForward 0 ///< Index of #FeedForward kernel #define def_k_ff_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in layer and n - number of outputs (neurons in next layer) #define def_k_ff_matrix_i 1 ///< Inputs tesor #define def_k_ff_matrix_o 2 ///< Output tensor #define def_k_ff_inputs 3 ///< Number of inputs #define def_k_ff_activation 4 ///< Activation type (#ENUM_ACTIVATION) ///@} ///\defgroup neuron_base_gr Gradients Calculation kernels /// Describes the process of gradients calculation for the Neuron Base. ///\details Detailed description on the link. ///@{ #define def_k_CalcOutputGradient 1 ///< Index of Output gradients calculation kernel (#CalcOutputGradient) #define def_k_cog_matrix_t 0 ///< Target tensor #define def_k_cog_matrix_o 1 ///< Output tensor #define def_k_cog_matrix_ig 2 ///< Tensor of gradients at previous layer #define def_k_cog_activation 3 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cog_error 4 ///< Error //--- #define def_k_CalcHiddenGradient 2 ///< Index of Hidden gradients calculation kernel (#CalcHiddenGradient) #define def_k_chg_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_chg_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_chg_matrix_o 2 ///< Output tensor #define def_k_chg_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_chg_outputs 4 ///< Number of outputs #define def_k_chg_activation 5 ///< Activation type (#ENUM_ACTIVATION) ///@} ///\defgroup neuron_base_opt Updating Weights Calculation kernel /// Describes the process of optimization weights for the Neuron Base. ///\details Detailed description on the link. /// For Adam optimization look the link. ///@{ #define def_k_UpdateWeightsMomentum 3 ///< Index SGD optomization Update weights kernel (#UpdateWeightsMomentum) #define def_k_uwm_matrix_w 0 ///< SGD Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwm_matrix_g 1 ///< SGD Tensor of gradients at current layer #define def_k_uwm_matrix_i 2 ///< SGD Inputs tesor #define def_k_uwm_matrix_dw 3 ///< SGD Matrix of delta weights in last correction #define def_k_uwm_inputs 4 ///< SGD Number of inputs #define def_k_uwm_learning_rates 5 ///< SGD Learning rates #define def_k_uwm_momentum 6 ///< SGD Momentum multiplier //--- #define def_k_UpdateWeightsAdam 4 ///< Index Adam optomization Update weights kernel (#UpdateWeightsAdam) #define def_k_uwa_matrix_w 0 ///< Adam Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwa_matrix_g 1 ///< Adam Tensor of gradients at current layer #define def_k_uwa_matrix_i 2 ///< Adam Inputs tesor #define def_k_uwa_matrix_m 3 ///< Adam Matrix of first momentum #define def_k_uwa_matrix_v 4 ///< Adam Matrix of seconfd momentum #define def_k_uwa_inputs 5 ///< Adam Number of inputs #define def_k_uwa_l 6 ///< Adam Learning rates #define def_k_uwa_b1 7 ///< Adam First momentum multiplier #define def_k_uwa_b2 8 ///< Adam Second momentum multiplier //--- #define def_k_UpdateWeightsLS 28 ///< Index Least Squares optomization Update weights kernel (#UpdateWeightsLS) #define def_k_uwls_matrix_w 0 ///< Least Squares Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwls_matrix_g 1 ///< Least Squares Tensor of gradients at current layer #define def_k_uwls_matrix_i 2 ///< Least Squares Inputs tesor #define def_k_uwls_matrix_xg 3 ///< Least Squares Matrix of summ x*g #define def_k_uwls_matrix_xx 4 ///< Least Squares Matrix of summ x*x #define def_k_uwls_inputs 5 ///< Least Squares Number of inputs #define def_k_uwls_l 6 ///< Least Squares Learning rates #define def_k_uwls_update 7 ///< Least Squares Update flag ///@} ///@} //--- ///\defgroup neuron_proof Pooling layer's neuron /// Describes the process for the Neuron of pooling layer. ///@{ ///\defgroup neuron_proof_ff Pooling layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of pooling layer. ///@{ #define def_k_FeedForwardProof 5 ///< Index of the kernel of the Pooling neuron for Feed forward process (#FeedForwardProof) #define def_k_ffp_matrix_i 0 ///< Inputs tesor #define def_k_ffp_matrix_o 1 ///< Output tensor #define def_k_ffp_inputs 2 ///< Number of inputs #define def_k_ffp_window 3 ///< Size of input window #define def_k_ffp_step 4 ///< Step size ///@} //--- ///\defgroup neuron_proof_gr Pooling layer's neuron Gradients Calculation kernels /// Describes the gradient calculation process for the Neuron of pooling layer. ///@{ #define def_k_CalcInputGradientProof 6 ///< Index of the kernel of the Pooling neuron to transfer gradient to previous layer (#CalcInputGradientProof) #define def_k_cigp_matrix_i 0 ///< Inputs tesor #define def_k_cigp_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_cigp_matrix_o 2 ///< Output tensor #define def_k_cigp_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_cigp_outputs 4 ///< Number of outputs #define def_k_cigp_window 5 ///< Size of input window #define def_k_cigp_step 6 ///< Step size ///@} ///@} //--- ///\defgroup neuron_conv Convolution layer's neuron /// Describes the process for the Neuron of convolution layer. ///@{ ///\defgroup neuron_conv_ff Convolution layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of convolution layer. ///@{ #define def_k_FeedForwardConv 7 ///< Index of the kernel of the convolution neuron for Feed forward process (#FeedForwardConv) #define def_k_ffc_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_ffc_matrix_i 1 ///< Inputs tesor #define def_k_ffc_matrix_o 2 ///< Output tensor #define def_k_ffc_inputs 3 ///< Number of inputs #define def_k_ffc_step 4 ///< Step size #define def_k_ffc_window_in 5 ///< Size of input window #define def_k_ffс_window_out 6 ///< Size of output window #define def_k_ffc_activation 7 ///< Activation type (#ENUM_ACTIVATION) ///@} //--- ///\defgroup neuron_conv_gr Convolution layer's neuron Gradients Calculation kernels /// Describes the gradient calculation process for the Neuron of convolution layer. ///@{ #define def_k_CalcHiddenGradientConv 8 ///< Index of the kernel of the convolution neuron to transfer gradient to previous layer (#CalcHiddenGradientConv) #define def_k_chgc_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_chgc_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_chgc_matrix_o 2 ///< Output tensor #define def_k_chgc_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_chgc_outputs 4 ///< Number of outputs #define def_k_chgc_step 5 ///< Step size #define def_k_chgc_window_in 6 ///< Size of input window #define def_k_chgc_window_out 7 ///< Size of output window #define def_k_chgc_activation 8 ///< Activation type (#ENUM_ACTIVATION) #define def_k_chgc_shift_out 9 ///< Activation type (#ENUM_ACTIVATION) ///@} //--- ///\defgroup neuron_conv_opt Convolution layer's neuron Update weights kernels /// Describes the optimization process for the Neuron of convolution layer. ///@{ #define def_k_UpdateWeightsConvMomentum 9 ///< Index of the kernel of the convolution neuron to update weights SGD (#UpdateWeightsConvMomentum) #define def_k_uwcm_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwcm_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwcm_matrix_i 2 ///< Inputs tesor #define def_k_uwcm_matrix_dw 3 ///< Matrix of delta weights in last correction #define def_k_uwcm_inputs 4 ///< Number of inputs #define def_k_uwcm_learning_rates 5 ///< Learning rates #define def_k_uwcm_momentum 6 ///< Momentum multiplier #define def_k_uwcm_window_in 7 ///< Size of input window #define def_k_uwcm_window_out 8 ///< Size of output window #define def_k_uwcm_step 9 ///< Step size //--- #define def_k_UpdateWeightsConvAdam 10 ///< Index of the kernel of the convolution neuron to update weights Adam (#UpdateWeightsConvAdam) #define def_k_uwca_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwca_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwca_matrix_i 2 ///< Inputs tesor #define def_k_uwca_matrix_m 3 ///< Matrix of first momentum #define def_k_uwca_matrix_v 4 ///< Matrix of seconfd momentum #define def_k_uwca_inputs 5 ///< Number of inputs #define def_k_uwca_l 6 ///< Learning rates #define def_k_uwca_b1 7 ///< First momentum multiplier #define def_k_uwca_b2 8 ///< Second momentum multiplier #define def_k_uwca_window_in 9 ///< Size of input window #define def_k_uwca_window_out 10 ///< Size of output window #define def_k_uwca_step 11 ///< Step size //--- #define def_k_UpdateWeightsConvLS 29 ///< Index of the kernel of the convolution neuron to update weights Least Squares(#UpdateWeightsConvLS) #define def_k_uwcls_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwcls_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwcls_matrix_i 2 ///< Inputs tesor #define def_k_uwcls_matrix_xg 3 ///< Matrix of first momentum #define def_k_uwcls_matrix_xx 4 ///< Matrix of seconfd momentum #define def_k_uwcls_inputs 5 ///< Number of inputs #define def_k_uwcls_l 6 ///< Learning rates #define def_k_uwcls_update 7 ///< Update flag #define def_k_uwcls_window_in 8 ///< Size of input window #define def_k_uwcls_window_out 9 ///< Size of output window #define def_k_uwcls_step 10 ///< Step size ///@} ///@} //--- ///\defgroup neuron_atten Attention layer's neuron /// Describes the process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ ///\defgroup neuron_atten_ff Attention layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ #define def_k_AttentionScore 11 ///< Index of the kernel of the attention neuron to calculate score matrix (#AttentionScore) #define def_k_as_querys 0 ///< Matrix of Querys #define def_k_as_keys 1 ///< Matriz of Keys #define def_k_as_score 2 ///< Matrix of Scores #define def_k_as_dimension 3 ///< Dimension of Key #define def_k_as_mask 4 ///< 1 - calc only previous units, 0 - calc all //--- #define def_k_AttentionOut 12 ///< Index of the Attention Neuron Output calculation kernel (#AttentionOut) #define def_k_aout_scores 0 ///< Matrix of Scores #define def_k_aout_values 1 ///< Matrix of Values #define def_k_aout_inputs 2 ///< Inputs tesor #define def_k_aout_out 3 ///< Output tesor //--- #define def_k_MatrixSum 13 ///< Index of the kernel for calculation Sum of 2 matrix with multiplyer (#SumMatrix) #define def_k_sum_matrix1 0 ///< First matrix #define def_k_sum_matrix2 1 ///< Second matrix #define def_k_sum_matrix_out 2 ///< Output matrix #define def_k_sum_dimension 3 ///< Dimension of matrix #define def_k_sum_multiplyer 4 ///< Multiplyer for output #define def_k_sum_shift_in1 5 #define def_k_sum_shift_in2 6 #define def_k_sum_shift_out 7 //--- #define def_k_Matrix5Sum 19 ///< Index of the kernel for calculation Sum of 2 matrix with multiplyer (#SumMatrix) #define def_k_sum5_matrix1 0 ///< First matrix #define def_k_sum5_matrix2 1 ///< Second matrix #define def_k_sum5_matrix3 2 ///< Third matrix #define def_k_sum5_matrix4 3 ///< Fourth matrix #define def_k_sum5_matrix5 4 ///< Fifth matrix #define def_k_sum5_matrix_out 5 ///< Output matrix #define def_k_sum5_dimension 6 ///< Dimension of matrix #define def_k_sum5_multiplyer 7 ///< Multiplyer for output //--- #define def_k_MHAttentionScore 20 ///< Index of the kernel of the multi-heads attention neuron to calculate score matrix (#MHAttentionScore) #define def_k_mhas_qkv 0 ///< Matrix of Queries, Keys, Values #define def_k_mhas_score 1 ///< Matrix of Scores #define def_k_mhas_dimension 2 ///< Dimension of Key #define def_k_mhas_mask 3 ///< 1 - calc only previous units, 0 - calc all //--- #define def_k_MHAttentionOut 21 ///< Index of the kernel of the multi-heads attention neuron to calculate multi-heads out matrix (#MHAttentionOut) #define def_k_mhao_score 0 ///< Matrix of Scores #define def_k_mhao_qkv 1 ///< Matrix of Queries, Keys, Values #define def_k_mhao_out 2 ///< Matrix of Outputs #define def_k_mhao_dimension 3 ///< Dimension of Key //--- #define def_k_ConcatenateMatrix 17 ///< Index of the Multi Head Attention Neuron Concatenate Output kernel (#ConcatenateBuffers) #define def_k_conc_input1 0 ///< Matrix of Buffer 1 #define def_k_conc_window1 1 ///< Window of Buffer 1 #define def_k_conc_input2 2 ///< Matrix of Buffer 2 #define def_k_conc_window2 3 ///< Window of Buffer 2 #define def_k_conc_input3 4 ///< Matrix of Buffer 3 #define def_k_conc_window3 5 ///< Window of Buffer 3 #define def_k_conc_input4 6 ///< Matrix of Buffer 4 #define def_k_conc_window4 7 ///< Window of Buffer 4 #define def_k_conc_out 8 ///< Output tesor ///@} //--- ///\defgroup neuron_atten_gr Attention layer's neuron Gradients Calculation /// Describes the gradients calculation process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ #define def_k_AttentionGradients 14 ///< Index of the kernel for gradients calculation process (#AttentionInsideGradients) #define def_k_ag_querys 0 ///< Matrix of Querys #define def_k_ag_querys_g 1 ///< Matrix of Querys' Gradients #define def_k_ag_keys 2 ///< Matrix of Keys #define def_k_ag_keys_g 3 ///< Matrix of Keys' Gradients #define def_k_ag_values 4 ///< Matrix of Values #define def_k_ag_values_g 5 ///< Matrix of Values' Gradients #define def_k_ag_scores 6 ///< Matrix of Scores #define def_k_ag_gradient 7 ///< Matrix of Gradients from previous iteration //--- #define def_k_DeconcatenateMatrix 18 ///< Index of the Multi Head Attention Neuron Deconcatenate Output kernel (#DeconcatenateBuffers) #define def_k_dconc_output1 0 ///< Matrix of Buffer 1 #define def_k_dconc_window1 1 ///< Window of Buffer 1 #define def_k_dconc_output2 2 ///< Matrix of Buffer 2 #define def_k_dconc_window2 3 ///< Window of Buffer 2 #define def_k_dconc_output3 4 ///< Matrix of Buffer 3 #define def_k_dconc_window3 5 ///< Window of Buffer 3 #define def_k_dconc_output4 6 ///< Matrix of Buffer 4 #define def_k_dconc_window4 7 ///< Window of Buffer 4 #define def_k_dconc_inputs 8 ///< Input tesor //--- #define def_k_MHAttentionGradients 22 ///< Index of the kernel for gradients calculation process (#AttentionInsideGradients) #define def_k_mhag_qkv 0 ///< Matrix of Queries, Keys, Values #define def_k_mhag_qkv_g 1 ///< Matrix of Gradients to Queries, Keys, Values #define def_k_mhag_score 2 ///< Matrix of Scores #define def_k_mhag_gradient 3 ///< Matrix of Gradients from previous iteration //--- #define def_k_Dropout 23 ///< Index of the kernel for Dropout process (#Dropout) #define def_k_dout_input 0 ///< Inputs Tensor #define def_k_dout_map 1 ///< Map Tensor #define def_k_dout_out 2 ///< Out Tensor #define def_k_dout_dimension 3 ///< Dimension of Inputs ///@} ///@} //--- ///\defgroup neuron_norm Kernels of matrix normalization process /// Describes the process of matrix normalization. ///\details Detailed description on the link. ///@{ #define def_k_Normilize 15 ///< Index of the kernel for matrix normalization (#Normalize) #define def_k_norm_buffer 0 ///< In/Out Matrix #define def_k_norm_dimension 1 ///< Dimension of matrix //--- #define def_k_NormilizeWeights 16 ///< Index of the kernel for weights matrix normalization (#NormalizeWeights) //--- #define def_k_BatchFeedForward 24 ///< Index of the kernel for Batch Normalization Feed Forward process (#CNeuronBathcNormOCL) #define def_k_bff_inputs 0 ///< Inputs data tenzor #define def_k_bff_options 1 ///< Tenzor of variables #define def_k_bff_output 2 ///< Tenzor of output data #define def_k_bff_batch 3 ///< Batch size #define def_k_bff_optimization 4 ///< Optimization type #define def_k_bff_activation 5 ///< Activation type //--- #define def_k_CalcHiddenGradientBatch 25 ///< Index of the Kernel of the Batch neuron to transfer gradient to previous layer (#CNeuronBathcNormOCL) #define def_k_bchg_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_bchg_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_bchg_matrix_i 2 ///<[in] Tensor of previous layer output #define def_k_bchg_matrix_ig 3 ///<[out] Tensor of gradients at previous layer #define def_k_bchg_activation 4 ///< Activation type (#ENUM_ACTIVATION) #define def_k_bchg_batch 5 ///< Batch size #define def_k_bchg_optimization 6 ///< Optimization type //--- #define def_k_UpdateBatchOptionsMomentum 26 ///< Index of the kernel for Describe the process of SGD optimization options for the Batch normalization Neuron (#CNeuronBatchNormOCL). #define def_k_buom_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_buom_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_buom_learning_rates 2 ///< Learning rates #define def_k_buom_momentum 3 ///< Momentum multiplier //--- #define def_k_UpdateBatchOptionsAdam 27 ///< Index of the kernel for Describe the process of Adam optimization options for the Batch normalization Neuron (#CNeuronBatchNormOCL). #define def_k_buoa_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_buoa_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_buoa_l 2 ///< Learning rates #define def_k_buoa_b1 3 ///< First momentum multiplier #define def_k_buoa_b2 4 ///< Second momentum multiplier ///@} ///\defgroup VAE neuron Kernels of Variant Aoutoencodre /// Describes the process of Variant Aoutoencodre. ///@{ #define def_k_VAEFeedForward 30 #define def_k_vaeff_inputs 0 #define def_k_vaeff_random 1 #define def_k_vaeff_outputd 2 //--- #define def_k_VAECalcHiddenGradient 31 #define def_k_vaehg_input 0 #define def_k_vaehg_inp_grad 1 #define def_k_vaehg_random 2 #define def_k_vaehg_gradient 3 #define def_k_vaehg_kld_mult 4 ///@} ///\defgroup LSTM neuron Kernels of RNN unit /// Describes the process of RNN. ///@{ #define def_k_LSTM_FeedForward 32 #define def_k_lstmff_inputs 0 #define def_k_lstmff_inputs_size 1 #define def_k_lstmff_weights 2 #define def_k_lstmff_concatenated 3 #define def_k_lstmff_memory 4 #define def_k_lstmff_outputs 5 //--- #define def_k_LSTM_ConcatenatedGradient 33 #define def_k_lstmcg_gradient 0 #define def_k_lstmcg_concatenated_gradient 1 #define def_k_lstmcg_memory 2 #define def_k_lstmcg_concatenated 3 //--- #define def_k_LSTM_HiddenGradient 34 #define def_k_lstmhg_concatenated_gradient 0 #define def_k_lstmhg_inputs_gradient 1 #define def_k_lstmhg_weights_gradient 2 #define def_k_lstmhg_hidden_state 3 #define def_k_lstmhg_inputs 4 #define def_k_lstmhg_weeights 5 #define def_k_lstmhg_output 6 #define def_k_lstmhg_hidden_size 7 #define def_k_lstmhg_inputs_size 8 //--- #define def_k_LSTM_UpdateWeightsAdam 35 #define def_k_lstmuw_weights 0 #define def_k_lstmuw_weights_gradient 1 #define def_k_lstmuw_matrix_m 2 #define def_k_lstmuw_matrix_v 3 #define def_k_lstmuw_l 4 #define def_k_lstmuw_b1 5 #define def_k_lstmuw_b2 6 ///@} ///\defgroup SoftMax activation Kernels ///@{ #define def_k_SoftMax_FeedForward 36 #define def_k_softmaxff_inputs 0 #define def_k_softmaxff_outputs 1 //--- #define def_k_SoftMax_HiddenGradient 37 #define def_k_softmaxhg_outputs 0 #define def_k_softmaxhg_output_gr 1 #define def_k_softmaxhg_input_gr 2 //--- #define def_k_SoftMax_OutputGradient 38 #define def_k_softmaxog_outputs 0 #define def_k_softmaxog_targets 1 #define def_k_softmaxog_output_gr 2 ///@} ///\defgroup SoftMax activation Kernels ///@{ #define def_k_FQF_Cosine 39 #define def_k_fqf_cosine_softmax 0 #define def_k_fqf_cosine_outputs 1 //--- #define def_k_FQF_Output 40 #define def_k_fqfout_quantiles 0 #define def_k_fqfout_delta_taus 1 #define def_k_fqfout_output 2 #define def_k_fqfout_total 3 //--- #define def_k_FQF_OutputGradient 41 #define def_k_fqfoutgr_quantiles 0 #define def_k_fqfoutgr_taus 1 #define def_k_fqfoutgr_output_gr 2 #define def_k_fqfoutgr_quantiles_gr 3 #define def_k_fqfoutgr_taus_gr 4 //--- #define def_k_FQF_QuantileGradient 42 #define def_k_fqfqgr_state_enbeding 0 #define def_k_fqfqgr_taus_embedding 1 #define def_k_fqfqgr_quantiles_gr 2 #define def_k_fqfqgr_state_gr 3 #define def_k_fqfqgr_taus_gr 4 //--- #define def_k_FQF_CosineGradient 43 #define def_k_fqfcosgr_softmax 0 #define def_k_fqfcosgr_output_gr 1 #define def_k_fqfcosgr_softmax_gr 2 //--- //--- #define def_k_MHSparseAttentionScore 44 ///< Index of the kernel of the multi-heads sparse attention neuron to calculate score matrix (#MHSparseAttentionScore) #define def_k_mhas_sparse 3 ///< less than 1.0 сoefficient of sparse //--- #define def_k_MHSparseAttentionOut 45 ///< Index of the kernel of the multi-heads sparse attention neuron to calculate multi-heads out matrix (#MHSparseAttentionOut) //--- #define def_k_FFMultiModels 46 ///< Index of the kernel of the multi-models neuron to calculate feed forward #define def_k_HGMultiModels 47 ///< Index of the kernel of the multi-models neuron to calculate hiden gradient #define def_k_chg_model 6 ///< Number of model to calculate #define def_k_UWMultiModels 48 ///< Index of the kernel of the multi-models neuron to update weights #define def_k_uwa_model 9 ///< Number of model to update ///@} ///@} #define def_k_ConcatFeedForward 49 ///< Index of #FeedForward kernel #define def_k_cff_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in layer and n - number of outputs (neurons in next layer) #define def_k_cff_matrix_i1 1 ///< Inputs tesor #define def_k_cff_matrix_i2 2 ///< Inputs tesor #define def_k_cff_matrix_o 3 ///< Output tensor #define def_k_cff_inputs1 4 ///< Number of inputs #define def_k_cff_inputs2 5 ///< Number of inputs #define def_k_cff_activation 6 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_ConcatCalcHiddenGradient 50 ///< Index of Hidden gradients calculation kernel (#CalcHiddenGradient) #define def_k_cchg_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cchg_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_cchg_matrix_o1 2 ///< Output tensor #define def_k_cchg_matrix_o2 3 ///< Output tensor #define def_k_cchg_matrix_ig1 4 ///< Tensor of gradients at previous layer #define def_k_cchg_matrix_ig2 5 ///< Tensor of gradients at previous layer #define def_k_cchg_outputs 6 ///< Number of outputs #define def_k_cchg_inputs1 7 ///< Number of inputs1 #define def_k_cchg_inputs2 8 ///< Number of inputs2 #define def_k_cchg_activation1 9 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cchg_activation2 10 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_ConcatUpdWeightsMomentum 51 ///< Index SGD optomization Update weights kernel (#UpdateWeightsMomentum) #define def_k_cuwm_matrix_w 0 ///< SGD Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cuwm_matrix_g 1 ///< SGD Tensor of gradients at current layer #define def_k_cuwm_matrix_i1 2 ///< SGD Inputs tesor #define def_k_cuwm_matrix_i2 3 ///< SGD Inputs tesor #define def_k_cuwm_matrix_dw 4 ///< SGD Matrix of delta weights in last correction #define def_k_cuwm_inputs1 5 ///< SGD Number of inputs #define def_k_cuwm_inputs2 6 ///< SGD Number of inputs #define def_k_cuwm_learning_rates 7 ///< SGD Learning rates #define def_k_cuwm_momentum 8 ///< SGD Momentum multiplier //--- #define def_k_ConcatUpdWeightsAdam 52 ///< Index Adam optomization Update weights kernel (#UpdateWeightsAdam) #define def_k_cuwa_matrix_w 0 ///< Adam Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cuwa_matrix_g 1 ///< Adam Tensor of gradients at current layer #define def_k_cuwa_matrix_i1 2 ///< Adam Inputs tesor #define def_k_cuwa_matrix_i2 3 ///< Adam Inputs tesor #define def_k_cuwa_matrix_m 4 ///< Adam Matrix of first momentum #define def_k_cuwa_matrix_v 5 ///< Adam Matrix of seconfd momentum #define def_k_cuwa_inputs1 6 ///< Adam Number of inputs #define def_k_cuwa_inputs2 7 ///< Adam Number of inputs #define def_k_cuwa_l 8 ///< Adam Learning rates #define def_k_cuwa_b1 9 ///< Adam First momentum multiplier #define def_k_cuwa_b2 10 ///< Adam Second momentum multiplier //--- #define def_k_SoftUpdate 53 #define def_k_su_target 0 #define def_k_su_source 1 #define def_k_su_tau 2 //--- #define def_k_SoftUpdateAdam 54 #define def_k_sua_target 0 #define def_k_sua_source 1 #define def_k_sua_matrix_m 2 #define def_k_sua_matrix_v 3 #define def_k_sua_tau 4 #define def_k_sua_b1 5 #define def_k_sua_b2 6 //--- #define def_k_SAC_AlphaLogProbs 55 #define def_k_sac_alp_outputs 0 #define def_k_sac_alp_quantiles 1 #define def_k_sac_alp_probs 2 #define def_k_sac_alp_alphas 3 #define def_k_sac_alp_log_probs 4 #define def_k_sac_alp_random 5 #define def_k_sac_alp_count_quants 6 #define def_k_sac_alp_activation 7 //--- #define def_k_SAC_AlphaGradients 56 #define def_k_sac_alg_outputs 0 #define def_k_sac_alg_gradient 1 #define def_k_sac_alg_log_probs 2 #define def_k_sac_alg_alphas_grad 3 #define def_k_sac_alg_activation 4 //--- #define def_k_SAC_OutputGradient 57 #define def_k_sacoutgr_quantiles 0 #define def_k_sacoutgr_taus 1 #define def_k_sacoutgr_output_gr 2 #define def_k_sacoutgr_quantiles_gr 3 #define def_k_sacoutgr_taus_gr 4 #define def_k_sacoutgr_outputs 5 #define def_k_sacoutgr_count_quants 6 #define def_k_sacoutgr_activation 7 //--- #define def_k_SAC_CalcLogProbs 58 #define def_k_sacclp_outputs 0 #define def_k_sacclp_quantiles 1 #define def_k_sacclp_probs 2 #define def_k_sacclp_alphas 3 #define def_k_sacclp_log_probs 4 #define def_k_sacclp_count_quants 5 #define def_k_sacclp_activation 6 //--- #define def_k_Embedding 59 #define def_k_emb_inputs 0 #define def_k_emb_outputs 1 #define def_k_emb_weights 2 #define def_k_emb_windows 3 #define def_k_emb_std 4 #define def_k_emb_stack_size 5 //--- #define def_k_EmbeddingHiddenGradient 60 #define def_k_ehg_inputs_gradient 0 #define def_k_ehg_outputs_gradient 1 #define def_k_ehg_weights 2 #define def_k_ehg_windows 3 #define def_k_ehg_std 4 #define def_k_ehg_window_out 5 //--- #define def_k_EmbeddingUpdateWeightsAdam 61 #define def_k_euw_weights 0 #define def_k_euw_gradient 1 #define def_k_euw_inputs 2 #define def_k_euw_matrix_m 3 #define def_k_euw_matrix_v 4 #define def_k_euw_windows 5 #define def_k_euw_std 6 #define def_k_euw_window_out 7 #define def_k_euw_learning_rate 8 #define def_k_euw_b1 9 #define def_k_euw_b2 10 //--- #define def_k_Transpose 62 #define def_k_tr_matrix_in 0 #define def_k_tr_matrix_out 1 //--- #define def_k_MH2AttentionOut 63 #define def_k_mh2ao_q 0 ///< Matrix of Queries #define def_k_mh2ao_kv 1 ///< Matrix of Keys, Values #define def_k_mh2ao_score 2 ///< Matrix of Scores #define def_k_mh2ao_out 3 ///< Matrix of Outputs #define def_k_mh2ao_dimension 4 ///< Dimension of Key #define def_k_mh2ao_heads_kv 5 ///< Heads of Key #define def_k_mh2ao_mask 6 ///< mask 1 - calc only previous units, 0 - calc all //--- #define def_k_MH2AttentionInsideGradients 64 #define def_k_mh2aig_q 0 ///< Matrix of Queries #define def_k_mh2aig_qg 1 ///< Matrix of Queries gradient #define def_k_mh2aig_kv 2 ///< Matrix of Keys, Values #define def_k_mh2aig_kvg 3 ///< Matrix of Keys, Values gradient #define def_k_mh2aig_score 4 ///< Matrix of Scores #define def_k_mh2aig_outg 5 ///< Matrix of Outputs gradient #define def_k_mh2aig_kunits 6 ///< Size of Key #define def_k_mh2aig_heads_kv 7 ///< Heads of Key //--- #define def_k_CGConv_HiddenGradient 65 #define def_k_cgc_matrix_g 0 ///<[in] Tensor of gradients at current layer #define def_k_cgc_matrix_f 1 ///<[in] Previous layer Output tensor #define def_k_cgc_matrix_s 2 ///<[in] Previous layer Output tensor #define def_k_cgc_matrix_fg 3 ///<[out] Tensor of gradients at previous layer #define def_k_cgc_matrix_sg 4 ///<[out] Tensor of gradients at previous layer #define def_k_cgc_activationf 5 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cgc_activations 6 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_XCiTFeedForward 66 #define def_k_XCiTff_qkv 0 #define def_k_XCiTff_score 1 #define def_k_XCiTff_out 2 //--- #define def_k_XCiTInsideGradients 67 #define def_k_XCiTig_qkv 0 #define def_k_XCiTig_qkv_g 1 #define def_k_XCiTig_scores 2 #define def_k_XCiTig_gradient 3 //--- #define def_k_DOTFeedForward 68 #define def_k_dot_qkv 0 #define def_k_dot_score 1 #define def_k_dot_rpb 2 #define def_k_dot_out 3 //--- #define def_k_DOTInsideGradients 69 #define def_k_dotg_qkv 0 #define def_k_dotg_qkv_g 1 #define def_k_dotg_scores 2 #define def_k_dotg_rpb 3 #define def_k_dotg_rpb_g 4 #define def_k_dotg_gradient 5 //--- #define def_k_RPBUpdateAdam 70 #define def_k_rpbw_rpb 0 #define def_k_rpbw_gradient 1 #define def_k_rpbw_matrix_m 2 #define def_k_rpbw_matrix_v 3 #define def_k_rpbw_b1 4 #define def_k_rpbw_b2 5 //--- #define def_k_GTEFeedForward 71 #define def_k_gteff_qkv 0 #define def_k_gteff_score 1 #define def_k_gteff_out 2 #define def_k_gteff_dimension 3 //--- #define def_k_GTEInsideGradients 72 #define def_k_gteig_qkv 0 #define def_k_gteig_qkv_g 1 #define def_k_gteig_scores 2 #define def_k_gteig_gradient 3 //--- #define def_k_FeedForwardNODEInpK 73 #define def_k_ffdopriInp_matrix_i 0 #define def_k_ffdopriInp_matrix_k1 1 #define def_k_ffdopriInp_matrix_k2 2 #define def_k_ffdopriInp_matrix_k3 3 #define def_k_ffdopriInp_matrix_k4 4 #define def_k_ffdopriInp_matrix_k5 5 #define def_k_ffdopriInp_matrix_k6 6 #define def_k_ffdopriInp_matrix_beta 7 #define def_k_ffdopriInp_matrix_o 8 //--- #define def_k_FeedForwardNODEF 74 #define def_k_ffdoprif_matrix_w 0 #define def_k_ffdoprif_matrix_i 1 #define def_k_ffdoprif_matrix_o 2 #define def_k_ffdoprif_dimension 3 #define def_k_ffdoprif_step 4 #define def_k_ffdoprif_activation 5 //--- #define def_k_HiddenGradientNODEInpK 75 //--- #define def_k_HiddenGradientNODEF 76 #define def_k_hddoprif_matrix_w 0 #define def_k_hddoprif_matrix_g 1 #define def_k_hddoprif_matrix_i 2 #define def_k_hddoprif_matrix_ig 3 #define def_k_hddoprif_dimension_out 4 #define def_k_hddoprif_activation 5 //--- #define def_k_NODEF_UpdateWeightsAdam 77 #define def_k_uwdoprif_matrix_w 0 #define def_k_uwdoprif_matrix_gk1 1 #define def_k_uwdoprif_matrix_gk2 2 #define def_k_uwdoprif_matrix_gk3 3 #define def_k_uwdoprif_matrix_gk4 4 #define def_k_uwdoprif_matrix_gk5 5 #define def_k_uwdoprif_matrix_gk6 6 #define def_k_uwdoprif_matrix_ik1 7 #define def_k_uwdoprif_matrix_ik2 8 #define def_k_uwdoprif_matrix_ik3 9 #define def_k_uwdoprif_matrix_ik4 10 #define def_k_uwdoprif_matrix_ik5 11 #define def_k_uwdoprif_matrix_ik6 12 #define def_k_uwdoprif_matrix_m 13 #define def_k_uwdoprif_matrix_v 14 #define def_k_uwdoprif_alpha 15 #define def_k_uwdoprif_lenth 16 #define def_k_uwdoprif_l 17 #define def_k_uwdoprif_b1 18 #define def_k_uwdoprif_b2 19 //--- #define def_k_TimeDerivative 78 #define def_k_tdqkv 0 #define def_k_tddqkv 1 #define def_k_tddimension 2 //--- #define def_k_HGTimeDerivative 79 //--- #define def_k_FeedForwardContAtt 80 #define def_k_caqkv 0 #define def_k_cadqkv 1 #define def_k_cascore 2 #define def_k_caout 3 #define def_k_cadimension 4 #define def_k_caheads 5 //--- #define def_k_HiddenGradientContAtt 81 #define def_k_hgcaqkv 0 #define def_k_hgcaqkv_g 1 #define def_k_hgcadqkv 2 #define def_k_hgcadqkv_g 3 #define def_k_hgcascore 4 #define def_k_hgcaout_g 5 #define def_k_hgcadimension 6 //--- #define def_k_RevInFeedForward 82 #define def_k_revffinputs 0 #define def_k_revffoptions 1 #define def_k_revffoutput 2 #define def_k_revffoptions_size 3 #define def_k_revffoptimization 4 //--- #define def_k_RevInHiddenGraddient 83 #define def_k_revhginputs 0 #define def_k_revhginputs_gr 1 #define def_k_revhgoptions 2 #define def_k_revhgoutput_gr 3 #define def_k_revhgoptions_size 4 #define def_k_revhgoptimization 5 #define def_k_revhgactivation 6 //--- #define def_k_DeActivation 84 #define def_k_deact_inputs 0 #define def_k_deact_inputs_gr 1 #define def_k_deact_output_gr 2 #define def_k_deact_activation 3 //--- #define def_k_PatchCreate 85 #define def_k_ptc_inputs 0 #define def_k_ptc_weights 1 #define def_k_ptc_outputs 2 #define def_k_ptc_inputs_total 3 #define def_k_ptc_window_in 4 #define def_k_ptc_step 5 #define def_k_ptc_activation 6 //--- #define def_k_PatchHiddenGradient 86 #define def_k_pthg_inputs 0 #define def_k_pthg_inputs_gr 1 #define def_k_pthg_weights 2 #define def_k_pthg_outputs_gr 3 #define def_k_pthg_window_in 4 #define def_k_pthg_step 5 #define def_k_pthg_window_out 6 #define def_k_pthg_outputs_total 7 #define def_k_pthg_activation 8 //--- #define def_k_PatchUpdateWeightsAdam 87 #define def_k_ptuwa_weights 0 #define def_k_ptuwa_outputs_gr 1 #define def_k_ptuwa_inputs 2 #define def_k_ptuwa_weights_m 3 #define def_k_ptuwa_weights_v 4 #define def_k_ptuwa_inputs_total 5 #define def_k_ptuwa_l 6 #define def_k_ptuwa_b1 7 #define def_k_ptuwa_b2 8 #define def_k_ptuwa_step 9 //--- #define def_k_MatMult 88 #define def_k_mm_matr1 0 #define def_k_mm_matr2 1 #define def_k_mm_result 2 #define def_k_mm_dimension 3 #define def_k_mm_multvarsecond 4 //--- #define def_k_FFT 89 #define def_k_fft_inputs_re 0 #define def_k_fft_inputs_im 1 #define def_k_fft_outputs_re 2 #define def_k_fft_outputs_im 3 #define def_k_fft_input_window 4 #define def_k_fft_input_complex 5 #define def_k_fft_output_window 6 #define def_k_fft_reverse 7 //--- #define def_k_ComplexLayer 90 #define def_k_cl_inputs_re 0 #define def_k_cl_inputs_im 1 #define def_k_cl_outputs_re 2 #define def_k_cl_outputs_im 3 //--- #define def_k_ComplexLayerGradient 91 //--- #define def_k_GradientMSA 92 #define def_k_gmsa_target 0 #define def_k_gmsa_forecast 1 #define def_k_gmsa_gradient 2 //--- #define def_k_CumulativeGradient 93 #define def_k_cg_gradient1 0 #define def_k_cg_gradient2 1 #define def_k_cg_gradient_out 2 #define def_k_cg_alpha 3 //--- #define def_k_FeedForwardComplexConv 94 #define def_k_CalcHiddenGradientComplexConv 95 #define def_k_UpdateWeightsComplexConvMomentum 96 #define def_k_UpdateWeightsComplexConvAdam 97 #define def_k_ComplexMHAttentionGradients 98 #define def_k_ComplexMHAttentionScore 99 #define def_k_ComplexMHAttentionOut 100 #define def_k_ComplexSoftMax_FeedForward 101 #define def_k_ComplexSoftMax_HiddenGradient 102 #define def_k_ComplexSoftMax_OutputGradient 103 //--- #define def_k_ComplexNormalize 104 #define def_k_cn_inputs 0 #define def_k_cn_outputs 1 #define def_k_cn_means 2 #define def_k_cn_vars 3 #define def_k_cn_dimension 4 //--- #define def_k_ComplexUnNormalize 105 //--- #define def_k_ComplexNormalizeGradient 106 #define def_k_cng_inputs_gr 0 #define def_k_cng_outputs_gr 1 #define def_k_cng_vars 2 #define def_k_cng_dimension 3 //--- #define def_k_ComplexUnNormalizeGradient 107 //--- #define def_k_MainFreqWeight 108 #define def_k_mfw_freq 0 #define def_k_mfw_weight 1 #define def_k_mfw_dimension 2 //--- #define def_k_WeightedSum 109 #define def_k_ws_inputs1 0 #define def_k_ws_inputs2 1 #define def_k_ws_outputs 2 #define def_k_ws_weight 3 #define def_k_ws_dimension 4 //--- #define def_k_WeightedSumGradient 110 //--- #define def_k_FeedForwardS3 111 #define def_k_s3_inputs 0 #define def_k_s3_probability 1 #define def_k_s3_weights 2 #define def_k_s3_outputs 3 #define def_k_s3_positions 4 #define def_k_s3_window 5 #define def_k_s3_total 6 //--- #define def_k_InsideGradientS3 112 #define def_k_s3g_inputs 0 #define def_k_s3g_inputs_gr 1 #define def_k_s3g_probability 2 #define def_k_s3g_probability_gr 3 #define def_k_s3g_weights 4 #define def_k_s3g_outputs_gr 5 #define def_k_s3g_positions 6 #define def_k_s3g_window 7 #define def_k_s3g_total 8 //--- #define def_k_WeightGradientS3 113 #define def_k_s3w_inputs 0 #define def_k_s3w_positions 1 #define def_k_s3w_outputs_gr 2 #define def_k_s3w_weights_gr 3 #define def_k_s3w_window 4 #define def_k_s3w_total 5 //--- #define def_k_MH2PyrAttentionOut 114 #define def_k_pam_q 0 #define def_k_pam_kv 1 #define def_k_pam_score 2 #define def_k_pam_out 3 #define def_k_pam_dimension 4 #define def_k_pam_heads_kv 5 #define def_k_pam_window 6 //--- #define def_k_PLR 115 #define def_k_plr_inputs 0 #define def_k_plr_outputs 1 #define def_k_plt_isttp 2 #define def_k_plr_transpose 3 #define def_k_plr_step 4 //--- #define def_k_PLRGrad 116 #define def_k_plrg_inputs_gr 0 #define def_k_plrg_outputs 1 #define def_k_plrg_outputs_gr 2 #define def_k_plrg_transpose 3 //--- #define def_k_UpdateWeightsAdamMini 117 #define def_k_wuam_matrix_w 0 #define def_k_wuam_matrix_g 1 #define def_k_wuam_matrix_i 2 #define def_k_wuam_matrix_m 3 #define def_k_wuam_matrix_v 4 #define def_k_wuam_l 5 #define def_k_wuam_b1 6 #define def_k_wuam_b2 7 //--- #define def_k_UpdateWeightsConvAdamMini 118 #define def_k_wucam_matrix_w 0 #define def_k_wucam_matrix_g 1 #define def_k_wucam_matrix_i 2 #define def_k_wucam_matrix_m 3 #define def_k_wucam_matrix_v 4 #define def_k_wucam_inputs 5 #define def_k_wucam_l 6 #define def_k_wucam_b1 7 #define def_k_wucam_b2 8 #define def_k_wucam_step 9 //--- #define def_k_CutTrendAndOther 119 #define def_k_ct_inputs 0 #define def_k_ct_plr 1 #define def_k_ct_trend 2 #define def_k_ct_other 3 //--- #define def_k_CutTrendAndOtherGradient 120 #define def_k_ctg_inputs_gr 0 #define def_k_ctg_plr 1 #define def_k_ctg_plr_gr 2 #define def_k_ctg_trend_gr 3 #define def_k_ctg_other_gr 4 //--- #define def_k_CutOneFromAnother 121 #define def_k_ctofa_inputs 0 #define def_k_ctofa_cut 1 #define def_k_ctofa_other 2 //--- #define def_k_CutOneFromAnotherGradient 122 //--- #define def_k_UniTrajPrepare 123 #define def_k_utp_history 0 #define def_k_utp_h_mask 1 #define def_k_utp_future 2 #define def_k_utp_f_mask 3 #define def_k_utp_output 4 #define def_k_utp_h_total 5 #define def_k_utp_f_total 6 //--- #define def_k_UniTrajPrepareGrad 124 #define def_k_utpg_history_gr 0 #define def_k_utpg_future_gr 1 #define def_k_utpg_output 2 #define def_k_utpg_output_gr 3 #define def_k_utpg_h_total 4 #define def_k_utpg_f_total 5 //--- #define def_k_UniTrajBTS 125 #define def_k_utbts_concat_inp 0 #define def_k_utbts_d_forw 1 #define def_k_utbts_d_bakw 2 #define def_k_utbts_total 3 //--- #define def_k_GateElementMul 126 #define def_k_gem_inputs1 0 #define def_k_gem_inputs2 1 #define def_k_gem_gate 2 #define def_k_gem_out 3 //--- #define def_k_GateElementMulGrad 127 #define def_k_gemg_inputs1 0 #define def_k_gemg_inputs1_gr 1 #define def_k_gemg_inputs2 2 #define def_k_gemg_inputs2_gr 3 #define def_k_gemg_gate 4 #define def_k_gemg_gate_gr 5 #define def_k_gemg_out 6 #define def_k_gemg_activ1 7 #define def_k_gemg_activ2 8 #define def_k_gemg_activ_gr 9 //--- #define def_k_HiVTPrepare 128 #define def_k_hivtp_data 0 #define def_k_hivtp_output 1 //--- #define def_k_TransposeRCD 129 //--- #define def_k_MatMultGrad 130 #define def_k_mmg_matr1 0 #define def_k_mmg_matr1_gr 1 #define def_k_mmg_matr2 2 #define def_k_mmg_matr2_gr 3 #define def_k_mmg_result_gr 4 #define def_k_mmg_dimension 5 #define def_k_mmg_multvarsecond 6 //--- #define def_k_OrthoganalLoss 131 #define def_k_ol_data 0 #define def_k_ol_grad 1 #define def_k_ol_add 2 //--- #define def_k_CalcDistance 132 #define def_k_cd_data 0 #define def_k_cd_distance 1 #define def_k_cd_dimension 2 //--- #define def_k_FeedForwardLocalMax 133 #define def_k_fflm_matrix_i 0 #define def_k_fflm_distance 1 #define def_k_fflm_matrix_o 2 #define def_k_fflm_radius 3 //--- #define def_k_CalcInputGradientLocalMax 134 #define def_k_cglm_matrix_i 0 #define def_k_cglm_matrix_ig 1 #define def_k_cglm_distance 2 #define def_k_cglm_matrix_o 3 #define def_k_cglm_matrix_g 4 #define def_k_cglm_radius 5 //--- #define def_k_MHMaskAttentionOut 135 #define def_k_mask_at_q 0 #define def_k_mask_at_kv 1 #define def_k_mask_at_score 2 #define def_k_mask_at_mask 3 #define def_k_mask_at_out 4 #define def_k_mask_at_dimension 5 #define def_k_mask_at_heads_kv 6 #define def_k_mask_at_mask_level 7 //--- #define def_k_MHMaskAttentionInsideGradients 136 #define def_k_mask_atg_q 0 #define def_k_mask_atg_q_g 1 #define def_k_mask_atg_kv 2 #define def_k_mask_atg_kv_g 3 #define def_k_mask_atg_mask 4 #define def_k_mask_atg_mask_g 5 #define def_k_mask_atg_scores 6 #define def_k_mask_atg_gradient 7 #define def_k_mask_atg_kunits 8 #define def_k_mask_atg_heads_kv 9 #define def_k_mask_atg_mask_level 10 //--- #define def_k_CalcPositionBias 137 #define def_k_cpb_data1 0 #define def_k_cpb_data2 1 #define def_k_cpb_result 2 #define def_k_cpb_dimension 3 //--- #define def_k_MHPosBiasAttentionOut 138 #define def_k_pbao_q 0 #define def_k_pbao_k 1 #define def_k_pbao_v 2 #define def_k_pbao_score 3 #define def_k_pbao_pos_bias 4 #define def_k_pbao_out 5 #define def_k_pbao_dimension 6 #define def_k_pbao_heads_kv 7 #define def_k_pbao_use_pos_bias 8 //--- #define def_k_MHPosBiasAttentionInsideGradients 139 #define def_k_pbaog_q 0 #define def_k_pbaog_q_g 1 #define def_k_pbaog_k 2 #define def_k_pbaog_k_g 3 #define def_k_pbaog_v 4 #define def_k_pbaog_v_g 5 #define def_k_pbaog_scores 6 #define def_k_pbaog_gradient 7 #define def_k_pbaog_kunits 8 #define def_k_pbaog_heads_kv 9 //--- #define def_k_DiversityLoss 140 #define def_k_dl_data 0 #define def_k_dl_grad 1 #define def_k_dl_dimension 2 #define def_k_dl_activation 3 #define def_k_dl_add 4 //--- #define def_k_MHRelativeAttentionOut 141 #define def_k_rat_q 0 #define def_k_rat_k 1 #define def_k_rat_v 2 #define def_k_rat_bk 3 #define def_k_rat_bv 4 #define def_k_rat_gc 5 #define def_k_rat_gp 6 #define def_k_rat_score 7 #define def_k_rat_out 8 #define def_k_rat_dimension 9 //--- #define def_k_MHRelativeAttentionInsideGradients 142 #define def_k_ratg_q 0 #define def_k_ratg_q_g 1 #define def_k_ratg_k 2 #define def_k_ratg_k_g 3 #define def_k_ratg_v 4 #define def_k_ratg_v_g 5 #define def_k_ratg_bk 6 #define def_k_ratg_bk_g 7 #define def_k_ratg_bv 8 #define def_k_ratg_bv_g 9 #define def_k_ratg_gc 10 #define def_k_ratg_gc_g 11 #define def_k_ratg_gp 12 #define def_k_ratg_gp_g 13 #define def_k_ratg_scores 14 #define def_k_ratg_gradient 15 #define def_k_ratg_kunits 16 //--- #define def_k_CalcAlignmentGradient 143 #define def_k_aliggr_matrix_o1 0 #define def_k_aliggr_matrix_o2 1 #define def_k_aliggr_matrix_g1 2 #define def_k_aliggr_matrix_g2 3 #define def_k_aliggr_activation 4 #define def_k_aliggr_add 5 //--- #define def_k_FeatureSmoothing 144 #define def_k_fs_feature 0 #define def_k_fs_outputs 1 #define def_k_fs_smoothing 2 //--- #define def_k_FeatureSmoothingGradient 145 //--- #define def_k_BatchFeedForwardAddNoise 146 #define def_k_normwithnoise_inputs 0 #define def_k_normwithnoise_options 1 #define def_k_normwithnoise_noise 2 #define def_k_normwithnoise_output 3 #define def_k_normwithnoise_batch 4 #define def_k_normwithnoise_optimization 5 #define def_k_normwithnoise_activation 6 #define def_k_normwithnoise_alpha 7 //--- #define def_k_HyperProjection 147 #define def_k_lp_inputs 0 #define def_k_lp_outputs 1 //--- #define def_k_HyperProjectionGrad 148 #define def_k_lpg_inputs 0 #define def_k_lpg_inputs_gr 1 #define def_k_lpg_outputs_gr 2 //--- #define def_k_LogMap 149 #define def_k_logmap_features 0 #define def_k_logmap_centroids 1 #define def_k_logmap_curvatures 2 #define def_k_logmap_outputs 3 #define def_k_logmap_product 4 #define def_k_logmap_distance 5 #define def_k_logmap_norma 6 //--- #define def_k_LogMapGrad 150 #define def_k_logmapgr_features 0 #define def_k_logmapgr_features_gr 1 #define def_k_logmapgr_centroids 2 #define def_k_logmapgr_centroids_gr 3 #define def_k_logmapgr_curvatures 4 #define def_k_logmapgr_curvatures_gr 5 #define def_k_logmapgr_outputs 6 #define def_k_logmapgr_outputs_gr 7 #define def_k_logmapgr_product 8 #define def_k_logmapgr_distance 9 #define def_k_logmapgr_norma 10 //--- #define def_k_CalcEpsilonWeights 151 #define def_k_epsw_matrix_w 0 #define def_k_epsw_matrix_g 1 #define def_k_epsw_matrix_i 2 #define def_k_epsw_matrix_epsw 3 #define def_k_epsw_rho 4 //--- #define def_k_CalcEpsilonWeightsConv 152 #define def_k_epswconv_matrix_w 0 #define def_k_epswconv_matrix_g 1 #define def_k_epswconv_matrix_i 2 #define def_k_epswconv_matrix_epsw 3 #define def_k_epswconv_inputs 4 #define def_k_epswconv_rho 5 #define def_k_epswconv_step 6 //--- #define def_k_PLRMultiAgents 153 //--- #define def_k_PLRMultiAgentsGrad 154 #define def_k_plrg_agents 4 //--- #define def_k_FeedForwardMHConv 155 //--- #define def_k_CalcHiddenGradientMHConv 156 #define def_k_chgc_heads 10 //--- #define def_k_UpdateWeightsMHConvAdam 157 #define def_k_uwca_heads 12 //--- #define def_k_MoreLessEqual 158 #define def_k_mle_inputs 0 #define def_k_mle_outputs 1 //--- #define def_k_MultiScaleRelativeAttentionOut 159 //--- #define def_k_SSM2D_FeedForward 160 #define def_k_ssm2d_ah 0 #define def_k_ssm2d_b_time 1 #define def_k_ssm2d_b_var 2 #define def_k_ssm2d_px_time 3 #define def_k_ssm2d_px_var 4 #define def_k_ssm2d_c_time 5 #define def_k_ssm2d_c_var 6 #define def_k_ssm2d_delta_time 7 #define def_k_ssm2d_delta_var 8 #define def_k_ssm2d_hidden 9 #define def_k_ssm2d_y 10 //--- #define def_k_SSM2D_CalcHiddenGradient 161 #define def_k_ssm2dhg_ah 0 #define def_k_ssm2dhg_grad_ah 1 #define def_k_ssm2dhg_b_time 2 #define def_k_ssm2dhg_grad_b_time 3 #define def_k_ssm2dhg_b_var 4 #define def_k_ssm2dhg_grad_b_var 5 #define def_k_ssm2dhg_px_time 6 #define def_k_ssm2dhg_grad_px_time 7 #define def_k_ssm2dhg_px_var 8 #define def_k_ssm2dhg_grad_px_var 9 #define def_k_ssm2dhg_c_time 10 #define def_k_ssm2dhg_grad_c_time 11 #define def_k_ssm2dhg_c_var 12 #define def_k_ssm2dhg_grad_c_var 13 #define def_k_ssm2dhg_delta_time 14 #define def_k_ssm2dhg_grad_delta_time 15 #define def_k_ssm2dhg_delta_var 16 #define def_k_ssm2dhg_grad_delta_var 17 #define def_k_ssm2dhg_hidden 18 #define def_k_ssm2dhg_grad_y 19 //--- #define def_k_PScan 162 #define def_k_psc_A 0 #define def_k_psc_X 1 #define def_k_psc_H 2 #define def_k_psc_X_out 3 //--- #define def_k_PScan_CalcHiddenGradient 163 #define def_k_pscgr_A 0 #define def_k_pscgr_X 1 #define def_k_pscgr_H 2 #define def_k_pscgr_grad_X_out 3 #define def_k_pscgr_grad_A 4 #define def_k_pscgr_grad_X 5 #define def_k_pscgr_grad_H 6 //--- #define def_k_DiagMatMult 164 #define def_k_diagmm_diag 0 #define def_k_diagmm_matr 1 #define def_k_diagmm_result 2 #define def_k_diagmm_activation 3 //--- #define def_k_DiagMatMultGrad 165 #define def_k_diagmmgr_diag 0 #define def_k_diagmmgr_grad_diag 1 #define def_k_diagmmgr_matr 2 #define def_k_diagmmgr_grad_matr 3 #define def_k_diagmmgr_grad_result 4 //--- #define def_k_TopKgates 166 #define def_k_topK_inputs 0 #define def_k_topK_noises 1 #define def_k_topK_gates 2 #define def_k_topK_k 3 //--- #define def_k_TopKgatesGrad 167 #define def_k_topKgr_inputs 0 #define def_k_topKgr_grad_inputs 1 #define def_k_topKgr_noises 2 #define def_k_topKgr_gates 3 #define def_k_topKgr_grad_gates 4 //--- #define def_k_MaskByDistance 168 #define def_k_maskDist_buf_real 0 #define def_k_maskDist_buf_imag 1 #define def_k_maskDist_mask 2 #define def_k_maskDist_dimension 3 //--- #define def_k_MaskAttention 169 #define def_k_maskatt_q 0 #define def_k_maskatt_kv 1 #define def_k_maskatt_scores 2 #define def_k_maskatt_masks 3 #define def_k_maskatt_out 4 #define def_k_maskatt_dimension 5 #define def_k_maskatt_heads_kv 6 //--- #define def_k_MaskAttentionGradients 170 #define def_k_maskattgr_q 0 #define def_k_maskattgr_q_g 1 #define def_k_maskattgr_kv 2 #define def_k_maskattgr_kv_g 3 #define def_k_maskattgr_scores 4 #define def_k_maskattgr_gradient 5 #define def_k_maskattgr_kunits 6 #define def_k_maskattgr_heads_kv 7 //--- #define def_k_FeedForwardMultWinConv 171 #define def_k_ffmwc_matrix_w 0 #define def_k_ffmwc_matrix_i 1 #define def_k_ffmwc_matrix_o 2 #define def_k_ffmwc_windows_in 3 #define def_k_ffmwc_inputs 4 #define def_k_ffmwc_windows_total 5 #define def_k_ffmwc_window_out 6 #define def_k_ffmwc_activation 7 //--- #define def_k_CalcHiddenGradientMultWinConv 172 #define def_k_chgmwc_matrix_w 0 #define def_k_chgmwc_matrix_i 1 #define def_k_chgmwc_matrix_ig 2 #define def_k_chgmwc_matrix_og 3 #define def_k_chgmwc_windows_in 4 #define def_k_chgmwc_outputs 5 #define def_k_chgmwc_windows_total 6 #define def_k_chgmwc_window_out 7 #define def_k_chgmwc_activation 8 //--- #define def_k_UpdateWeightsMultWinConvAdam 173 #define def_k_uwmwc_matrix_w 0 #define def_k_uwmwc_matrix_og 1 #define def_k_uwmwc_matrix_i 2 #define def_k_uwmwc_matrix_m 3 #define def_k_uwmwc_matrix_v 4 #define def_k_uwmwc_windows_in 5 #define def_k_uwmwc_windows_total 6 #define def_k_uwmwc_window_out 7 #define def_k_uwmwc_inputs 8 #define def_k_uwmwc_outputs 9 #define def_k_uwmwc_l 10 #define def_k_uwmwc_b1 11 #define def_k_uwmwc_b2 12 //--- #define def_k_MaskAttentionComplex 174 #define def_k_maskattcom_q 0 #define def_k_maskattcom_kv 1 #define def_k_maskattcom_scores 2 #define def_k_maskattcom_masks 3 #define def_k_maskattcom_out 4 #define def_k_maskattcom_dimension 5 #define def_k_maskattcom_heads_kv 6 //--- #define def_k_MaskAttentionGradientsComplex 175 #define def_k_maskattcomgr_q 0 #define def_k_maskattcomgr_q_g 1 #define def_k_maskattcomgr_kv 2 #define def_k_maskattcomgr_kv_g 3 #define def_k_maskattcomgr_scores 4 #define def_k_maskattcomgr_mask 5 #define def_k_maskattcomgr_mask_g 6 #define def_k_maskattcomgr_gradient 7 #define def_k_maskattcomgr_kunits 8 #define def_k_maskattcomgr_heads_kv 9 //--- #define def_k_CSLSTM_FeedForward 176 #define def_k_cslstmff_concatenated 0 #define def_k_cslstmff_memory 1 #define def_k_cslstmff_output 2 //--- #define def_k_CSLSTM_CalcHiddenGradient 177 #define def_k_cslstmhg_concatenated 0 #define def_k_cslstmhg_concatenated_grad 1 #define def_k_cslstmhg_memory 2 #define def_k_cslstmhg_output_grad 3 //--- #define def_k_ProbAttentionQeuryImp 178 #define def_k_probat_querys 0 #define def_k_probat_keys_values 1 #define def_k_probat_index_keys 2 #define def_k_probat_querys_imp 3 #define def_k_probat_dimension 4 //--- #define def_k_TopKImportanceToIndex 179 #define def_k_imptoind_importance 0 #define def_k_imptoind_indexes 1 #define def_k_imptoind_tok_k 2 //--- #define def_k_QIndexAttention 180 #define def_k_indatt_q 0 #define def_k_indatt_kv 1 #define def_k_indatt_scores 2 #define def_k_indatt_indexes 3 #define def_k_indatt_out 4 #define def_k_indatt_dimension 5 #define def_k_indatt_heads_kv 6 //--- #define def_k_QIndexAttentionGradients 181 #define def_k_indattgr_q 0 #define def_k_indattgr_q_g 1 #define def_k_indattgr_kv 2 #define def_k_indattgr_kv_g 3 #define def_k_indattgr_indexes 4 #define def_k_indattgr_scores 5 #define def_k_indattgr_gradient 6 #define def_k_indattgr_kunits 7 #define def_k_indattgr_heads_kv 8 //--- #define def_k_TSPositonEncoder 182 #define def_k_tspe_data 0 #define def_k_tspe_time 1 #define def_k_tspe_output 2 #define def_k_tspe_period 3 //--- #define def_k_FeedForwardMultWinConvWPad 183 #define def_k_ffmwconvwpad_matrix_w 0 #define def_k_ffmwconvwpad_matrix_i 1 #define def_k_ffmwconvwpad_matrix_o 2 #define def_k_ffmwconvwpad_windows_in 3 #define def_k_ffmwconvwpad_inputs 4 #define def_k_ffmwconvwpad_step 5 #define def_k_ffmwconvwpad_window_out 6 #define def_k_ffmwconvwpad_activation 7 //--- #define def_k_CalcHiddenGradientMultWinConvWPad 184 #define def_k_hgmwconvwpad_matrix_w 0 #define def_k_hgmwconvwpad_matrix_i 1 #define def_k_hgmwconvwpad_matrix_ig 2 #define def_k_hgmwconvwpad_matrix_og 3 #define def_k_hgmwconvwpad_windows_in 4 #define def_k_hgmwconvwpad_outputs 5 #define def_k_hgmwconvwpad_step 6 #define def_k_hgmwconvwpad_window_out 7 #define def_k_hgmwconvwpad_filters 8 #define def_k_hgmwconvwpad_activation 9 //--- #define def_k_UpdateWeightsMultWinConvAdamWPad 185 #define def_k_uwmwconvwpad_matrix_w 0 #define def_k_uwmwconvwpad_matrix_og 1 #define def_k_uwmwconvwpad_matrix_i 2 #define def_k_uwmwconvwpad_matrix_m 3 #define def_k_uwmwconvwpad_matrix_v 4 #define def_k_uwmwconvwpad_windows_in 5 #define def_k_uwmwconvwpad_windows_total 6 #define def_k_uwmwconvwpad_window_out 7 #define def_k_uwmwconvwpad_inputs 8 #define def_k_uwmwconvwpad_step 9 #define def_k_uwmwconvwpad_outputs 10 #define def_k_uwmwconvwpad_l 11 #define def_k_uwmwconvwpad_b1 12 #define def_k_uwmwconvwpad_b2 13 //--- #define def_k_ConcatDiff 186 #define def_k_concdiff_data 0 #define def_k_concdiff_output 1 #define def_k_concdiff_step 2 //--- #define def_k_FeedForwardMaskMultWinConv 187 #define def_k_ffmmwc_matrix_w 0 #define def_k_ffmmwc_matrix_i 1 #define def_k_ffmmwc_masks 2 #define def_k_ffmmwc_matrix_o 3 #define def_k_ffmmwc_inputs 4 #define def_k_ffmmwc_window_in 5 #define def_k_ffmmwc_windows_total 6 #define def_k_ffmmwc_activation 7 //--- #define def_k_CalcHiddenGradientMaskMultWinConv 188 #define def_k_chgmmwc_matrix_w 0 #define def_k_chgmmwc_matrix_i 1 #define def_k_chgmmwc_matrix_ig 2 #define def_k_chgmmwc_matrix_og 3 #define def_k_chgmmwc_masks 4 #define def_k_chgmmwc_masks_g 5 #define def_k_chgmmwc_outputs 6 #define def_k_chgmmwc_window_in 7 #define def_k_chgmmwc_window_out 8 #define def_k_chgmmwc_activation 9 //--- #define def_k_UpdateWeightsMaskMultWinConvAdam 189 #define def_k_uwmmwc_matrix_w 0 #define def_k_uwmmwc_matrix_og 1 #define def_k_uwmmwc_matrix_i 2 #define def_k_uwmmwc_masks 3 #define def_k_uwmmwc_matrix_m 4 #define def_k_uwmmwc_matrix_v 5 #define def_k_uwmmwc_windows_total 6 #define def_k_uwmmwc_inputs 7 #define def_k_uwmmwc_outputs 8 #define def_k_uwmmwc_l 9 #define def_k_uwmmwc_b1 10 #define def_k_uwmmwc_b2 11 //--- #define def_k_MainFreq 190 #define def_k_mf_freq_r 0 #define def_k_mf_freq_im 1 #define def_k_mf_main_freq 2 #define def_k_mf_dimension 3 //--- #define def_k_FeedForwardAdaptConv 191 #define def_k_ffac_matrix_w 0 #define def_k_ffac_matrix_i 1 #define def_k_ffac_matrix_o 2 #define def_k_ffac_main_freq 3 #define def_k_ffac_inputs 4 #define def_k_ffac_window_in 5 #define def_k_ffac_activation 6 //--- #define def_k_CalcHiddenGradientAdaptConv 192 #define def_k_chgac_matrix_w 0 #define def_k_chgac_matrix_i 1 #define def_k_chgac_matrix_ig 2 #define def_k_chgac_matrix_og 3 #define def_k_chgac_main_freq 4 #define def_k_chgac_outputs 5 #define def_k_chgac_window_in 6 #define def_k_chgac_window_out 7 #define def_k_chgac_activation 8 //--- #define def_k_UpdateWeightsAdaptConvAdam 193 #define def_k_uwac_matrix_w 0 #define def_k_uwac_matrix_og 1 #define def_k_uwac_matrix_i 2 #define def_k_uwac_matrix_m 3 #define def_k_uwac_matrix_v 4 #define def_k_uwac_main_freq 5 #define def_k_uwac_inputs 6 #define def_k_uwac_outputs 7 #define def_k_uwac_l 8 #define def_k_uwac_b1 9 #define def_k_uwac_b2 10 //--- #define def_k_RoPE 194 #define def_k_rope_inputs 0 #define def_k_rope_position_emb 1 #define def_k_rope_outputs 2 //--- #define def_k_CalcHiddenGradRoPE 195 //--- #define def_k_MatrixDif 196 #define def_k_dif_matrix1 0 #define def_k_dif_matrix2 1 #define def_k_dif_matrix_out 2 #define def_k_dif_multiplyer 3 #define def_k_dif_shift_in1 4 #define def_k_dif_shift_in2 5 #define def_k_dif_shift_out 6 //--- #define def_k_MatrixDifGrad 197 //--- #define def_k_IdentMatrixDif 198 #define def_k_iddif_matrix_in 0 #define def_k_iddif_matrix_out 1 #define def_k_iddif_multiplyer 2 #define def_k_iddif_shift_in 3 #define def_k_iddif_shift_out 4 //--- #define def_k_IdentMatrixDifGrad 199 //--- #define def_k_SumVecMatrix 200 #define def_k_svecmat_vector_in 0 #define def_k_svecmat_matrix_in 1 #define def_k_svecmat_matrix_out 2 #define def_k_svecmat_multiplyer 3 #define def_k_svecmat_shift_in1 4 #define def_k_svecmat_shift_in2 5 #define def_k_svecmat_shift_out 6 //--- #define def_k_SumVecMatrixGrad 201 //--- #define def_k_InterpolationAttention 202 #define def_k_ia_matrix_in 0 #define def_k_ia_W 1 #define def_k_ia_A 2 #define def_k_ia_GL 3 #define def_k_ia_Adj 4 #define def_k_ia_H 5 #define def_k_ia_Atten 6 #define def_k_ia_matrix_out 7 #define def_k_ia_dimension 8 //--- #define def_k_InterpolationAttentionGrad 203 #define def_k_iag_matrix_in 0 #define def_k_iag_matrix_in_gr 1 #define def_k_iag_W 2 #define def_k_iag_W_gr 3 #define def_k_iag_A 4 #define def_k_iag_A_gr 5 #define def_k_iag_GL 6 #define def_k_iag_GL_gr 7 #define def_k_iag_Adj 8 #define def_k_iag_H 9 #define def_k_iag_H_gr 10 #define def_k_iag_Atten 11 #define def_k_iag_matrix_out_gr 12 #define def_k_iag_dimension 13 //--- #define def_k_IdentMatrixSum 204 #define def_k_idsum_matrix_in 0 #define def_k_idsum_matrix_out 1 #define def_k_idsum_multiplyer 2 #define def_k_idsum_shift_in 3 #define def_k_idsum_shift_out 4 //--- #define def_k_Activation 205 #define def_k_act_inputs 0 #define def_k_act_outputs 1 #define def_k_act_activation 2 //--- #define def_k_PeriodNorm 206 #define def_k_pn_inputs 0 #define def_k_pn_mean_stdevs 1 #define def_k_pn_outputs 2 #define def_k_pn_total_inputs 3 //--- #define def_k_PeriodNormGrad 207 #define def_k_png_inputs 0 #define def_k_png_inputs_gr 1 #define def_k_png_mean_stdevs 2 #define def_k_png_mean_stdevs_gr 3 #define def_k_png_outputs 4 #define def_k_png_outputs_gr 5 #define def_k_png_total_inputs 6 //--- #define def_k_AdaptSpatialNorm 208 #define def_k_asn_inputs 0 #define def_k_asn_attention 1 #define def_k_asn_mean_stdevs 2 #define def_k_asn_outputs 3 //--- #define def_k_AdaptSpatialNormGrad 209 #define def_k_asng_inputs 0 #define def_k_asng_inputs_gr 1 #define def_k_asng_attention 2 #define def_k_asng_attention_gr 3 #define def_k_asng_mean_stdevs 4 #define def_k_asng_mean_stdevs_gr 5 #define def_k_asng_outputs_gr 6 #define def_k_asng_total_inputs 7 //--- #define def_k_AttentNorm 210 #define def_k_atn_inputs 0 #define def_k_atn_attention 1 #define def_k_atn_means 2 #define def_k_atn_stdevs 3 #define def_k_atn_outputs 4 #define def_k_atn_total_inputs 5 #define def_k_atn_segment_size 6 //--- #define def_k_AttentNormGrad 211 #define def_k_atng_inputs 0 #define def_k_atng_inputs_gr 1 #define def_k_atng_attention 2 #define def_k_atng_attention_gr 3 #define def_k_atng_means 4 #define def_k_atng_stdevs 5 #define def_k_atng_means_gr 6 #define def_k_atng_outputs_gr 7 #define def_k_atng_total_inputs 8 #define def_k_atng_segment_size 9 //--- #define def_k_ChebStep 212 #define def_k_cheb_support 0 #define def_k_cheb_outputs 1 #define def_k_cheb_step 2 //--- #define def_k_ChebStepGrad 213 #define def_k_chebgr_support 0 #define def_k_chebgr_support_g 1 #define def_k_chebgr_outputs 2 #define def_k_chebgr_outputs_g 3 #define def_k_chebgr_step 4 //--- #define def_k_SignificantNeighborsSampling 214 #define def_k_sns_data 0 #define def_k_sns_candidates 1 #define def_k_sns_random_cands 2 #define def_k_sns_neighbors 3 #define def_k_sns_dimension 4 //--- #define def_k_SparseMHScores 215 #define def_k_smhs_data 0 #define def_k_smhs_indexes 1 #define def_k_smhs_scores 2 #define def_k_smhs_sparse 3 //--- #define def_k_SparseMHScoresGrad 216 #define def_k_smhs_gr_data_gr 0 #define def_k_smhs_gr_indexes 1 #define def_k_smhs_gr_scores 2 #define def_k_smhs_gr_scores_gr 3 //--- #define def_k_SparseMatMult 217 #define def_k_smm_sparse_index 0 #define def_k_smm_sparse_data 1 #define def_k_smm_full 2 #define def_k_smm_result 3 #define def_k_smm_full_rows 4 //--- #define def_k_SparseMatMultGrad 218 #define def_k_smm_gr_sparse_index 0 #define def_k_smm_gr_sparse_data 1 #define def_k_smm_gr_sparse_gr 2 #define def_k_smm_gr_full 3 #define def_k_smm_gr_full_gr 4 #define def_k_smm_gr_result_gr 5 #define def_k_smm_gr_sparse_rows 6 #define def_k_smm_gr_sparse_cols 7 #define def_k_smm_gr_full_rows 8 #define def_k_smm_gr_full_cols 9 //--- #define def_k_RandomWalk 219 #define def_k_rw_data 0 #define def_k_rw_inv_diag 1 #define def_k_rw_norm 2 #define def_k_rw_total_cols 3 //--- #define def_k_ConcatByLabel 220 #define def_k_cbl_data 0 #define def_k_cbl_label 1 #define def_k_cbl_embedding1 2 #define def_k_cbl_embedding2 3 #define def_k_cbl_output 4 #define def_k_cbl_dimension_data 5 #define def_k_cbl_dimension_emb1 6 #define def_k_cbl_dimension_emb2 7 #define def_k_cbl_frame1 8 #define def_k_cbl_frame2 9 #define def_k_cbl_period1 10 #define def_k_cbl_period2 11 //--- #define def_k_ConcatByLabelGrad 221 #define def_k_cbl_gr_units 12 //--- #define def_k_GlobalLocalAttention 222 #define def_k_glatt_q 0 #define def_k_glatt_kv 1 #define def_k_glatt_scores 2 #define def_k_glatt_mask 3 #define def_k_glatt_label 4 #define def_k_glatt_out 5 #define def_k_glatt_dimension 6 #define def_k_glatt_total_kv 7 #define def_k_glatt_total_mask 8 //--- #define def_k_GlobalLocalAttentionGrad 223 #define def_k_glatt_gr_q 0 #define def_k_glatt_gr_q_gr 1 #define def_k_glatt_gr_kv 2 #define def_k_glatt_gr_kv_gr 3 #define def_k_glatt_gr_scores 4 #define def_k_glatt_gr_mask 5 #define def_k_glatt_gr_mask_gr 6 #define def_k_glatt_gr_label 7 #define def_k_glatt_gr_out_gr 8 #define def_k_glatt_gr_dimension 9 #define def_k_glatt_gr_total_q 10 #define def_k_glatt_gr_total_kv 11 #define def_k_glatt_gr_total_mask 12 //--- #define def_k_SparseSoftMax 224 #define def_k_ssoftmax_data 0 #define def_k_ssoftmax_outputs 1 #define def_k_ssoftmax_indexes 2 #define def_k_ssoftmax_out_dimension 3 //--- #define def_k_SparseSoftMaxGrad 225 #define def_k_ssoftmax_gr_data_gr 0 #define def_k_ssoftmax_gr_outputs 1 #define def_k_ssoftmax_gr_outputs_gr 2 #define def_k_ssoftmax_gr_indexes 3 #define def_k_ssoftmax_gr_out_dimension 4 //--- #define def_k_FloatToSpike 226 #define def_k_fts_values 0 #define def_k_fts_levels 1 #define def_k_fts_outputs 2 //--- #define def_k_FloatToSpikeGrad 227 #define def_k_ftsg_values 0 #define def_k_ftsg_values_gr 1 #define def_k_ftsg_levels_gr 2 #define def_k_ftsg_gradients 3 //--- #define def_k_SpikeMHAttention 228 #define def_k_smhat_qkv 0 #define def_k_smhat_diag_bias 1 #define def_k_smhat_scores 2 #define def_k_smhat_out 3 #define def_k_smhat_dimension 4 #define def_k_smhat_mask_future 5 //--- #define def_k_SpikeMHAttentionGrad 229 #define def_k_smhat_gr_qkv 0 #define def_k_smhat_gr_qkv_gr 1 #define def_k_smhat_gr_diag_bias 2 #define def_k_smhat_gr_diag_bias_gr 3 #define def_k_smhat_gr_scores 4 #define def_k_smhat_gr_gradients 5 #define def_k_smhat_gr_dimension 6 #define def_k_smhat_gr_mask_future 7 //--- #define def_k_STFS 230 #define def_k_stfs_inputs 0 #define def_k_stfs_mask_time 1 #define def_k_stfs_mask_spatial 2 #define def_k_stfs_outputs 3 //--- #define def_k_STFSGrad 231 //--- #define def_k_AddToStack 232 #define def_k_ats_inputs 0 #define def_k_ats_stack 1 #define def_k_ats_stack_size 2 //--- #define def_k_AggregationByTime 233 #define def_k_agt_inputs 0 #define def_k_agt_stack 1 #define def_k_agt_outputs 2 #define def_k_agt_stack_size 3 #define def_k_agt_levels 4 //--- #define def_k_AggregationByTimeGrad 234 #define def_k_agtg_inputs_gr 0 #define def_k_agtg_outputs_gr 1 #define def_k_agtg_levels 2 //--- #define def_k_GRU 235 #define def_k_gru_XH 0 #define def_k_gru_prev_state 1 #define def_k_gru_outputs 2 //--- #define def_k_GRU_Grad 236 #define def_k_gru_gr_XH 0 #define def_k_gru_gr_XH_gr 1 #define def_k_gru_gr_prev_state 2 #define def_k_gru_gr_outputs_gr 3 //--- #define def_k_ScalarToVector 237 #define def_k_stv_scalar 0 #define def_k_stv_vector_in 1 #define def_k_stv_vector_out 2 //--- #define def_k_ScalarToVectorGrad 238 #define def_k_stvg_scalar 0 #define def_k_stvg_scalar_gr 1 #define def_k_stvg_vector_in 2 #define def_k_stvg_vector_in_gr 3 #define def_k_stvg_vector_out_gr 4 #define def_k_stvg_dimension 5 //--- #define def_k_CalcFlow 239 #define def_k_cf_value 0 #define def_k_cf_prev_value 1 #define def_k_cf_flow 2 //--- #define def_k_DilatedCorrelation 240 #define def_k_dcor_feature 0 #define def_k_dcor_shifts 1 #define def_k_dcor_correlations 2 #define def_k_dcor_dimension 3 //--- #define def_k_DilatedCorrelationGrad 241 #define def_k_dcorgr_feature 0 #define def_k_dcorgr_feature_gr 1 #define def_k_dcorgr_shifts 2 #define def_k_dcorgr_corr_gr 3 #define def_k_dcorgr_total_corr 4 //--- #define def_k_DilatedDifference 242 #define def_k_ddiff_feature 0 #define def_k_ddiff_shifts 1 #define def_k_ddiff_differences 2 //--- #define def_k_DilatedDifferenceGrad 243 #define def_k_ddiffgr_feature 0 #define def_k_ddiffgr_feature_gr 1 #define def_k_ddiffgr_shifts 2 #define def_k_ddiffgr_differences_gr 3 #define def_k_ddiffgr_total_shifts 4 //--- #define def_k_PerturbedMatrix 244 #define def_k_pm_inputs 0 #define def_k_pm_perturb 1 #define def_k_pm_output 2 #define def_k_pm_perturb_mult 3 //--- #define def_k_PerturbedMatrixGrad 245 //--- #define def_k_LinearUpsample 246 #define def_k_lu_data 0 #define def_k_lu_upsample 1 //--- #define def_k_LinearUpsampleGrad 247 #define def_k_lug_data_gr 0 #define def_k_lug_upsample_gr 1 #define def_k_lug_dimension_htr 2 //--- #define def_k_MixExpertsPredict 248 #define def_k_moepred_experts 0 #define def_k_moepred_outputs 1 //--- #define def_k_MixExpertsPredictGrad 249 #define def_k_moepredgr_experts 0 #define def_k_moepredgr_experts_gr 1 #define def_k_moepredgr_outputs_gr 2 //--- #define def_k_MHFAT 250 #define def_k_mhfat_q 0 #define def_k_mhfat_kv 1 #define def_k_mhfat_scale 2 #define def_k_mhfat_scores 3 #define def_k_mhfat_out 4 #define def_k_mhfat_dimension 5 #define def_k_mhfat_mask_future 6 //--- #define def_k_MHFATGrad 251 #define def_k_mhfat_gr_q 0 #define def_k_mhfat_gr_q_gr 1 #define def_k_mhfat_gr_kv 2 #define def_k_mhfat_gr_kv_gr 3 #define def_k_mhfat_gr_scale 4 #define def_k_mhfat_gr_scale_gr 5 #define def_k_mhfat_gr_scores 6 #define def_k_mhfat_gr_gradients 7 #define def_k_mhfat_gr_dimension 8 #define def_k_mhfat_gr_total_k 9 #define def_k_mhfat_gr_mask_future 10 //--- #define def_k_SparseConcatenate 252 #define def_k_sparconc_sparse_index 0 #define def_k_sparconc_sparse_data 1 #define def_k_sparconc_full 2 #define def_k_sparconc_result 3 #define def_k_sparconc_full_rows 4 //--- #define def_k_SparseConcatenateGrad 253 #define def_k_sparconc_gr_sparse_index 0 #define def_k_sparconc_gr_sparse_data 1 #define def_k_sparconc_gr_sparse_gr 2 #define def_k_sparconc_gr_full 3 #define def_k_sparconc_gr_full_gr 4 #define def_k_sparconc_gr_result_gr 5 #define def_k_sparconc_gr_sparse_rows 6 #define def_k_sparconc_gr_sparse_cols 7 #define def_k_sparconc_gr_full_rows 8 #define def_k_sparconc_gr_full_cols 9 //--- #define def_k_MHFlashAttention 254 #define def_k_mhflat_q 0 #define def_k_mhflat_kv 1 #define def_k_mhflat_logsumexp 2 #define def_k_mhflat_out 3 #define def_k_mhflat_dimension 4 #define def_k_mhflat_total_kv 5 #define def_k_mhflat_mask_future 6 //--- #define def_k_MHFlashAttentionGrad 255 #define def_k_mhflatg_q 0 #define def_k_mhflatg_q_gr 1 #define def_k_mhflatg_kv 2 #define def_k_mhflatg_kv_gr 3 #define def_k_mhflatg_logsumexp 4 #define def_k_mhflatg_out 5 #define def_k_mhflatg_out_gr 6 #define def_k_mhflatg_dimension 7 #define def_k_mhflatg_total_q 8 #define def_k_mhflatg_total_kv 9 #define def_k_mhflatg_mask_future 10 //--- #define def_k_MHFlashSTCA 256 #define def_k_mhstca_query 0 #define def_k_mhstca_X 1 #define def_k_mhstca_logsumexp 2 #define def_k_mhstca_output 3 #define def_k_mhstca_dimension 4 #define def_k_mhstca_total_X 5 #define def_k_mhstca_mask_future 6 //--- #define def_k_MHFlashSTCAGrad 257 #define def_k_mhstca_gr_query 0 #define def_k_mhstca_gr_query_gr 1 #define def_k_mhstca_gr_X 2 #define def_k_mhstca_gr_X_gr 3 #define def_k_mhstca_gr_logsumexp 4 #define def_k_mhstca_gr_output 5 #define def_k_mhstca_gr_output_gr 6 #define def_k_mhstca_gr_dimension 7 #define def_k_mhstca_gr_total_q 8 #define def_k_mhstca_gr_total_X 9 #define def_k_mhstca_gr_mask_future 10 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "NeuroNet.cl" as string cl_program ///\defgroup enums ENUM ///@{ //+------------------------------------------------------------------+ /// Enum of activation formula used //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { None = -1, ///< Without activation formula TANH = 0, ///< Use \f$tanh(x)\f$ for activation neuron SIGMOID = 1, ///< Use \f$\frac{1}{1+e^x}\f$ for activation neuron LReLU = 2, ///< For activation neuron use LReLU \f[\left\{ \begin{array} a x>=0, \ x \\x<0, \ 0.01*x \end{array} \right.\f] SoftPlus = 3, GELU = 4, MinusSoftPlus = 5, ELU = 6, }; //+------------------------------------------------------------------+ /// Enum of optimization method used //+------------------------------------------------------------------+ enum ENUM_OPTIMIZATION { SGD = 0, ///< Stochastic gradient descent ADAM = 1, ///< Adam LS = 2, ///< Least Squares ADAM_MINI = 3 ///< Adam-mini }; ///@} //+------------------------------------------------------------------+ ///\class CConnection ///\brief Class of connection to anothe neuron ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CConnection : public CObject { public: float weight; ///< Current weight float deltaWeight; ///< last delta of weight used in SGD optimization float mt; ///< First moment in Adam optimization float vt; ///< Second moment in Adam optimization /** Constructor @param[in] w initial weight */ CConnection(float w) { weight = w; deltaWeight = 0; mt = 0; vt = 0; } /** Destructor */ ~CConnection() {}; //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual int Type(void) const { return defConnect; } ///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CConnection::Save(int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; //--- if(FileWriteDouble(file_handle, weight) <= 0) ReturnFalse; if(FileWriteDouble(file_handle, deltaWeight) <= 0) ReturnFalse; if(FileWriteDouble(file_handle, mt) <= 0) ReturnFalse; if(FileWriteDouble(file_handle, vt) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CConnection::Load(int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; //--- weight = (float)FileReadDouble(file_handle); deltaWeight = (float)FileReadDouble(file_handle); mt = (float)FileReadDouble(file_handle); vt = (float)FileReadDouble(file_handle); //--- return true; } //+------------------------------------------------------------------+ ///\class CArrayCon ///\brief Array of connections to anothe neuron ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CArrayCon : public CArrayObj { public: /** Constructor */ CArrayCon(void) {}; /** Destructor */ ~CArrayCon(void) {}; //--- virtual bool CreateElement(int const index); ///< Method for cearing new element by index @param[in] index Index of new element @return logical result of operation virtual void IncreaseTotal() { m_data_total++; } ///< Increase number of elements in array virtual int Type(void) const { return defArrayConnects; } ///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CArrayCon::CreateElement(int index) { if(index < 0 || index >= m_data_max) ReturnFalse; //--- xor128; float weigh = (float)(rnd_w / UINT_MAX - 0.5); m_data[index] = new CConnection(weigh / 100); if(!CheckPointer(m_data[index]) != POINTER_INVALID) ReturnFalse; //--- return (true); } //+------------------------------------------------------------------+ ///\class CLayer /// Class of neurons collection in one layer of Neural Net. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CLayer; class CNeuronBaseOCL; //+------------------------------------------------------------------+ ///\class CNeuronBase /// The base class of neuron. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronBase : public CObject { protected: float outputVal; ///< Output value float prevVal; ///< Previous output value uint m_myIndex; ///< Index of neuron in layer float gradient; ///< Current gradient of neuron CArrayCon *Connections; ///< Array of connections with neurons in next layer ENUM_ACTIVATION activation; ///< Activation type (#ENUM_ACTIVATION) ENUM_OPTIMIZATION optimization; ///< Optimization method (#ENUM_OPTIMIZATION) int t; ///< Count of iterations //--- virtual bool feedForward(CLayer *prevLayer) { ReturnFalse; } ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool calcHiddenGradients(CLayer *&nextLayer) { ReturnFalse; } ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer. virtual bool updateInputWeights(CLayer *&prevLayer) { ReturnFalse; } ///< Method for updating weights.@param prevLayer Pointer to previos layer. virtual float activationFunction(float x); ///< Method to calculate activation function.@param x Input data. @return Result of activation function. virtual float SigmoidFunction(float x) { return (float)MathPow(1 + exp(-x), -1); } ///< Calculating Sigmoid \f$\frac{1}{1+e^x}\f$.@param x Input data.@return Result of calculation virtual float TanhFunction(float x) { return (float)tanh(x); } ///< Calculating \f$tanh(x)\f$.@param x Input data.@return Result of calculation virtual CLayer *getOutputLayer(void) { return NULL; } ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer. public: /** Constructor */ CNeuronBase(void); /** Destructor */~CNeuronBase(void); virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type); ///< Method of initialization class.@param numOutputs Number of connections to next layer.@param myIndex Index of neuron in layer.@param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } ///< Set the type of activation function (#ENUM_ACTIVATION) //--- //static float lr; static float alpha; ///< Multiplier to momentum in SGD optimization //--- virtual void setOutputVal(float val) { prevVal = outputVal; outputVal = val; } ///< Set the output value virtual float getOutputVal() { return outputVal; } ///< Return result of feed forward operations.@return Output value virtual float getPrevVal() { return prevVal; } ///< Return result of feed forward operations at previous iteration.@return Previous output value virtual void setGradient(float val) { gradient = val; } ///< Set gradient value to neuron. virtual float getGradient() { return gradient; } ///< Return gradient of neuron.@return Gradient virtual CArrayCon *getConnections() { return Connections;} ///< Method to get access to array of connections.@return Pointer to connections array virtual float activationFunctionDerivative(float x); ///< Calculate derivative of activation function.@param[in] x Input data@return Derivative virtual float SigmoidFunctionDerivative(float x) { return x * (1 - x); } ///< Calculate derivative of Sigmoid function.@param x Input data@return Derivative virtual float TanhFunctionDerivative(float x) { return (1 + x) * (1 - x); } ///< Calculate derivative of \f$tanh(x)\f$.@param x Input data@return Derivative //--- virtual bool feedForward(CObject *&SourceObject); ///< Dispatch method for defining the subroutine for Feed Forward process.@param SourceObject Pointer to previos layer. virtual bool calcHiddenGradients(CObject *&TargetObject); ///< Dispatch method for defining the subroutine for transfer gradient to previous layer.@param TargetObject Pointer to next layer. virtual bool updateInputWeights(CObject *&SourceObject); ///< Dispatch method for defining the subroutine for updating weights.@param SourceObject Pointer to previos layer. //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle)///< Load method @param[in] file_handle handle of file @return logical result of operation { activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); return(Connections.Load(file_handle)); } //--- virtual int Type(void) const { return defNeuronBase; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ //float CNeuronBase::lr=0.0000001; // net learning rate float CNeuronBase::alpha = (float)0.8; // momentum //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::CNeuronBase(void) : outputVal(1), gradient(0), activation(TANH), t(1), optimization(SGD) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBase::~CNeuronBase(void) { DeleteObj(Connections); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(Connections) == POINTER_INVALID) { Connections = new CArrayCon(); if(CheckPointer(Connections) == POINTER_INVALID) ReturnFalse; } //--- if(Connections.Reserve(fmax(numOutputs, 1))) for(uint c = 0; c < numOutputs; c++) { if(!Connections.CreateElement(c)) ReturnFalse; Connections.IncreaseTotal(); } //--- m_myIndex = myIndex; optimization = optimization_type; return true; } //+------------------------------------------------------------------+ ///\class CNeuron /// Class of neuron for full connected layers. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuron : public CNeuronBase { private: virtual bool feedForward(CLayer *prevLayer); ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool calcHiddenGradients(CLayer *&nextLayer); ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer. virtual bool updateInputWeights(CLayer *&prevLayer); ///< Method for updating weights.@param prevLayer Pointer to previos layer. public: /** Constructor */ CNeuron(void) {}; /** Destructor */~CNeuron(void) { Connections.Shutdown(); } //--- virtual bool calcOutputGradients(float targetVals); ///< Method of output gradients calculation.@param targetVals Traget value virtual float sumDOW(CLayer *&nextLayer) ; ///< A method for collecting gradients from the next layer.@param[in] nextLayer Pointer to next layer@return Total gradient to neuron. virtual int Type(void) const { return defNeuron; } ///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); int total = prevLayer.Total(); for(int n = 0; n < total && !IsStopped(); n++) { CNeuron *neuron = prevLayer.At(n); CConnection *con = neuron.Connections.At(m_myIndex); if(CheckPointer(con) == POINTER_INVALID) continue; if(optimization == SGD) con.weight += con.deltaWeight = (gradient != 0 ? lr * neuron.getOutputVal() * gradient : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0); else { con.mt = b1 * con.mt + (1 - b1) * gradient; con.vt = (float)(b2 * con.vt + (1 - b2) * pow(gradient, 2) + 0.00000001); con.weight += con.deltaWeight = (float)(lt * con.mt / sqrt(con.vt)); } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuron::sumDOW(CLayer *&nextLayer) { float sum = 0.0; int total = nextLayer.Total() - 1; for(int n = 0; n < total; n++) { CConnection *con = Connections.At(n); if(CheckPointer(con) == POINTER_INVALID) continue; float weight = con.weight; if(weight != 0) { CNeuron *neuron = nextLayer.At(n); sum += weight * neuron.gradient; } } return sum; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron::calcHiddenGradients(CLayer *&nextLayer) { float targetVal = sumDOW(nextLayer) + outputVal; return calcOutputGradients(targetVal); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron::calcOutputGradients(float targetVal) { float delta = (targetVal > 1 ? 1 : targetVal < -1 ? -1 : targetVal) - outputVal; gradient = (delta != 0 ? delta * activationFunctionDerivative(outputVal) : 0); return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Type() != defLayer) ReturnFalse; //--- prevVal = outputVal; float sum = 0.0; int total = prevLayer.Total(); for(int n = 0; n < total && !IsStopped(); n++) { CNeuron *temp = prevLayer.At(n); float val = temp.getOutputVal(); if(val != 0) { CConnection *con = temp.Connections.At(m_myIndex); if(CheckPointer(con) == POINTER_INVALID) continue; sum += val * con.weight; } } outputVal = activationFunction(MathMin(MathMax(sum, -18), 18)); //--- return true; } //+------------------------------------------------------------------+ ///\class COpenCLMy /// Class for working with OpenCL //+------------------------------------------------------------------+ class COpenCLMy : public COpenCL { protected: uaLongChar uLocalDimension; public: /** Constructor */ COpenCLMy(void) {}; /** Destructor */~COpenCLMy(void) { for(int i = 0; i < m_kernels_total; i++) KernelFree(i); } bool Initialize(const string program, const bool show_log = true); template int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags); ///< Method for creating OpenCL buffer from array.@param data[] Array of data.@param data_array_offset Offset of data in array.@param data_array_count Number of data items in array.@param flags Buffer's properties (CL_MEM_READ_WRITE, CL_MEM_WRITE_ONLY, CL_MEM_READ_ONLY, CL_MEM_ALLOC_HOST_PTR) int AddBuffer(uint size_in_bytes, const uint flags); //--- long GetMaxLocalSize(const int dimension) const { return uLocalDimension.data[dimension]; } ulong GetBufferSize(const int buffer_index) const; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool COpenCLMy::Initialize(const string program, const bool show_log = true) { if(!COpenCL::Initialize(program, show_log)) ReturnFalse; uint size = 0; if(!CLGetDeviceInfo(GetContext(), CL_DEVICE_MAX_WORK_ITEM_SIZES, uLocalDimension.cdata, size)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ ulong COpenCLMy::GetBufferSize(const int buffer_index) const { //--- check parameters if(buffer_index < 0 || buffer_index >= m_buffers_total) return(0); if(m_buffers[buffer_index] == INVALID_HANDLE) return(0); if(m_context == INVALID_HANDLE || m_program == INVALID_HANDLE) return(0); //--- return CLGetInfoInteger(buffer_index, CL_BUFFER_SIZE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CLayer: public CArrayObj { private: uint iOutputs; ///< Number of output connections from 1 neuron to neurons in next layer. int iFileHandle; ///< File handle for download result of previous study. COpenCLMy *OpenCL; ///< Class for working with OpenCL public: /** Constructor */ CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL); ///< @param[in] outputs Number of output connections from 1 neuron to neurons in next layer @param[in] handle File handle for download result of previous study @param[in] OpenCL Pointer to class for working with OpenCL /** Destructor */~CLayer(void) {}; //--- virtual bool CreateElement(int const index); ///< Method for creating new element in layer virtual void IncreaseTotal() { m_data_total++; } ///< Method for increase number of items in layer virtual int Type(void) const { return defLayer; } ///< Identificator of class.@return Type of class virtual bool Load(const int file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual uint Outputs(void) { return iOutputs; } virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); virtual bool WeightsUpdate(CLayer *source, float tau); virtual bool ClearStates(void); CObject *At(const int index) const { if(index >= m_data_total || (-index) > m_data_total) return NULL; if(index < 0) return(m_data[m_data_total + index]); return(m_data[index]); } virtual CNeuronBaseOCL* operator[](const int index) const { return (CNeuronBaseOCL*)At(index); } virtual CObject* AsObject(void) { return GetPointer(this); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::CreateElement(int index) { if(index >= m_data_max) ReturnFalse; //--- bool result = false; CNeuronBase *temp = NULL; CNeuronProof *temp_p = NULL; CNeuronBaseOCL *temp_ocl = NULL; CNeuronProofOCL *temp_proof_ocl = NULL; CNeuronConvOCL *temp_con_ocl = NULL; CNeuronAttentionOCL *temp_at_ocl = NULL; CNeuronMH2AttentionOCL *temp_mh2at_ocl = NULL; CNeuronMLMHAttentionOCL *temp_mlat_ocl = NULL; CNeuronDropoutOCL *temp_drop_ocl = NULL; CNeuronBatchNormOCL *temp_batch_ocl = NULL; CVAE *vae = NULL; CNeuronLSTMOCL *lstm = NULL; CNeuronSoftMaxOCL *softmax = NULL; CNeuronFQF *fqf = NULL; CNeuronMultiModel *multi_model = NULL; CNeuronConcatenate *concat = NULL; CNeuronEmbeddingOCL *emb = NULL; CNeuronPositionEncoder *pe = NULL; CNeuronTransposeOCL *tr = NULL; CNeuronCGConvOCL *cgc = NULL; CNeuronDOTOCL *dot = NULL; CNeuronFAQOCL *faq = NULL; CNeuronCrossAttention *cross = NULL; CNeuronGTE *gte = NULL; CNeuronCCMROCL *ccmr = NULL; CNeuronNODEOCL *node = NULL; CNeuronConformer *conf = NULL; CNeuronRevINDenormOCL *revin = NULL; CNeuronClientOCL *client = NULL; CNeuronLearnabledPE *lpe = NULL; CNeuronUShapeAttention *usa = NULL; CNeuronPatching *patch = NULL; CNeuronTiDEOCL *tide = NULL; CNeuronFEDW *fedw = NULL; CNeuronFITSOCL *fits = NULL; CNeuronFreDFOCL *fredf = NULL; CNeuronATFNetOCL *atf = NULL; CNeuronComplexMLMHAttention *compl_att = NULL; CNeuronS3 *s3 = NULL; CNeuronMLMHAttentionMLKV *mlkv = NULL; CNeuronMLCrossAttentionMLKV *crmlkv = NULL; CNeuronCSCMOCL *cscm = NULL; CNeuronSPyrAttentionOCL *spyr = NULL; CNeuronPLROCL *plr = NULL; CNeuronTPMEncoder *tpme = NULL; CNeuronTPM *tpm = NULL; CNeuronSTNNEncoder *stnne = NULL; CNeuronSTNNDecoder *stnnd = NULL; CNeuronSparseTSF *sptsf = NULL; CNeuronTEMPOOCL *tempo = NULL; CNeuronInjectTST *inject = NULL; CNeuronMambaOCL *mamba = NULL; CNeuronTrajLLMOCL *traj = NULL; CNeuronUniTraj *utraj = NULL; CNeuronHiVTOCL *hivt = NULL; CNeuronPointNetOCL *pn = NULL; CNeuronSceneSpecific *ss = NULL; CNeuronSEFormer *sef = NULL; CNeuronMVMHAttentionMLKV *mv = NULL; CNeuronMVCrossAttentionMLKV *mvc = NULL; CNeuronSPFormer *spf = NULL; CNeuronMAFT *maft = NULL; CNeuronGRES *gres = NULL; CResidualConv *res = NULL; CNeuronRefMask *refm = NULL; CNeuronOCM *ocm = NULL; CNeuronRelativeSelfAttention *rat = NULL; CNeuronRMAT *rmat = NULL; CNeuronMHAttentionPooling *atpool = NULL; CNeuronMotifs *motif = NULL; CNeuronMultiScaleAttention *msat = NULL; CNeuronMolformer *molf = NULL; CNeuronAMCT *amct = NULL; CNeuronRelativeCrossAttention *rcat = NULL; CNeuronNAFS *nafs = NULL; CNeuronHyperProjection *lorenz = NULL; CNeuronHyperboloids *logmap = NULL; CNeuronHypDiff *hypdif = NULL; CNeuronBaseSAMOCL *sam_base = NULL; CNeuronConvSAMOCL *sam_conv = NULL; CNeuronPSBlock *ps_block = NULL; CNeuronPSformer *psformer = NULL; CNeuronMASA *masa = NULL; CNeuronMarketObserver *mo = NULL; CNeuronRLAgent *rl_ag = NULL; CNeuronControlAgent *contr_ag = NULL; CNeuronMASAAT *masaat = NULL; CNeuronMHConvOCL *mhconv = NULL; CNeuronMHFeedForward *mhff = NULL; CNeuronDMHAttention *dmhat = NULL; CNeuronCrossDMHAttention *crdmhat = NULL; CNeuronMultitaskStockformer *mtstockf = NULL; CNeuronFinMem *fin_mem = NULL; CNeuronFinAgent *fin_agent = NULL; CNeuronFinConManager *fin_con = NULL; CNeuronMacroHFT *mhft = NULL; CNeuronMacroHFTHyperAgent *mhft_hyp = NULL; CNeuronMacroHFTvsRiskManager *mhft_risk = NULL; CNeuronHidformer *hidf = NULL; CNeuronResNeXtBlock *resnext = NULL; CNeuronChimera *chimera = NULL; CNeuronMoT *mot = NULL; CNeuronHypridDecoder *hypridDecod = NULL; CNeuronAttraos *attraos = NULL; CNeuronDUET *duet = NULL; CNeuronTransposeRCDOCL *trrcd = NULL; CNeuronTransposeVRCOCL *tvrc = NULL; CNeuronMultiWindowsConvOCL *mwconv = NULL; CNeuronAdaBN *adabn = NULL; CNeuronCATCH *catch = NULL; CNeuronSkillsEncoder *skills = NULL; CNeuronHiSSDLowLevelControler *hissd = NULL; CNeuronLinerAttention *lineatt = NULL; CNeuronCGLSTMOCL *cglstm = NULL; CNeuronMHProbAttention *probatt = NULL; CNeuronTSPositionEncoder *tspe = NULL; CMamba4CastEmbeding* m4cemb = NULL; CNeuronMultiWindowsConvWPadOCL *mwconvpad = NULL; CNeuronConcatDiff *condiff = NULL; CNeuronMantisPatching *mantpatch = NULL; CNeuronMantisAttentionUnit *mantatt = NULL; CNeuronTimeFoundPatching *tf_patch = NULL; CNeuronTimeFoundTransformerUnit *tf_tu = NULL; CNeuronSwiGLUOCL *swiglu = NULL; CNeuronTimeMoESparseExperts *timemoesp = NULL; CNeuronTimeMoEAttention *timemoeatt = NULL; CNeuronAdaptConv *ac = NULL; CNeuronRoPE *rope = NULL; CNeuronK2VAEEncoder *k2vae = NULL; CNeuronInterpolationAttention *ia = NULL; CNeuronGinARCell *ginar_cell = NULL; CNeuronGinAR *ginar = NULL; CNeuronPeriodNorm *per_norm = NULL; CNeuronAdaptSpatialNorm *adapt_norm = NULL; CNeuronSCNNEncoder* scnn_enc = NULL; CNeuronSCNN* scnn = NULL; CNeuronAttentNorm* atn = NULL; CNeuronSAttentNorm* satn = NULL; CNeuronPolynomialRegression* pol_regr = NULL; CNeuronSSCNNEncoder* sscnn_enc = NULL; CNeuronSSCNN* sscnn = NULL; CCircleParams* circl_param = NULL; CNeuronTQMHA* tqmha = NULL; CNeuronHimNetGCRU* gcru = NULL; CNeuronHimNetEncoder* hn_enc = NULL; CNeuronHimNetDecoder* hn_dec = NULL; CNeuronSNSMHAttention* snsatt = NULL; CNeuronFastGConv* fgcru = NULL; CNeuronSAGDFN* sagdfn = NULL; CNeuronSpatialEmbedding* semb = NULL; CNeuronTempEmbedding* temb = NULL; CNeuronGlobalLocalAttention* glatt = NULL; CNeuronExtralonger* exlon = NULL; CNeuronGraphAttention* gratt = NULL; CNeuronSparseGraphAttention* sgratt = NULL; CNeuronGlobLocGraphAtt* glgratt = NULL; CNeuronExtralongerGraph* exlongr = NULL; CNeuronSpikeConv* spike = NULL; CNeuronGateLineAttention* glineatt = NULL; CNeuronSpikeSparseAttention* spikeatt = NULL; CNeuronMoEConv *moecon = NULL; CNeuronSpikingBrain* spikebrain = NULL; CNeuronSparseSoftMax* sparsoftmax = NULL; CNeuronSpikeActivation* spikeact = NULL; CNeuronSpikeResNeXtBlock* spikeresblock = NULL; CNeuronSpikeResNeXtBottleneck* spikeresbottleneck = NULL; CNeuronSpikeResNeXtResidual* spikeresresid = NULL; CNeuronSSAM* ssam = NULL; CNeuronSSAMResNeXtBlock* ssamblock = NULL; CNeuronSTFS* stfs = NULL; CNeuronSpikeConvGRU* sp_conv_gru = NULL; CNeuronAddToStack* stack = NULL; CNeuronStackCorrelation* st_correl = NULL; CNeuronSTEFlowNetEncoder* ste_encod = NULL; CNeuronSTEFlowNetResidualBlock* ste_resid = NULL; CNeuronSTEFlowNetDecoder* ste_decod = NULL; CNeuronSTEFlowNet* ste = NULL; CNeuronMultiScaleStackCorrelation* mul_correl = NULL; CNeuronSpikeConvGRU2D* sp_conv_gru2D = NULL; CNeuronRAFT* raft = NULL; CNeuronMultScalStackCorrelBySegments* segm_corr = NULL; CNeuronTMAMFE* mfe = NULL; CNeuronTMAMPA* mpa = NULL; CNeuronTMA* tma = NULL; CNeuronFERENet* fere = NULL; CNeuronSpikeSTConvGRU* st_convgru = NULL; CNeuronFGDModule* fgd = NULL; CNeuronEVMGRFlowNet* evmgr = NULL; CNeuronSpikeConvBlock* spike_block = NULL; CNeuronMSRes* msres = NULL; CNeuronSpikeQKAttention* qkatt = NULL; CNeuronSDSA* sdsa = NULL; CNeuronSDformerUBlock* sdublock = NULL; CNeuronSDformerFlow* sdflow = NULL; CNeuronSpikePatchStak* sp_patch_stak = NULL; CNeuronSpike2DSSMOCL* sp_ssm = NULL; CNeuronSpikeMamba2D* sp_mamba = NULL; CNeuronSpikeSTSSM* sp_stssm = NULL; CNeuronESTMEncoder* sp_estm = NULL; CNeuronCreateFlow* cr_flow = NULL; CNeuronBiDirectCorrelation* bi_corr = NULL; CNeuronSATMA* satma = NULL; CNeuronBAT* bat = NULL; CNeuronSpikeSCM* scm = NULL; CNeuronSpikeFAM* fam = NULL; CNeuronSpikeMDC* mdc = NULL; CNeuronMDS* mds = NULL; CNeuronSpikeADM* adm = NULL; CNeuronCDCSelfCorrector* selfcorrect = NULL; CNeuronSpikeCDC* cdc = NULL; CNeuronSpikeEEMFLow* eemflow = NULL; CNeuronSpikeDepthWiseConv* dw_conv = NULL; CNeuronMultiScaleExcitation* excit = NULL; CNeuronMultiScaleDifference* ms_diff = NULL; CNeuronCorrelationEncoder* corr_enc = NULL; CNeuronEDCFlow* edcflow = NULL; CNeuronCreateICEFlow* ice = NULL; CNeuronSpikeMFE* sp_mfe = NULL; CNeuronSpikeMHCrossAttention* sp_crosatt = NULL; CNeuronSpikeMixFusion* mix_fusion = NULL; CNeuronSTFlow* stflow = NULL; CNeuronPSSE* pse = NULL; CNeuronSpikeDepthWiseResidual* dwr = NULL; CNeuronSpikeSuperKernelBlock* sk_block = NULL; CNeuronSpikeGMA* gma = NULL; CNeuronETROF* etrof = NULL; CNeuronPSSEFlow* pseflow = NULL; CNeuronSpikeUSR* usr = NULL; CNeuronInitialFlow* init_flow = NULL; CNeuronSpikeSMR* smr = NULL; CNeuronEVAFlow* eva = NULL; CNeuronResFlowHTR* htr = NULL; CNeuronResFlow* resflow = NULL; CNeuronConfGateTailAwareMoE* tailaware = NULL; CNeuronResFlowLattice* lattice = NULL; CFieldAwareParams* fapar = NULL; CNeuronFieldAwareConv* faconv = NULL; CNeuronMHFAT* fat = NULL; CNeuronMHCrossFAT* cross_fat = NULL; CNeuronFieldPatternEmbedding* faemb = NULL; CNeuronAutoToken* autotoken = NULL; CNeuronMultiMixAttention* mixattention = NULL; CNeuronSparseMoEConv* sparsemoe = NULL; CNeuronMTmixAttBlock* mixattenblock = NULL; CCrossMHFlashAttention* flash_att = NULL; CNeuronMixFeedForward* mixff = NULL; CNeuronOneTrans* onetrans = NULL; CNeuronMHTHCrossAttention* thcatt = NULL; CNeuronSTCA* stca = NULL; //--- uint windows[] = {1, 1, 1}; uint mfe_windows[] = {3, 3}; uint sdflow_win[] = {1, 1, 1, 1, 1}; uint sdflow_layers[] = {1, 1, 1, 1}; vector vec = vector::Ones(2); if(iFileHandle <= 0) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID || !temp.Init(iOutputs, index, SGD)) ReturnFalse; result = true; } else { int type = FileReadInteger(iFileHandle); switch(type) { case defNeuron: temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) result = false; result = temp.Init(iOutputs, index, ADAM); break; case defNeuronProof: temp_p = new CNeuronProof(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } DeleteObj(temp_p); break; case defNeuronConv: temp_p = new CNeuronConv(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } DeleteObj(temp_p); break; case defNeuronLSTM: temp_p = new CNeuronLSTM(); if(CheckPointer(temp_p) == POINTER_INVALID) result = false; if(temp_p.Init(iOutputs, index, 1, 1, 1, ADAM)) { temp = temp_p; result = true; } DeleteObj(temp_p); break; case defNeuronBaseOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_ocl = new CNeuronBaseOCL(); if(CheckPointer(temp_ocl) == POINTER_INVALID) result = false; if(temp_ocl.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = temp_ocl; return true; } DeleteObj(temp_ocl); break; case defNeuronBaseSAMOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; sam_base = new CNeuronBaseSAMOCL(); if(CheckPointer(sam_base) == POINTER_INVALID) result = false; if(sam_base.Init(iOutputs, index, OpenCL, 1, 0, ADAM, 1)) { m_data[index] = sam_base; return true; } DeleteObj(sam_base); break; case defNeuronProofOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_proof_ocl = new CNeuronProofOCL(); if(CheckPointer(temp_proof_ocl) == POINTER_INVALID) result = false; if(temp_proof_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = temp_proof_ocl; return true; } case defNeuronConvOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_con_ocl = new CNeuronConvOCL(); if(CheckPointer(temp_con_ocl) == POINTER_INVALID) result = false; if(temp_con_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = temp_con_ocl; return true; } break; case defNeuronConvSAMOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; sam_conv = new CNeuronConvSAMOCL(); if(CheckPointer(sam_conv) == POINTER_INVALID) result = false; if(sam_conv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = sam_conv; return true; } break; case defNeuronAttentionOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_at_ocl = new CNeuronAttentionOCL(); if(CheckPointer(temp_at_ocl) == POINTER_INVALID) result = false; if(temp_at_ocl.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { m_data[index] = temp_at_ocl; return true; } break; case defNeuronMHAttentionOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_at_ocl = new CNeuronMHAttentionOCL(); if(CheckPointer(temp_at_ocl) == POINTER_INVALID) result = false; if(temp_at_ocl.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { m_data[index] = temp_at_ocl; return true; } break; case defNeuronMLMHAttentionOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_mlat_ocl = new CNeuronMLMHAttentionOCL(); if(CheckPointer(temp_mlat_ocl) == POINTER_INVALID) result = false; if(temp_mlat_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = temp_mlat_ocl; return true; } break; case defNeuronMLMHSparseAttentionOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_mlat_ocl = new CNeuronMLMHSparseAttention(); if(CheckPointer(temp_mlat_ocl) == POINTER_INVALID) result = false; if(temp_mlat_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = temp_mlat_ocl; return true; } break; case defNeuronDropoutOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_drop_ocl = new CNeuronDropoutOCL(); if(CheckPointer(temp_drop_ocl) == POINTER_INVALID) result = false; if(temp_drop_ocl.Init(iOutputs, index, OpenCL, 1.0f, 0.1f, ADAM, 1)) { m_data[index] = temp_drop_ocl; return true; } break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_batch_ocl = (type == defNeuronBatchNormWithNoise ? new CNeuronBatchNormWithNoise() : new CNeuronBatchNormOCL()); if(CheckPointer(temp_batch_ocl) == POINTER_INVALID) result = false; if(temp_batch_ocl.Init(iOutputs, index, OpenCL, 1, 1, ADAM)) { m_data[index] = temp_batch_ocl; return true; } break; case defNeuronVAEOCL: if(!OpenCL) ReturnFalse; vae = new CVAE(); if(!vae) result = false; if(vae.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = vae; return true; } break; case defNeuronLSTMOCL: if(!OpenCL) ReturnFalse; lstm = new CNeuronLSTMOCL(); if(!lstm) result = false; if(lstm.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = lstm; return true; } break; case defNeuronSoftMaxOCL: if(!OpenCL) ReturnFalse; softmax = new CNeuronSoftMaxOCL(); if(!softmax) result = false; if(softmax.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = softmax; return true; } break; case defNeuronFQF: if(!OpenCL) ReturnFalse; fqf = new CNeuronFQF(); if(!fqf) result = false; if(fqf.Init(iOutputs, index, OpenCL, 1, 32, 32, ADAM, 1)) { m_data[index] = fqf.AsObject(); return true; } break; case defNeuronMultiModels: if(!OpenCL) ReturnFalse; multi_model = new CNeuronMultiModel(); if(!multi_model) result = false; if(multi_model.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = multi_model; return true; } break; case defNeuronConcatenate: if(!OpenCL) ReturnFalse; concat = new CNeuronConcatenate(); if(!concat) result = false; if(concat.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = concat; return true; } break; case defNeuronSoftActorCritic: if(!OpenCL) ReturnFalse; fqf = new CNeuronSoftActorCritic(); if(!fqf) ReturnFalse; if(fqf.Init(iOutputs, index, OpenCL, 1, 32, 32, ADAM, 1)) { m_data[index] = fqf.AsObject(); return true; } break; case defNeuronEmbeddingOCL: if(!OpenCL) ReturnFalse; emb = new CNeuronEmbeddingOCL(); if(!emb) ReturnFalse; if(!emb.Init(iOutputs, index, OpenCL, 1, 1, windows)) { DeleteObj(emb); ReturnFalse; } m_data[index] = emb; return true; break; case defNeuronPEOCL: if(!OpenCL) ReturnFalse; pe = new CNeuronPositionEncoder(); if(!pe) ReturnFalse; if(!pe.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { DeleteObj(pe); ReturnFalse; } m_data[index] = pe; return true; break; case defNeuronTransposeOCL: if(!OpenCL) ReturnFalse; tr = new CNeuronTransposeOCL(); if(!tr) ReturnFalse; if(!tr.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { DeleteObj(tr); ReturnFalse; } m_data[index] = tr; return true; break; case defNeuronMH2AttentionOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_mh2at_ocl = new CNeuronMH2AttentionOCL(); if(CheckPointer(temp_mh2at_ocl) == POINTER_INVALID) result = false; if(temp_mh2at_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = temp_mh2at_ocl; return true; } DeleteObj(temp_mh2at_ocl); break; case defNeuronCGConvOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; cgc = new CNeuronCGConvOCL(); if(CheckPointer(cgc) == POINTER_INVALID) result = false; if(cgc.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { m_data[index] = cgc; return true; } DeleteObj(cgc); break; case defNeuronMFTOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_mlat_ocl = new CNeuronMFTOCL(); if(CheckPointer(temp_mlat_ocl) == POINTER_INVALID) result = false; if(temp_mlat_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = temp_mlat_ocl; return true; } DeleteObj(temp_mlat_ocl); break; case defNeuronXCiTOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; temp_mlat_ocl = new CNeuronXCiTOCL(); if(CheckPointer(temp_mlat_ocl) == POINTER_INVALID) result = false; if(temp_mlat_ocl.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = temp_mlat_ocl; return true; } DeleteObj(temp_mlat_ocl); break; case defNeuronDOTOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; dot = new CNeuronDOTOCL(); if(CheckPointer(dot) == POINTER_INVALID) result = false; if(dot.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = dot; return true; } DeleteObj(dot); break; case defNeuronFAQOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; faq = new CNeuronFAQOCL(); if(CheckPointer(faq) == POINTER_INVALID) result = false; if(faq.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = faq; return true; } DeleteObj(faq); break; case defNeuronCrossAttenOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; cross = new CNeuronCrossAttention(); if(CheckPointer(cross) == POINTER_INVALID) result = false; if(cross.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = cross; return true; } DeleteObj(cross); break; case defNeuronGTE: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; gte = new CNeuronGTE(); if(CheckPointer(gte) == POINTER_INVALID) result = false; if(gte.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = gte; return true; } DeleteObj(gte); break; case defNeuronCCMROCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; ccmr = new CNeuronCCMROCL(); if(CheckPointer(ccmr) == POINTER_INVALID) result = false; if(ccmr.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = ccmr; return true; } DeleteObj(ccmr); break; case defNeuronNODEOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; node = new CNeuronNODEOCL(); if(CheckPointer(node) == POINTER_INVALID) result = false; if(node.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = node; return true; } DeleteObj(node); break; case defNeuronConformerOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; conf = new CNeuronConformer(); if(CheckPointer(conf) == POINTER_INVALID) result = false; if(conf.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = conf; return true; } DeleteObj(conf); break; case defNeuronRevInDenormOCL: if(CheckPointer(OpenCL) == POINTER_INVALID) ReturnFalse; revin = new CNeuronRevINDenormOCL(); if(CheckPointer(revin) == POINTER_INVALID) result = false; if(revin.Init(iOutputs, index, OpenCL, 1, -1, NULL)) { m_data[index] = revin; return true; } DeleteObj(revin); break; case defNeuronClientOCL: client = new CNeuronClientOCL(); if(!client) ReturnFalse; if(client.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, windows, ADAM, 1)) { m_data[index] = client; return true; } DeleteObj(client); break; case defNeuronLearnabledPE: lpe = new CNeuronLearnabledPE(); if(!lpe) ReturnFalse; if(lpe.Init(iOutputs, index, OpenCL, 1, ADAM, 1)) { m_data[index] = lpe; return true; } DeleteObj(lpe); break; case defNeuronUShapeAttention: usa = new CNeuronUShapeAttention(); if(!usa) ReturnFalse; if(usa.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = usa; return true; } DeleteObj(usa); break; case defNeuronPatchingOCL: patch = new CNeuronPatching(); if(!patch) ReturnFalse; if(patch.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = patch; return true; } DeleteObj(patch); break; //--- case defNeuronTiDEOCL: tide = new CNeuronTiDEOCL(); if(!tide) ReturnFalse; if(tide.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, windows, ADAM, 1)) { m_data[index] = tide; return true; } DeleteObj(tide); break; //--- case defNeuronFEDW: fedw = new CNeuronFEDW(); if(!fedw) ReturnFalse; if(fedw.Init(iOutputs, 0, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = fedw; return true; } DeleteObj(fedw); break; //--- case defNeuronFITSOCL: fits = new CNeuronFITSOCL(); if(!fits) ReturnFalse; if(fits.Init(iOutputs, 0, OpenCL, 1, 1, 1, 0.1f, ADAM, 1)) { m_data[index] = fits; return true; } DeleteObj(fits); break; //--- case defNeuronFreDFOCL: fredf = new CNeuronFreDFOCL(); if(!fredf) ReturnFalse; if(fredf.Init(iOutputs, 0, OpenCL, 1, 1, 0.8f, false, ADAM, 1)) { m_data[index] = fredf; return true; } DeleteObj(fredf); break; //--- case defNeuronComplexMLMHAttentionOCL: compl_att = new CNeuronComplexMLMHAttention(); if(!compl_att) ReturnFalse; if(compl_att.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = compl_att; return true; } DeleteObj(compl_att); break; //--- case defNeuronATFNetOCL: atf = new CNeuronATFNetOCL(); if(!atf) ReturnFalse; if(atf.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, windows, ADAM, 1)) { m_data[index] = atf; return true; } DeleteObj(atf); break; //--- case defNeuronS3: s3 = new CNeuronS3(); if(!s3) ReturnFalse; if(s3.Init(iOutputs, 0, OpenCL, 1, 1, ADAM, 1)) { m_data[index] = s3; return true; } DeleteObj(s3); break; //--- case defNeuronMLMHAttentionMLKV: mlkv = new CNeuronMLMHAttentionMLKV(); if(!mlkv) ReturnFalse; if(mlkv.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = mlkv; return true; } DeleteObj(mlkv); break; //--- case defNeuronMVMHAttentionMLKV: mv = new CNeuronMVMHAttentionMLKV(); if(CheckPointer(mv) == POINTER_INVALID) result = false; if(mv.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = mv; return true; } DeleteObj(mv); break; //--- case defNeuronMLMHSceneConditionAttention: mlkv = new CNeuronMLMHSceneConditionAttention(); if(!mlkv) ReturnFalse; if(mlkv.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = mlkv; return true; } DeleteObj(mlkv); break; //--- case defNeuronMLCrossAttentionMLKV: crmlkv = new CNeuronMLCrossAttentionMLKV(); if(!crmlkv) ReturnFalse; if(crmlkv.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = crmlkv; return true; } DeleteObj(crmlkv); break; //--- case defNeuronMVCrossMHAttentionMLKV: mvc = new CNeuronMVCrossAttentionMLKV(); if(!mvc) ReturnFalse; if(mvc.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = mvc; return true; } DeleteObj(mvc); break; //--- case defNeuronCSCMOCL: cscm = new CNeuronCSCMOCL(); if(!cscm) ReturnFalse; if(cscm.Init(iOutputs, 0, OpenCL, windows, 1, 1, false, ADAM, 1)) { m_data[index] = cscm; return true; } DeleteObj(cscm); break; //--- case defNeuronSPyrAttentionMLKV: spyr = new CNeuronSPyrAttentionOCL(); if(!spyr) ReturnFalse; if(spyr.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 2, ADAM, 1)) { m_data[index] = spyr; return true; } DeleteObj(spyr); break; //--- case defNeuronPLROCL: plr = new CNeuronPLROCL(); if(!plr) ReturnFalse; if(plr.Init(iOutputs, 0, OpenCL, 1, 1, false, ADAM, 1)) { m_data[index] = plr; return true; } DeleteObj(plr); break; //--- case defNeuronTPM: tpm = new CNeuronTPM(); if(!tpm) ReturnFalse; if(tpm.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = tpm; return true; } DeleteObj(tpm); break; case defNeuronTPMEncoder: tpme = new CNeuronTPMEncoder(); if(!tpme) ReturnFalse; if(tpme.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = tpme; return true; } DeleteObj(tpme); break; case defNeuronSTNNEncoder: stnne = new CNeuronSTNNEncoder(); if(!stnne) ReturnFalse; if(stnne.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = stnne; return true; } DeleteObj(stnne); break; case defNeuronSTNNDecoder: stnnd = new CNeuronSTNNDecoder(); if(!stnnd) ReturnFalse; if(stnnd.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = stnnd; return true; } DeleteObj(stnnd); break; case defNeuronSparseTSF: sptsf = new CNeuronSparseTSF(); if(!sptsf) ReturnFalse; if(sptsf.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = sptsf; return true; } DeleteObj(sptsf); break; case defNeuronTEMPOOCL: tempo = new CNeuronTEMPOOCL(); if(!tempo) ReturnFalse; if(tempo.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = tempo; return true; } DeleteObj(tempo); break; case defNeuronInjectTST: inject = new CNeuronInjectTST(); if(!inject) ReturnFalse; if(inject.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = inject; return true; } DeleteObj(inject); break; case defNeuronMambaOCL: mamba = new CNeuronMambaOCL(); if(!mamba) ReturnFalse; if(mamba.Init(iOutputs, 0, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = mamba; return true; } DeleteObj(mamba); break; case defNeuronTrajLLMOCL: traj = new CNeuronTrajLLMOCL(); if(!traj) ReturnFalse; if(traj.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = traj; return true; } DeleteObj(traj); break; //--- case defNeuronUniTrajOCL: utraj = new CNeuronUniTraj(); if(!utraj) ReturnFalse; if(utraj.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = utraj; return true; } DeleteObj(utraj); break; //--- case defNeuronHiVTOCL: hivt = new CNeuronHiVTOCL(); if(!hivt) ReturnFalse; if(hivt.Init(iOutputs, 0, OpenCL, 1, 1, 1, 10, 1, 1, ADAM, 1)) { m_data[index] = hivt; return true; } DeleteObj(hivt); break; //--- case defNeuronPointNetOCL: pn = new CNeuronPointNetOCL(); if(!pn) ReturnFalse; if(pn.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = pn; return true; } DeleteObj(pn); break; //--- case defNeuronPointNet2OCL: pn = new CNeuronPointNet2OCL(); if(!pn) ReturnFalse; if(pn.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = pn; return true; } DeleteObj(pn); break; //--- case defNeuronPointFormer: pn = new CNeuronPointFormer(); if(!pn) ReturnFalse; if(pn.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = pn; return true; } DeleteObj(pn); break; case defNeuronHyperDet: pn = new CNeuronHyperDet(); if(!pn) ReturnFalse; if(pn.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, ADAM, 1)) { m_data[index] = pn; return true; } DeleteObj(pn); break; case defNeuronSceneSpecific: ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(ss.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = ss; return true; } DeleteObj(ss); break; case defNeuronSEFormer: sef = new CNeuronSEFormer(); if(!sef) ReturnFalse; if(sef.Init(iOutputs, 0, OpenCL, 1, 1, 1, false, 1, 1, ADAM, 1)) { m_data[index] = sef; return true; } DeleteObj(sef); break; case defNeuronSPFormer: spf = new CNeuronSPFormer(); if(!spf) ReturnFalse; if(spf.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 20, 1, 1, 1, ADAM, 2)) { m_data[index] = spf; return true; } DeleteObj(spf); break; case defNeuronMAFT: maft = new CNeuronMAFT(); if(!maft) ReturnFalse; if(maft.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 20, 1, 1, 1, ADAM, 2)) { m_data[index] = maft; return true; } DeleteObj(maft); break; case defNeuronGRES: gres = new CNeuronGRES(); if(!gres) ReturnFalse; if(gres.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 20, 1, 5, 1, 1, ADAM, 2)) { m_data[index] = gres; return true; } DeleteObj(gres); break; case defResidualConv: res = new CResidualConv(); if(!res) ReturnFalse; if(res.Init(iOutputs, 0, OpenCL, 1, 1, 1, ADAM, 2)) { m_data[index] = res; return true; } DeleteObj(res); break; case defNeuronRefMask: refm = new CNeuronRefMask(); if(!refm) ReturnFalse; if(refm.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = refm; return true; } DeleteObj(refm); break; case defNeuronOCM: ocm = new CNeuronOCM(); if(!ocm) ReturnFalse; if(ocm.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = ocm; return true; } DeleteObj(ocm); break; case defNeuronRelativeSelfAttention: rat = new CNeuronRelativeSelfAttention(); if(!rat) ReturnFalse; if(rat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = rat; return true; } DeleteObj(rat); break; case defNeuronRelativeCrossAttention: rcat = new CNeuronRelativeCrossAttention(); if(!rcat) ReturnFalse; if(rcat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = rcat; return true; } DeleteObj(rcat); break; case defNeuronRMAT: rmat = new CNeuronRMAT(); if(!rmat) ReturnFalse; if(rmat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = rmat; return true; } DeleteObj(rmat); break; case defNeuronMHAttentionPooling: atpool = new CNeuronMHAttentionPooling(); if(!atpool) ReturnFalse; if(atpool.Init(iOutputs, 0, OpenCL, 1, 1, 1, ADAM, 2)) { m_data[index] = atpool; return true; } DeleteObj(atpool); break; case defNeuronMotifs: motif = new CNeuronMotifs(); if(!motif) ReturnFalse; if(motif.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = motif; return true; } DeleteObj(motif); break; case defNeuronMultiScaleAttention: msat = new CNeuronMultiScaleAttention(); if(!msat) ReturnFalse; if(msat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = msat; return true; } DeleteObj(msat); break; case defNeuronMolformer: molf = new CNeuronMolformer(); if(!molf) ReturnFalse; if(molf.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = molf; return true; } DeleteObj(molf); break; case defNeuronAMCT: amct = new CNeuronAMCT(); if(!amct) ReturnFalse; if(amct.Init(iOutputs, 0, OpenCL, 1, 1, 1, 5, 1, 1, ADAM, 2)) { m_data[index] = amct; return true; } DeleteObj(amct); break; case defNeuronNAFS: nafs = new CNeuronNAFS(); if(!nafs) ReturnFalse; if(nafs.Init(iOutputs, 0, OpenCL, 1, 1, 1, ADAM, 2)) { m_data[index] = nafs; return true; } DeleteObj(nafs); break; case defNeuronDiffusion: usa = new CNeuronDiffusion(); if(!usa) ReturnFalse; if(usa.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = usa; return true; } DeleteObj(usa); break; case defNeuronHyperProjection: lorenz = new CNeuronHyperProjection(); if(!lorenz) ReturnFalse; if(lorenz.Init(iOutputs, index, OpenCL, 1, 1, ADAM, 1)) { m_data[index] = lorenz; return true; } DeleteObj(lorenz); break; case defNeuronHyperboloids: logmap = new CNeuronHyperboloids(); if(!logmap) ReturnFalse; if(logmap.Init(iOutputs, index, OpenCL, 1, 1, 1, ADAM, 1)) { m_data[index] = logmap; return true; } DeleteObj(logmap); break; case defNeuronHypDiff: hypdif = new CNeuronHypDiff(); if(!hypdif) ReturnFalse; if(hypdif.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = hypdif; return true; } DeleteObj(hypdif); break; case defNeuronPSBlock: ps_block = new CNeuronPSBlock(); if(!ps_block) ReturnFalse; if(ps_block.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 0.0f, ADAM, 1)) { m_data[index] = ps_block; return true; } DeleteObj(ps_block); break; case defNeuronPSformer: psformer = new CNeuronPSformer(); if(!psformer) ReturnFalse; if(psformer.Init(iOutputs, index, OpenCL, 1, 1, 1, 0, ADAM, 1)) { m_data[index] = psformer; return true; } DeleteObj(psformer); break; case defNeuronMASA: masa = new CNeuronMASA(); if(!masa) ReturnFalse; if(masa.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, 1, 0, 1, 3, 1, 1, -1, NULL, ADAM, 1)) { m_data[index] = masa; return true; } DeleteObj(masa); break; case defNeuronMarketObserver: mo = new CNeuronMarketObserver(); if(!mo) ReturnFalse; if(mo.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = mo; return true; } DeleteObj(mo); break; case defNeuronRLAgent: rl_ag = new CNeuronRLAgent(); if(!rl_ag) ReturnFalse; if(rl_ag.Init(iOutputs, index, OpenCL, 1, 1, 1, 0, 1, 3, ADAM, 1)) { m_data[index] = rl_ag; return true; } DeleteObj(rl_ag); break; case defNeuronControlAgent: contr_ag = new CNeuronControlAgent(); if(!contr_ag) ReturnFalse; if(contr_ag.Init(iOutputs, index, OpenCL, 1, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = contr_ag; return true; } DeleteObj(contr_ag); break; case defNeuronMASAAT: masaat = new CNeuronMASAAT(); if(!masaat) ReturnFalse; if(masaat.Init(iOutputs, index, OpenCL, 1, 1, 1, 3, 1, vec, 1, ADAM, 1)) { m_data[index] = masaat; return true; } DeleteObj(masaat); break; case defNeuronMHConvOCL: mhconv = new CNeuronMHConvOCL(); if(!mhconv) ReturnFalse; if(mhconv.Init(iOutputs, index, OpenCL, 1, 1, 1, 3, 1, 1, ADAM, 1)) { m_data[index] = mhconv; return true; } DeleteObj(mhconv); break; case defNeuronMHFeedForward: mhff = new CNeuronMHFeedForward(); if(!mhff) ReturnFalse; if(mhff.Init(iOutputs, index, OpenCL, 1, 1, 1, 3, 1, ADAM, 1)) { m_data[index] = mhff; return true; } DeleteObj(mhff); break; case defNeuronDMHAttention: dmhat = new CNeuronDMHAttention(); if(!dmhat) ReturnFalse; if(dmhat.Init(iOutputs, index, OpenCL, 1, 1, 1, 3, 1, ADAM, 1)) { m_data[index] = dmhat; return true; } DeleteObj(dmhat); break; case defNeuronCrossDMHAttention: crdmhat = new CNeuronCrossDMHAttention(); if(!crdmhat) ReturnFalse; if(crdmhat.Init(iOutputs, index, OpenCL, 1, 1, 1, 3, 1, 1, 1, ADAM, 1)) { m_data[index] = crdmhat; return true; } DeleteObj(crdmhat); break; case defNeuronMultitaskStockformer: mtstockf = new CNeuronMultitaskStockformer(); if(!mtstockf) ReturnFalse; if(mtstockf.Init(iOutputs, index, OpenCL, 1, 1, 5, 3, 1, 1, 1, ADAM, 1)) { m_data[index] = mtstockf; return true; } DeleteObj(mtstockf); break; case defNeuronFinMem: fin_mem = new CNeuronFinMem(); if(!fin_mem) ReturnFalse; if(fin_mem.Init(iOutputs, index, OpenCL, 5, 5, 5, 3, 5, 5, ADAM, 1)) { m_data[index] = fin_mem; return true; } DeleteObj(fin_mem); break; case defNeuronFinAgent: fin_agent = new CNeuronFinAgent(); if(!fin_agent) ReturnFalse; if(fin_agent.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 5, 5, 2, ADAM, 1)) { m_data[index] = fin_agent; return true; } DeleteObj(fin_agent); break; case defNeuronFinConManager: fin_con = new CNeuronFinConManager(); if(!fin_con) ReturnFalse; if(fin_con.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 5, 5, 6, ADAM, 1)) { m_data[index] = fin_con; return true; } DeleteObj(fin_con); break; case defNeuronMacroHFT: mhft = new CNeuronMacroHFT(); if(!mhft) ReturnFalse; if(mhft.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 1, 5, 6, ADAM, 1)) { m_data[index] = mhft; return true; } DeleteObj(mhft); break; case defNeuronMacroHFTHyperAgent: mhft_hyp = new CNeuronMacroHFTHyperAgent(); if(!mhft_hyp) ReturnFalse; if(mhft_hyp.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 1, 6, 6, ADAM, 1)) { m_data[index] = mhft_hyp; return true; } DeleteObj(mhft_hyp); break; case defNeuronMacroHFTvsRiskManager: mhft_risk = new CNeuronMacroHFTvsRiskManager(); if(!mhft_risk) ReturnFalse; if(mhft_risk.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 6, 6, 6, ADAM, 1)) { m_data[index] = mhft_risk; return true; } DeleteObj(mhft_risk); break; case defNeuronMultiScaleRelativeSelfAttention: rat = new CNeuronMultiScaleRelativeSelfAttention(); if(!rat) ReturnFalse; if(rat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = rat; return true; } DeleteObj(rat); break; case defNeuronMultiScaleRelativeCrossAttention: rcat = new CNeuronMultiScaleRelativeCrossAttention(); if(!rcat) ReturnFalse; if(rcat.Init(iOutputs, 0, OpenCL, 1, 1, 1, 1, 1, 1, ADAM, 2)) { m_data[index] = rcat; return true; } DeleteObj(rcat); break; case defNeuronHidformer: hidf = new CNeuronHidformer(); if(!hidf) ReturnFalse; if(hidf.Init(iOutputs, index, OpenCL, 5, 5, 10, 3, 1, 5, 6, ADAM, 1)) { m_data[index] = hidf; return true; } DeleteObj(hidf); break; case defNeuronResNeXtBlock: resnext = new CNeuronResNeXtBlock(); if(!resnext) ReturnFalse; if(resnext.Init(iOutputs, index, OpenCL, 5, 5, 3, 2, 10, 4, 32, ADAM, 1)) { m_data[index] = resnext; return true; } DeleteObj(resnext); break; case defNeuronChimera: chimera = new CNeuronChimera(); if(!chimera) ReturnFalse; if(chimera.Init(iOutputs, index, OpenCL, 5, 5, 3, 3, ADAM, 1)) { m_data[index] = chimera; return true; } DeleteObj(chimera); break; case defNeuronMoT: mot = new CNeuronMoT(); if(!mot) ReturnFalse; if(mot.Init(iOutputs, index, OpenCL, 5, 5, ADAM, 1)) { m_data[index] = mot; return true; } DeleteObj(mot); break; case defNeuronHypridDecoder: hypridDecod = new CNeuronHypridDecoder(); if(!hypridDecod) ReturnFalse; if(hypridDecod.Init(iOutputs, index, OpenCL, 5, 5, 3, 3, 1, 5, 6, ADAM, 1)) { m_data[index] = hypridDecod; return true; } DeleteObj(hypridDecod); break; case defNeuronAttraos: attraos = new CNeuronAttraos(); if(!attraos) ReturnFalse; if(attraos.Init(iOutputs, index, OpenCL, 5, 5, 10, ADAM, 1)) { m_data[index] = attraos; return true; } DeleteObj(attraos); break; case defNeuronDUET: duet = new CNeuronDUET(); if(!duet) ReturnFalse; if(duet.Init(iOutputs, index, OpenCL, 5, 5, 10, 2, 10, 4, 2, ADAM, 1)) { m_data[index] = duet; return true; } DeleteObj(duet); break; case defNeuronTransposeRCDOCL: trrcd = new CNeuronTransposeRCDOCL(); if(!trrcd) ReturnFalse; if(trrcd.Init(iOutputs, index, OpenCL, 5, 5, 10, ADAM, 1)) { m_data[index] = trrcd; return true; } DeleteObj(trrcd); break; case defNeuronTransposeVRCOCL: tvrc = new CNeuronTransposeVRCOCL(); if(!tvrc) ReturnFalse; if(tvrc.Init(iOutputs, index, OpenCL, 5, 5, 10, ADAM, 1)) { m_data[index] = tvrc; return true; } DeleteObj(tvrc); break; case defNeuronMultiWindowsConvOCL: mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv) ReturnFalse; if(mwconv.Init(iOutputs, index, OpenCL, windows, 5, 5, 1, ADAM, 1)) { m_data[index] = mwconv; return true; } DeleteObj(mwconv); break; case defNeuronAdaBN: adabn = new CNeuronAdaBN(); if(!adabn) ReturnFalse; if(adabn.Init(iOutputs, index, OpenCL, 5, 5, 10, windows, 2, 1, ADAM, 1)) { m_data[index] = adabn; return true; } DeleteObj(adabn); break; case defNeuronCATCH: catch = new CNeuronCATCH(); if(!catch) ReturnFalse; if(catch.Init(iOutputs, index, OpenCL, 8, 5, 4, 1, 2, 1, ADAM, 1)) { m_data[index] = catch; return true; } DeleteObj(catch); break; case defNeuronSkillsEncoder: skills = new CNeuronSkillsEncoder(); if(!skills) ReturnFalse; if(skills.Init(iOutputs, index, OpenCL, 8, 5, 4, 4, 1, 2, 2, ADAM, 1)) { m_data[index] = skills; return true; } DeleteObj(skills); break; case defNeuronHiSSDLowLevelControler: hissd = new CNeuronHiSSDLowLevelControler(); if(!hissd) ReturnFalse; if(hissd.Init(iOutputs, index, OpenCL, 8, 5, 4, 4, 1, 2, 1, 1, 2, ADAM, 1)) { m_data[index] = hissd; return true; } DeleteObj(hissd); break; case defNeuronLinerAttention: lineatt = new CNeuronLinerAttention(); if(!lineatt) ReturnFalse; if(lineatt.Init(iOutputs, index, OpenCL, 3, 3, 4, 1, ADAM, 1)) { m_data[index] = lineatt; return true; } DeleteObj(lineatt); break; case defNeuronCGLSTMOCL: cglstm = new CNeuronCGLSTMOCL(); if(!cglstm) ReturnFalse; if(cglstm.Init(iOutputs, index, OpenCL, 8, 3, 1, ADAM, 1)) { m_data[index] = cglstm; return true; } DeleteObj(cglstm); break; case defNeuronMHProbAttention: probatt = new CNeuronMHProbAttention(); if(!probatt) ReturnFalse; if(probatt.Init(iOutputs, index, OpenCL, 3, 3, 1, 1, ADAM, 1)) { m_data[index] = probatt; return true; } DeleteObj(probatt); break; case defNeuronTSPositionEncoder: tspe = new CNeuronTSPositionEncoder(); if(!tspe) ReturnFalse; if(tspe.Init(iOutputs, index, OpenCL, 3, 3, windows, 1, ADAM, 1)) { m_data[index] = tspe; return true; } DeleteObj(tspe); break; case defMamba4CastEmbeding: m4cemb = new CMamba4CastEmbeding(); if(!m4cemb) ReturnFalse; if(m4cemb.Init(iOutputs, index, OpenCL, 3, 9, 3, windows, ADAM, 1)) { m_data[index] = m4cemb; return true; } DeleteObj(m4cemb); break; case defNeuronMultiWindowsConvWPadOCL: mwconvpad = new CNeuronMultiWindowsConvWPadOCL(); if(!mwconvpad) ReturnFalse; if(mwconvpad.Init(iOutputs, index, OpenCL, windows, 1, 5, 1, 1, ADAM, 1)) { m_data[index] = mwconvpad; return true; } DeleteObj(mwconvpad); break; case defNeuronConcatDiff: condiff = new CNeuronConcatDiff(); if(!condiff) ReturnFalse; if(condiff.Init(iOutputs, index, OpenCL, 12, 1, 2, ADAM, 1)) { m_data[index] = condiff; return true; } DeleteObj(condiff); break; case defNeuronMantisPatching: mantpatch = new CNeuronMantisPatching(); if(!mantpatch) ReturnFalse; if(mantpatch.Init(iOutputs, index, OpenCL, 12, 3, 2, 5, 5, ADAM, 1)) { m_data[index] = mantpatch; return true; } DeleteObj(mantpatch); break; case defNeuronMantisAttentionUnit: mantatt = new CNeuronMantisAttentionUnit(); if(!mantatt) ReturnFalse; if(mantatt.Init(iOutputs, index, OpenCL, 12, 3, 2, 5, 5, 1, 5, ADAM, 1)) { m_data[index] = mantatt; return true; } DeleteObj(mantatt); break; case defNeuronTimeFoundPatching: tf_patch = new CNeuronTimeFoundPatching(); if(!tf_patch) ReturnFalse; if(tf_patch.Init(iOutputs, index, OpenCL, 12, 3, 2, 5, 5, ADAM, 1)) { m_data[index] = tf_patch; return true; } DeleteObj(tf_patch); break; case defNeuronTimeFoundTransformerUnit: tf_tu = new CNeuronTimeFoundTransformerUnit(); if(!tf_tu) ReturnFalse; if(tf_tu.Init(iOutputs, index, OpenCL, 3, 3, 2, 2, 5, 1, 1, 1, ADAM, 1)) { m_data[index] = tf_tu; return true; } DeleteObj(tf_tu); break; case defNeuronSwiGLUOCL: swiglu = new CNeuronSwiGLUOCL(); if(!swiglu) ReturnFalse; if(swiglu.Init(iOutputs, index, OpenCL, 3, 3, 2, 2, 1, ADAM, 1)) { m_data[index] = swiglu; return true; } DeleteObj(swiglu); break; case defNeuronTimeMoESparseExperts: timemoesp = new CNeuronTimeMoESparseExperts(); if(!timemoesp) ReturnFalse; if(timemoesp.Init(iOutputs, index, OpenCL, 3, 3, 2, 2, 5, 2, ADAM, 1)) { m_data[index] = timemoesp; return true; } DeleteObj(timemoesp); break; case defNeuronTimeMoEAttention: timemoeatt = new CNeuronTimeMoEAttention(); if(!timemoeatt) ReturnFalse; if(timemoeatt.Init(iOutputs, index, OpenCL, 3, 3, 2, 3, 5, 1, 1, 5, 2, 2, ADAM, 1)) { m_data[index] = timemoeatt; return true; } DeleteObj(timemoeatt); break; case defNeuronAdaptConv: ac = new CNeuronAdaptConv(); if(!ac) ReturnFalse; if(ac.Init(iOutputs, index, OpenCL, 3, 3, 2, 3, ADAM, 1)) { m_data[index] = ac; return true; } DeleteObj(ac); break; case defNeuronRoPE: rope = new CNeuronRoPE(); if(!rope) ReturnFalse; if(rope.Init(iOutputs, index, OpenCL, 3, 4, 1, ADAM, 1)) { m_data[index] = rope; return true; } DeleteObj(rope); break; case defNeuronK2VAEEncoder: k2vae = new CNeuronK2VAEEncoder(); if(!k2vae) ReturnFalse; if(k2vae.Init(iOutputs, index, OpenCL, 3, 3, 5, 1, 1, 2, 3, 3, 2, ADAM, 1)) { m_data[index] = k2vae; return true; } DeleteObj(k2vae); break; case defNeuronInterpolationAttention: ia = new CNeuronInterpolationAttention(); if(!ia) ReturnFalse; if(ia.Init(iOutputs, index, OpenCL, 3, 3, ADAM, 1)) { m_data[index] = ia; return true; } DeleteObj(ia); break; case defNeuronGinARCell: ginar_cell = new CNeuronGinARCell(); if(!ginar_cell) ReturnFalse; if(ginar_cell.Init(iOutputs, index, OpenCL, 3, 3, ADAM, 1)) { m_data[index] = ginar_cell; return true; } DeleteObj(ginar_cell); break; case defNeuronGinAR: ginar = new CNeuronGinAR(); if(!ginar) ReturnFalse; if(ginar.Init(iOutputs, index, OpenCL, 3, 3, ADAM, 1)) { m_data[index] = ginar; return true; } DeleteObj(ginar); break; case defNeuronPeriodNorm: per_norm = new CNeuronPeriodNorm(); if(!per_norm) ReturnFalse; if(per_norm.Init(iOutputs, index, OpenCL, 3, 3, 1, ADAM, 1)) { m_data[index] = per_norm; return true; } DeleteObj(per_norm); break; case defNeuronAdaptSpatialNorm: adapt_norm = new CNeuronAdaptSpatialNorm(); if(!adapt_norm) ReturnFalse; if(adapt_norm.Init(iOutputs, index, OpenCL, 3, 1, ADAM, 1)) { m_data[index] = adapt_norm; return true; } DeleteObj(adapt_norm); break; case defNeuronSCNNEncoder: scnn_enc = new CNeuronSCNNEncoder(); if(!scnn_enc) ReturnFalse; if(scnn_enc.Init(iOutputs, index, OpenCL, 3, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = scnn_enc; return true; } DeleteObj(scnn_enc); break; case defNeuronSCNN: scnn = new CNeuronSCNN(); if(!scnn) ReturnFalse; if(scnn.Init(iOutputs, index, OpenCL, 3, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = scnn; return true; } DeleteObj(scnn); break; case defNeuronAttentNorm: atn = new CNeuronAttentNorm(); if(!atn) ReturnFalse; if(atn.Init(iOutputs, index, OpenCL, 3, 3, 1, ADAM, 1)) { m_data[index] = atn; return true; } DeleteObj(atn); break; case defNeuronSAttentNorm: satn = new CNeuronSAttentNorm(); if(!satn) ReturnFalse; if(satn.Init(iOutputs, index, OpenCL, 3, 3, ADAM, 1)) { m_data[index] = satn; return true; } DeleteObj(satn); break; case defNeuronPolynomialRegression: pol_regr = new CNeuronPolynomialRegression(); if(!pol_regr) ReturnFalse; if(pol_regr.Init(iOutputs, index, OpenCL, 3, 3, 3, ADAM, 1)) { m_data[index] = pol_regr; return true; } DeleteObj(pol_regr); break; case defNeuronSSCNNEncoder: sscnn_enc = new CNeuronSSCNNEncoder(); if(!sscnn_enc) ReturnFalse; if(sscnn_enc.Init(iOutputs, index, OpenCL, 3, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = sscnn_enc; return true; } DeleteObj(sscnn_enc); break; case defNeuronSSCNN: sscnn = new CNeuronSSCNN(); if(!sscnn) ReturnFalse; if(sscnn.Init(iOutputs, index, OpenCL, 3, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = sscnn; return true; } DeleteObj(sscnn); break; case defCircleParams: circl_param = new CCircleParams(); if(!circl_param) ReturnFalse; if(circl_param.Init(iOutputs, index, OpenCL, 3, 1, ADAM, 1)) { m_data[index] = circl_param; return true; } DeleteObj(circl_param); break; case defNeuronTQMHA: tqmha = new CNeuronTQMHA(); if(!tqmha) ReturnFalse; if(tqmha.Init(iOutputs, index, OpenCL, 3, 1, 1, 5, 3, 1, ADAM, 1)) { m_data[index] = tqmha; return true; } DeleteObj(tqmha); break; case defNeuronHimNetGCRU: gcru = new CNeuronHimNetGCRU(); if(!gcru) ReturnFalse; if(gcru.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, 1, ADAM, 1)) { m_data[index] = gcru; return true; } DeleteObj(gcru); break; case defNeuronHimNetEncoder: hn_enc = new CNeuronHimNetEncoder(); if(!hn_enc) ReturnFalse; if(hn_enc.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, 1, 2, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = hn_enc; return true; } DeleteObj(hn_enc); break; case defNeuronHimNetDecoder: hn_dec = new CNeuronHimNetDecoder(); if(!hn_dec) ReturnFalse; if(hn_dec.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, 1, 2, ADAM, 1)) { m_data[index] = hn_dec; return true; } DeleteObj(hn_dec); break; case defNeuronSNSMHAttention: snsatt = new CNeuronSNSMHAttention(); if(!snsatt) ReturnFalse; if(snsatt.Init(iOutputs, index, OpenCL, 3, 3, 1, 2, 0.5f, ADAM, 1)) { m_data[index] = snsatt; return true; } DeleteObj(snsatt); break; case defNeuronFastGConv: fgcru = new CNeuronFastGConv(); if(!fgcru) ReturnFalse; if(fgcru.Init(iOutputs, index, OpenCL, 3, 3, 2, ADAM, 1)) { m_data[index] = fgcru; return true; } DeleteObj(fgcru); break; case defNeuronSAGDFN: sagdfn = new CNeuronSAGDFN(); if(!sagdfn) ReturnFalse; if(sagdfn.Init(iOutputs, index, OpenCL, 3, 2, 1, 1, 1, 1, 0.5f, 1, 3, 1, ADAM, 1)) { m_data[index] = sagdfn; return true; } DeleteObj(sagdfn); break; case defNeuronSpatialEmbedding: semb = new CNeuronSpatialEmbedding(); if(!semb) ReturnFalse; if(semb.Init(iOutputs, index, OpenCL, 3, 2, 2, ADAM, 1)) { m_data[index] = semb; return true; } DeleteObj(semb); break; case defNeuronTempEmbedding: temb = new CNeuronTempEmbedding(); if(!temb) ReturnFalse; if(temb.Init(iOutputs, index, OpenCL, 3, 2, 1, 1, 1, 1, 1, 1, ADAM, 1)) { m_data[index] = temb; return true; } DeleteObj(temb); break; case defNeuronGlobalLocalAttention: glatt = new CNeuronGlobalLocalAttention(); if(!glatt) ReturnFalse; if(glatt.Init(iOutputs, index, OpenCL, 3, 2, 1, 1, 1, 0.5f, ADAM, 1)) { m_data[index] = glatt; return true; } DeleteObj(glatt); break; case defNeuronExtralonger: exlon = new CNeuronExtralonger(); if(!exlon) ReturnFalse; if(exlon.Init(iOutputs, index, OpenCL, 3, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 0.5f, ADAM, 1)) { m_data[index] = exlon; return true; } DeleteObj(exlon); break; case defNeuronExtralongerGraph: exlongr = new CNeuronExtralongerGraph(); if(!exlongr) ReturnFalse; if(exlongr.Init(iOutputs, index, OpenCL, 3, 1, 2, 1, 2, 1, 1, 1, 1, 2, 2, 1, 0.5f, ADAM, 1)) { m_data[index] = exlongr; return true; } DeleteObj(exlongr); break; case defNeuronGraphAttention: gratt = new CNeuronGraphAttention(); if(!gratt) ReturnFalse; if(gratt.Init(iOutputs, index, OpenCL, 3, 1, 2, 2, 0.5f, ADAM, 1)) { m_data[index] = gratt; return true; } DeleteObj(gratt); break; case defNeuronSparseGraphAttention: sgratt = new CNeuronSparseGraphAttention(); if(!sgratt) ReturnFalse; if(sgratt.Init(iOutputs, index, OpenCL, 3, 1, 2, 0.5f, 2, 1, ADAM, 1)) { m_data[index] = sgratt; return true; } DeleteObj(sgratt); break; case defNeuronGlobLocGraphAtt: glgratt = new CNeuronGlobLocGraphAtt(); if(!glgratt) ReturnFalse; if(glgratt.Init(iOutputs, index, OpenCL, 3, 1, 2, 0.5f, 2, 1, ADAM, 1)) { m_data[index] = glgratt; return true; } DeleteObj(glgratt); break; case defNeuronSpikeConv: spike = new CNeuronSpikeConv(); if(!spike) ReturnFalse; if(spike.Init(iOutputs, index, OpenCL, 3, 1, 2, 1, 1, ADAM, 1)) { m_data[index] = spike; return true; } DeleteObj(spike); break; case defNeuronGateLineAttention: glineatt = new CNeuronGateLineAttention(); if(!glineatt) ReturnFalse; if(glineatt.Init(iOutputs, index, OpenCL, 3, 1, 2, 1, ADAM, 1)) { m_data[index] = glineatt; return true; } DeleteObj(glineatt); break; case defNeuronSpikeSparseAttention: spikeatt = new CNeuronSpikeSparseAttention(); if(!spikeatt) ReturnFalse; if(spikeatt.Init(iOutputs, index, OpenCL, 3, 1, 2, 0.5f, 1, 1, ADAM, 1)) { m_data[index] = spikeatt; return true; } DeleteObj(spikeatt); break; case defNeuronMoEConv: moecon = new CNeuronMoEConv(); if(!moecon) ReturnFalse; if(moecon.Init(iOutputs, index, OpenCL, 3, 1, 2, 2, 0.5f, 1, ADAM, 1)) { m_data[index] = moecon; return true; } DeleteObj(moecon); break; case defNeuronSpikingBrain: spikebrain = new CNeuronSpikingBrain(); if(!spikebrain) ReturnFalse; if(spikebrain.Init(iOutputs, index, OpenCL, 3, 1, 2, 0.5f, 1, 1, ADAM, 1)) { m_data[index] = spikebrain; return true; } DeleteObj(spikebrain); break; case defNeuronSparseSoftMax: sparsoftmax = new CNeuronSparseSoftMax(); if(!sparsoftmax) ReturnFalse; if(sparsoftmax.Init(iOutputs, index, OpenCL, 3, 2, 1, ADAM, 1)) { m_data[index] = sparsoftmax; return true; } DeleteObj(sparsoftmax); break; case defNeuronSpikeActivation: spikeact = new CNeuronSpikeActivation(); if(!spikeact) ReturnFalse; if(spikeact.Init(iOutputs, index, OpenCL, 3, ADAM, 1)) { m_data[index] = spikeact; return true; } DeleteObj(spikeact); break; case defNeuronSpikeResNeXtBlock: spikeresblock = new CNeuronSpikeResNeXtBlock(); if(!spikeresblock) ReturnFalse; if(spikeresblock.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, 2, 3, ADAM, 1)) { m_data[index] = spikeresblock; return true; } DeleteObj(spikeresblock); break; case defNeuronSpikeResNeXtBottleneck: spikeresbottleneck = new CNeuronSpikeResNeXtBottleneck(); if(!spikeresbottleneck) ReturnFalse; if(spikeresbottleneck.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, 2, 3, ADAM, 1)) { m_data[index] = spikeresbottleneck; return true; } DeleteObj(spikeresbottleneck); break; case defNeuronSpikeResNeXtResidual: spikeresresid = new CNeuronSpikeResNeXtResidual(); if(!spikeresresid) ReturnFalse; if(spikeresresid.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, ADAM, 1)) { m_data[index] = spikeresresid; return true; } DeleteObj(spikeresresid); break; case defNeuronSSAM: ssam = new CNeuronSSAM(); if(!ssam) ReturnFalse; if(ssam.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, true, ADAM, 1)) { m_data[index] = ssam; return true; } DeleteObj(ssam); break; case defNeuronSSAMResNeXtBlock: ssamblock = new CNeuronSSAMResNeXtBlock(); if(!ssamblock) ReturnFalse; if(ssamblock.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, 3, 3, 1, 3, ADAM, 1)) { m_data[index] = ssamblock; return true; } DeleteObj(ssamblock); break; case defNeuronSTFS: stfs = new CNeuronSTFS(); if(!stfs) ReturnFalse; if(stfs.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, ADAM, 1)) { m_data[index] = stfs; return true; } DeleteObj(stfs); break; case defNeuronSpikeConvGRU: sp_conv_gru = new CNeuronSpikeConvGRU(); if(!sp_conv_gru) ReturnFalse; if(sp_conv_gru.Init(iOutputs, index, OpenCL, 3, 3, 3, ADAM, 1)) { m_data[index] = sp_conv_gru; return true; } DeleteObj(sp_conv_gru); break; case defNeuronAddToStack: stack = new CNeuronAddToStack(); if(!stack) ReturnFalse; if(stack.Init(iOutputs, index, OpenCL, 3, 3, 3, ADAM, 1)) { m_data[index] = stack; return true; } DeleteObj(stack); break; case defNeuronStackCorrelation: st_correl = new CNeuronStackCorrelation(); if(!st_correl) ReturnFalse; if(st_correl.Init(iOutputs, index, OpenCL, 3, 3, 3, ADAM, 1)) { m_data[index] = st_correl; return true; } DeleteObj(st_correl); break; case defNeuronSTEFlowNetEncoder: ste_encod = new CNeuronSTEFlowNetEncoder(); if(!ste_encod) ReturnFalse; if(ste_encod.Init(iOutputs, index, OpenCL, windows, windows, 3, 3, 1, 3, 3, ADAM, 1)) { m_data[index] = ste_encod; return true; } DeleteObj(ste_encod); break; case defNeuronSTEFlowNet: ste = new CNeuronSTEFlowNet(); if(!ste) ReturnFalse; if(ste.Init(iOutputs, index, OpenCL, windows, windows, 3, 3, 1, 3, 3, ADAM, 1)) { m_data[index] = ste; return true; } DeleteObj(ste); break; case defNeuronSTEFlowNetDecoder: ste_decod = new CNeuronSTEFlowNetDecoder(); if(!ste_decod) ReturnFalse; if(ste_decod.Init(iOutputs, index, OpenCL, windows, windows, 3, 3, 1, 3, NULL, ADAM, 1)) { m_data[index] = ste_decod; return true; } DeleteObj(ste_decod); break; case defNeuronSTEFlowNetResidualBlock: ste_resid = new CNeuronSTEFlowNetResidualBlock(); if(!ste_resid) ReturnFalse; if(ste_resid.Init(iOutputs, index, OpenCL, 3, 3, 3, 1, ADAM, 1)) { m_data[index] = ste_resid; return true; } DeleteObj(ste_resid); break; case defNeuronMultiScaleStackCorrelation: mul_correl = new CNeuronMultiScaleStackCorrelation(); if(!mul_correl) ReturnFalse; if(mul_correl.Init(iOutputs, index, OpenCL, 3, 3, 3, 2, ADAM, 1)) { m_data[index] = mul_correl; return true; } DeleteObj(mul_correl); break; case defNeuronSpikeConvGRU2D: sp_conv_gru2D = new CNeuronSpikeConvGRU2D(); if(!sp_conv_gru2D) ReturnFalse; if(sp_conv_gru2D.Init(iOutputs, index, OpenCL, 3, 3, 3, ADAM, 1)) { m_data[index] = sp_conv_gru2D; return true; } DeleteObj(sp_conv_gru2D); break; case defNeuronRAFT: raft = new CNeuronRAFT(); if(!raft) ReturnFalse; if(raft.Init(iOutputs, index, OpenCL, windows, windows, 3, 3, 3, 2, ADAM, 1)) { m_data[index] = raft; return true; } DeleteObj(raft); break; case defNeuronMultScalStackCorrelBySegments: segm_corr = new CNeuronMultScalStackCorrelBySegments(); if(!segm_corr) ReturnFalse; if(segm_corr.Init(iOutputs, index, OpenCL, 2, 2, 3, 3, 2, ADAM, 1)) { m_data[index] = segm_corr; return true; } DeleteObj(segm_corr); break; case defNeuronTMAMFE: mfe = new CNeuronTMAMFE(); if(!mfe) ReturnFalse; if(mfe.Init(iOutputs, index, OpenCL, 2, mfe_windows, 3, ADAM, 1)) { m_data[index] = mfe; return true; } DeleteObj(mfe); break; case defNeuronTMAMPA: mpa = new CNeuronTMAMPA(); if(!mpa) ReturnFalse; if(mpa.Init(iOutputs, index, OpenCL, 2, 2, 3, 3, 2, ADAM, 1)) { m_data[index] = mpa; return true; } DeleteObj(mpa); break; case defNeuronTMA: tma = new CNeuronTMA(); if(!tma) ReturnFalse; if(tma.Init(iOutputs, index, OpenCL, windows, windows, 3, 3, 3, 2, 1, 2, 1, ADAM, 1)) { m_data[index] = tma; return true; } DeleteObj(tma); break; case defNeuronSpikeSTConvGRU: st_convgru = new CNeuronSpikeSTConvGRU(); if(!st_convgru) ReturnFalse; if(st_convgru.Init(iOutputs, index, OpenCL, 3, mfe_windows, 3, ADAM, 1)) { m_data[index] = st_convgru; return true; } DeleteObj(st_convgru); break; case defNeuronFERENet: fere = new CNeuronFERENet(); if(!fere) ReturnFalse; if(fere.Init(iOutputs, index, OpenCL, 3, windows, ADAM, 1)) { m_data[index] = fere; return true; } DeleteObj(fere); break; case defNeuronFGDModule: fgd = new CNeuronFGDModule(); if(!fgd) ReturnFalse; if(fgd.Init(iOutputs, index, OpenCL, 3, 3, 3, 3, ADAM, 1)) { m_data[index] = fgd; return true; } DeleteObj(fgd); break; case defNeuronEVMGRFlowNet: evmgr = new CNeuronEVMGRFlowNet(); if(!evmgr) ReturnFalse; if(evmgr.Init(iOutputs, index, OpenCL, 3, windows, ADAM, 1)) { m_data[index] = evmgr; return true; } DeleteObj(evmgr); break; case defNeuronSpikeConvBlock: spike_block = new CNeuronSpikeConvBlock(); if(!spike_block) ReturnFalse; if(spike_block.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, 1, ADAM, 1)) { m_data[index] = spike_block; return true; } DeleteObj(spike_block); break; case defNeuronMSRes: msres = new CNeuronMSRes(); if(!msres) ReturnFalse; if(msres.Init(iOutputs, index, OpenCL, 3, 1, 1, ADAM, 1)) { m_data[index] = msres; return true; } DeleteObj(msres); break; case defNeuronSpikeQKAttention: qkatt = new CNeuronSpikeQKAttention(); if(!qkatt) ReturnFalse; if(qkatt.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, ADAM, 1)) { m_data[index] = qkatt; return true; } DeleteObj(qkatt); break; case defNeuronSDSA: sdsa = new CNeuronSDSA(); if(!sdsa) ReturnFalse; if(sdsa.Init(iOutputs, index, OpenCL, 3, 1, 1, 2, ADAM, 1)) { m_data[index] = sdsa; return true; } DeleteObj(sdsa); break; case defNeuronSDformerUBlock: sdublock = new CNeuronSDformerUBlock(); if(!sdublock) ReturnFalse; if(sdublock.Init(iOutputs, index, OpenCL, 3, windows, mfe_windows, mfe_windows, 2, ADAM, 1)) { m_data[index] = sdublock; return true; } DeleteObj(sdublock); break; case defNeuronSDformerFlow: sdflow = new CNeuronSDformerFlow(); if(!sdflow) ReturnFalse; if(sdflow.Init(iOutputs, index, OpenCL, 3, sdflow_win, mfe_windows, sdflow_layers, 2, ADAM, 1)) { m_data[index] = sdflow; return true; } DeleteObj(sdflow); break; case defNeuronSpikePatchStak: sp_patch_stak = new CNeuronSpikePatchStak(); if(!sp_patch_stak) ReturnFalse; if(sp_patch_stak.Init(iOutputs, index, OpenCL, 3, 2, 2, 1, ADAM, 1)) { m_data[index] = sp_patch_stak; return true; } DeleteObj(sp_patch_stak); break; case defNeuronSpike2DSSMOCL: sp_ssm = new CNeuronSpike2DSSMOCL(); if(!sp_ssm) ReturnFalse; if(sp_ssm.Init(iOutputs, index, OpenCL, 3, 3, 2, 2, ADAM, 1)) { m_data[index] = sp_ssm; return true; } DeleteObj(sp_ssm); break; case defNeuronSpikeMamba2D: sp_mamba = new CNeuronSpikeMamba2D(); if(!sp_mamba) ReturnFalse; if(sp_mamba.Init(iOutputs, index, OpenCL, 3, 3, 2, ADAM, 1)) { m_data[index] = sp_mamba; return true; } DeleteObj(sp_mamba); break; case defNeuronSpikeSTSSM: sp_stssm = new CNeuronSpikeSTSSM(); if(!sp_stssm) ReturnFalse; if(sp_stssm.Init(iOutputs, index, OpenCL, 3, 2, 2, 2, 2, ADAM, 1)) { m_data[index] = sp_stssm; return true; } DeleteObj(sp_stssm); break; case defNeuronESTMEncoder: sp_estm = new CNeuronESTMEncoder(); if(!sp_estm) ReturnFalse; if(sp_estm.Init(iOutputs, index, OpenCL, mfe_windows, windows, 2, 2, windows, ADAM, 1)) { m_data[index] = sp_estm; return true; } DeleteObj(sp_estm); break; case defNeuronCreateFlow: cr_flow = new CNeuronCreateFlow(); if(!cr_flow) ReturnFalse; if(cr_flow.Init(iOutputs, index, OpenCL, 5, 2, 2, 1, ADAM, 1)) { m_data[index] = cr_flow; return true; } DeleteObj(cr_flow); break; case defNeuronBiDirectCorrelation: bi_corr = new CNeuronBiDirectCorrelation(); if(!bi_corr) ReturnFalse; if(bi_corr.Init(iOutputs, index, OpenCL, 2, 5, 1, 2, ADAM, 1)) { m_data[index] = bi_corr; return true; } DeleteObj(bi_corr); break; case defNeuronSATMA: satma = new CNeuronSATMA(); if(!satma) ReturnFalse; if(satma.Init(iOutputs, index, OpenCL, 2, 5, 1, 2, 1, ADAM, 1)) { m_data[index] = satma; return true; } DeleteObj(satma); break; case defNeuronBAT: bat = new CNeuronBAT(); if(!bat) ReturnFalse; if(bat.Init(iOutputs, index, OpenCL, 2, 2, 5, 1, ADAM, 1)) { m_data[index] = bat; return true; } DeleteObj(bat); break; case defNeuronBATvADM: bat = new CNeuronBATvADM(); if(!bat) ReturnFalse; if(bat.Init(iOutputs, index, OpenCL, 2, 2, 5, 1, ADAM, 1)) { m_data[index] = bat; return true; } DeleteObj(bat); break; case defNeuronSpikeSCM: scm = new CNeuronSpikeSCM; if(!scm) ReturnFalse; if(scm.Init(iOutputs, index, OpenCL, 3, 3, 2, 1, ADAM, 1)) { m_data[index] = scm; return true; } DeleteObj(scm); break; case defNeuronSpikeFAM: fam = new CNeuronSpikeFAM; if(!fam) ReturnFalse; if(fam.Init(iOutputs, index, OpenCL, 3, 2, 1, ADAM, 1)) { m_data[index] = fam; return true; } DeleteObj(fam); break; case defNeuronSpikeMDC: mdc = new CNeuronSpikeMDC; if(!mdc) ReturnFalse; if(mdc.Init(iOutputs, index, OpenCL, 3, 3, 2, 1, ADAM, 1)) { m_data[index] = mdc; return true; } DeleteObj(mdc); break; case defNeuronMDS: mds = new CNeuronMDS; if(!mds) ReturnFalse; if(mds.Init(iOutputs, index, OpenCL, 3, 2, 1, ADAM, 1)) { m_data[index] = mds; return true; } DeleteObj(mds); break; case defNeuronSpikeADM: adm = new CNeuronSpikeADM; if(!adm) ReturnFalse; if(adm.Init(iOutputs, index, OpenCL, 3, 2, 1, ADAM, 1)) { m_data[index] = adm; return true; } DeleteObj(adm); break; case defNeuronCDCSelfCorrector: selfcorrect = new CNeuronCDCSelfCorrector; if(!selfcorrect) ReturnFalse; if(selfcorrect.Init(iOutputs, index, OpenCL, 3, windows, 3, 2, 1, ADAM, 1)) { m_data[index] = selfcorrect; return true; } DeleteObj(selfcorrect); break; case defNeuronSpikeCDC: cdc = new CNeuronSpikeCDC; if(!cdc) ReturnFalse; if(cdc.Init(iOutputs, index, OpenCL, 3, windows, 2, 1, 1, ADAM, 1)) { m_data[index] = cdc; return true; } DeleteObj(cdc); break; case defNeuronSpikeEEMFLow: eemflow = new CNeuronSpikeEEMFLow; if(!eemflow) ReturnFalse; if(eemflow.Init(iOutputs, index, OpenCL, 3, 2, 5, 1, 1, ADAM, 1)) { m_data[index] = eemflow; return true; } DeleteObj(eemflow); break; case defNeuronSpikeDepthWiseConv: dw_conv = new CNeuronSpikeDepthWiseConv; if(!dw_conv) ReturnFalse; if(dw_conv.Init(iOutputs, index, OpenCL, 3, 2, 3, 1, 5, 1, ADAM, 1)) { m_data[index] = dw_conv; return true; } DeleteObj(dw_conv); break; case defNeuronMultiScaleExcitation: excit = new CNeuronMultiScaleExcitation; if(!excit) ReturnFalse; if(excit.Init(iOutputs, index, OpenCL, 3, 2, 2, 1, ADAM, 1)) { m_data[index] = excit; return true; } DeleteObj(excit); break; case defNeuronMultiScaleDifference: ms_diff = new CNeuronMultiScaleDifference; if(!ms_diff) ReturnFalse; if(ms_diff.Init(iOutputs, index, OpenCL, 3, 2, 5, windows, ADAM, 1)) { m_data[index] = ms_diff; return true; } DeleteObj(ms_diff); break; case defNeuronCorrelationEncoder: corr_enc = new CNeuronCorrelationEncoder; if(!corr_enc) ReturnFalse; if(corr_enc.Init(iOutputs, index, OpenCL, 3, 2, 5, 2, ADAM, 1)) { m_data[index] = corr_enc; return true; } DeleteObj(corr_enc); break; case defNeuronEDCFlow: edcflow = new CNeuronEDCFlow; if(!edcflow) ReturnFalse; if(edcflow.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 5, windows, 2, ADAM, 1)) { m_data[index] = edcflow; return true; } DeleteObj(edcflow); break; case defNeuronCreateICEFlow: ice = new CNeuronCreateICEFlow; if(!ice) ReturnFalse; if(ice.Init(iOutputs, index, OpenCL, 3, 5, 5, 1, ADAM, 1)) { m_data[index] = ice; return true; } DeleteObj(ice); break; case defNeuronSpikeMFE: sp_mfe = new CNeuronSpikeMFE; if(!sp_mfe) ReturnFalse; if(sp_mfe.Init(iOutputs, index, OpenCL, 3, mfe_windows, 5, ADAM, 1)) { m_data[index] = sp_mfe; return true; } DeleteObj(sp_mfe); break; case defNeuronSpikeMHCrossAttention: sp_crosatt = new CNeuronSpikeMHCrossAttention; if(!sp_crosatt) ReturnFalse; if(sp_crosatt.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 5, 2, ADAM, 1)) { m_data[index] = sp_crosatt; return true; } DeleteObj(sp_crosatt); break; case defNeuronSpikeMixFusion: mix_fusion = new CNeuronSpikeMixFusion; if(!mix_fusion) ReturnFalse; if(mix_fusion.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, ADAM, 1)) { m_data[index] = mix_fusion; return true; } DeleteObj(mix_fusion); break; case defNeuronSTFlow: stflow = new CNeuronSTFlow; if(!stflow) ReturnFalse; if(stflow.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 5, 2, ADAM, 1)) { m_data[index] = stflow; return true; } DeleteObj(stflow); break; case defNeuronPSSE: pse = new CNeuronPSSE; if(!pse) ReturnFalse; if(pse.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, ADAM, 1)) { m_data[index] = pse; return true; } DeleteObj(pse); break; case defNeuronSpikeDepthWiseResidual: dwr = new CNeuronSpikeDepthWiseResidual; if(!dwr) ReturnFalse; if(dwr.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 5, 2, ADAM, 1)) { m_data[index] = dwr; return true; } DeleteObj(dwr); break; case defNeuronSpikeSuperKernelBlock: sk_block = new CNeuronSpikeSuperKernelBlock; if(!sk_block) ReturnFalse; if(sk_block.Init(iOutputs, index, OpenCL, 3, 5, mfe_windows, mfe_windows, 5, 2, ADAM, 1)) { m_data[index] = sk_block; return true; } DeleteObj(sk_block); break; case defNeuronSpikeGMA: gma = new CNeuronSpikeGMA; if(!gma) ReturnFalse; if(gma.Init(iOutputs, index, OpenCL, 3, 5, 2, ADAM, 1)) { m_data[index] = gma; return true; } DeleteObj(gma); break; case defNeuronETROF: etrof = new CNeuronETROF; if(!etrof) ReturnFalse; if(etrof.Init(iOutputs, index, OpenCL, 3, 5, 5, ADAM, 1)) { m_data[index] = etrof; return true; } DeleteObj(etrof); break; case defNeuronPSSEFlow: pseflow = new CNeuronPSSEFlow; if(!pseflow) ReturnFalse; if(pseflow.Init(iOutputs, index, OpenCL, 3, 5, 5, ADAM, 1)) { m_data[index] = pseflow; return true; } DeleteObj(pseflow); break; case defNeuronSpikeUSR: usr = new CNeuronSpikeUSR; if(!usr) ReturnFalse; if(usr.Init(iOutputs, index, OpenCL, 3, 5, 5, 1, ADAM, 1)) { m_data[index] = usr; return true; } DeleteObj(usr); break; case defNeuronInitialFlow: init_flow = new CNeuronInitialFlow; if(!init_flow) ReturnFalse; if(init_flow.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 5, ADAM, 1)) { m_data[index] = init_flow; return true; } DeleteObj(init_flow); break; case defNeuronSpikeSMR: smr = new CNeuronSpikeSMR; if(!smr) ReturnFalse; if(smr.Init(iOutputs, index, OpenCL, 3, 5, 5, 5, ADAM, 1)) { m_data[index] = smr; return true; } DeleteObj(smr); break; case defNeuronEVAFlow: eva = new CNeuronEVAFlow; if(!eva) ReturnFalse; if(eva.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, ADAM, 1)) { m_data[index] = eva; return true; } DeleteObj(eva); break; case defNeuronResFlowHTR: htr = new CNeuronResFlowHTR; if(!htr) ReturnFalse; if(htr.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, ADAM, 1)) { m_data[index] = htr; return true; } DeleteObj(htr); break; case defNeuronResFlow: resflow = new CNeuronResFlow; if(!resflow) ReturnFalse; if(resflow.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, 2, ADAM, 1)) { m_data[index] = resflow; return true; } DeleteObj(resflow); break; case defNeuronConfGateTailAwareMoE: tailaware = new CNeuronConfGateTailAwareMoE; if(!tailaware) ReturnFalse; if(tailaware.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, ADAM, 1)) { m_data[index] = tailaware; return true; } DeleteObj(tailaware); break; case defNeuronResFlowLattice: lattice = new CNeuronResFlowLattice; if(!lattice) ReturnFalse; if(lattice.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, 2, ADAM, 1)) { m_data[index] = lattice; return true; } DeleteObj(lattice); break; case defFieldAwareParams: fapar = new CFieldAwareParams; if(!fapar) ReturnFalse; if(fapar.Init(iOutputs, index, OpenCL, 4, 5, 2, 3, 1, ADAM, 1)) { m_data[index] = fapar; return true; } DeleteObj(fapar); break; case defNeuronFieldAwareConv: faconv = new CNeuronFieldAwareConv; if(!faconv) ReturnFalse; if(faconv.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, 1, ADAM, 1)) { m_data[index] = faconv; return true; } DeleteObj(faconv); break; case defNeuronMHFAT: fat = new CNeuronMHFAT; if(!fat) ReturnFalse; if(fat.Init(iOutputs, index, OpenCL, 3, 5, 2, 3, 2, 2, 1, ADAM, 1)) { m_data[index] = fat; return true; } DeleteObj(fat); break; case defNeuronMHCrossFAT: cross_fat = new CNeuronMHCrossFAT; if(!cross_fat) ReturnFalse; if(cross_fat.Init(iOutputs, index, OpenCL, 3, 5, 2, 3, 2, 2, 2, 2, 1, ADAM, 1)) { m_data[index] = cross_fat; return true; } DeleteObj(cross_fat); break; case defNeuronFieldPatternEmbedding: faemb = new CNeuronFieldPatternEmbedding; if(!faemb) ReturnFalse; if(faemb.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, ADAM, 1)) { m_data[index] = faemb; return true; } DeleteObj(faemb); break; case defNeuronAutoToken: autotoken = new CNeuronAutoToken; if(!autotoken) ReturnFalse; if(autotoken.Init(iOutputs, index, OpenCL, 3, 5, 5, 3, 2, 1, ADAM, 1)) { m_data[index] = autotoken; return true; } DeleteObj(autotoken); break; case defNeuronMultiMixAttention: mixattention = new CNeuronMultiMixAttention; if(!mixattention) ReturnFalse; if(mixattention.Init(iOutputs, index, OpenCL, 4, 5, 2, 3, 2, 1, ADAM, 1)) { m_data[index] = mixattention; return true; } DeleteObj(mixattention); break; case defNeuronSparseMoEConv: sparsemoe = new CNeuronSparseMoEConv; if(!sparsemoe) ReturnFalse; if(sparsemoe.Init(iOutputs, index, OpenCL, 3, 5, 5, 2, 1, ADAM, 1)) { m_data[index] = sparsemoe; return true; } DeleteObj(sparsemoe); break; case defNeuronMTmixAttBlock: mixattenblock = new CNeuronMTmixAttBlock; if(!mixattenblock) ReturnFalse; if(mixattenblock.Init(iOutputs, index, OpenCL, 4, 5, 2, 3, 2, 1, ADAM, 1)) { m_data[index] = mixattenblock; return true; } DeleteObj(mixattenblock); break; case defCrossMHFlashAttention: flash_att = new CCrossMHFlashAttention; if(!flash_att) ReturnFalse; if(flash_att.Init(iOutputs, index, OpenCL, 4, 2, false, 3, 2, ADAM, 1)) { m_data[index] = flash_att; return true; } DeleteObj(flash_att); break; case defNeuronMixFeedForward: mixff = new CNeuronMixFeedForward; if(!mixff) ReturnFalse; if(mixff.Init(iOutputs, index, OpenCL, 4, 5, 2, 2, 1, ADAM, 1)) { m_data[index] = mixff; return true; } DeleteObj(mixff); break; case defNeuronOneTrans: onetrans = new CNeuronOneTrans; if(!onetrans) ReturnFalse; if(onetrans.Init(iOutputs, index, OpenCL, windows, 1, 2, 2, 2, 1, 2, 2, 1, ADAM, 1)) { m_data[index] = onetrans; return true; } DeleteObj(onetrans); break; case defNeuronSTCA: stca = new CNeuronSTCA; if(!stca) ReturnFalse; if(stca.Init(iOutputs, index, OpenCL, windows, 1, 2, 2, 2, 10, 2, 2, 3, 2, ADAM, 1)) { m_data[index] = stca; return true; } DeleteObj(stca); break; case defNeuronMHTHCrossAttention: thcatt = new CNeuronMHTHCrossAttention; if(!thcatt) ReturnFalse; if(thcatt.Init(iOutputs, index, OpenCL, 6, 1, 2, 2, 2, 1, 3, 2, ADAM, 1)) { m_data[index] = thcatt; return true; } DeleteObj(thcatt); break; default: result = false; break; } } if(result) m_data[index] = temp; else DeleteObj(temp); //--- return (result); } //+------------------------------------------------------------------+ ///\class CArrayLayer /// Class of layers collection in Neural Net. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CArrayLayer : public CArrayObj { public: /** Constructor */ CArrayLayer(void) {}; /** Destructor */~CArrayLayer(void) {}; //--- virtual bool CreateElement(uint neurons, uint outputs); ///< Method for creating new element virtual int Type(void) const { return defArrayLayer; } ///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CArrayLayer::CreateElement(uint neurons, uint outputs) { if(neurons <= 0) ReturnFalse; //--- CLayer *layer = new CLayer(outputs); if(!CheckPointer(layer) != POINTER_INVALID) ReturnFalse; //--- if(!layer.Reserve(neurons + 1)) ReturnFalse; for(uint i = 0; i <= neurons; i++) { if(!layer.CreateElement(i)) ReturnFalse; layer.IncreaseTotal(); } //--- return (Add(layer)); } //+------------------------------------------------------------------+ ///\class CNeuronProof /// Class of pooling layer ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronProof : public CNeuronBase { protected: CLayer *OutputLayer; ///< Layer of output data. Used for connection with next layer. int iWindow; ///< Input window size int iStep; ///< Size of step virtual bool feedForward(CLayer *prevLayer); ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool calcHiddenGradients(CLayer *&nextLayer); ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer. public: /** Constructor */ CNeuronProof(void) {}; /** Destructor */~CNeuronProof(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); ///< Method of initialization class.@param numOutputs Number of connections to next layer.@param myIndex Index of neuron in layer.@param window Size of input window @param step Step size.@param units_countNumber of neurons.@param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer. virtual bool calcInputGradients(CLayer *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index); ///< Method to transfer gradients to neuron in previous layer @param[in] prevNeuron Pointer to neuron.@param[in] index Index of neuron in previous layer //--- methods for working with files virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation virtual int Type(void) const { return defNeuronProof; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ ///\class CNeuronConv /// Class of convolution layer ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronConv : public CNeuronProof { protected: float param; //PReLU param virtual bool feedForward(CLayer *prevLayer); ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool calcHiddenGradients(CLayer *&nextLayer); ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer. virtual float activationFunction(float x); ///< Method to calculate activation function.@param x Input data. @return Result of activation function. virtual bool updateInputWeights(CLayer *&prevLayer); ///< Method for updating weights.@param prevLayer Pointer to previos layer. public: /** Constructor */ CNeuronConv() : param((float)0.01) { }; /** Destructor */~CNeuronConv(void) { }; //--- virtual bool calcInputGradients(CLayer *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index);///< Method to transfer gradients to neuron in previous layer @param[in] prevNeuron Pointer to neuron.@param[in] index Index of neuron in previous layer virtual float activationFunctionDerivative(float x); ///< Calculate derivative of activation function.@param[in] x Input data@return Derivative virtual int Type(void) const { return defNeuronConv; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::feedForward(CObject *&SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronProof *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = feedForward(temp_l); break; case defNeuronConv: case defNeuronProof: case defNeuronLSTM: temp_n = SourceObject; result = feedForward(temp_n.getOutputLayer()); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::updateInputWeights(CObject *&SourceObject) { bool result = false; //--- if(CheckPointer(SourceObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronProof *temp_n; switch(SourceObject.Type()) { case defLayer: temp_l = SourceObject; result = updateInputWeights(temp_l); break; case defNeuronConv: case defNeuronProof: case defNeuronLSTM: temp_n = SourceObject; temp_l = temp_n.getOutputLayer(); result = updateInputWeights(temp_l); break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::feedForward(CLayer *prevLayer) { bool result = false; //--- if(CheckPointer(prevLayer) == POINTER_INVALID) return result; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; CConnection *con; result = true; for(int i = 0; (i < total && result); i += iStep) { float sum = 0; for(int j = 0; (j < iWindow && result); j++) { temp = prevLayer.At(i + j); con = Connections.At(j); if(CheckPointer(temp) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) ReturnFalse; float val = temp.getOutputVal(); sum += val * con.weight; } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setOutputVal(activationFunction(sum)); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuronConv::activationFunction(float x) { if(x >= 0) return x; return param * x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::calcHiddenGradients(CObject *&TargetObject) { bool result = false; //--- if(CheckPointer(TargetObject) == POINTER_INVALID) return result; //--- CLayer *temp_l; CNeuronProof *temp_n; switch(TargetObject.Type()) { case defLayer: temp_l = TargetObject; result = calcHiddenGradients(temp_l); break; case defNeuronConv: case defNeuronProof: case defNeuronLSTM: switch(Type()) { case defNeuron: temp_n = TargetObject; result = temp_n.calcInputGradients(GetPointer(this), m_myIndex); break; case defNeuronLSTM: temp_n = TargetObject; temp_l = getOutputLayer(); if(!temp_n.calcInputGradients(temp_l)) { result = false; break; } result = calcHiddenGradients(temp_l); break; default: //temp_n=GetPointer(this); temp_l =/*temp_n*/getOutputLayer(); temp_n = TargetObject; result = temp_n.calcInputGradients(temp_l); break; } break; } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) ReturnFalse; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(temp.sumDOW(nextLayer)*activationFunctionDerivative(temp.getOutputVal())); } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuronConv::activationFunctionDerivative(float x) { if(x >= 0) return 1; return param; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) ReturnFalse; //--- CConnection *con; float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); for(int n = 0; n < iWindow && !IsStopped(); n++) { con = Connections.At(n); if(CheckPointer(con) == POINTER_INVALID) continue; float delta = 0; int total_i = OutputLayer.Total(); CNeuron *prev, *out; for(int i = 0; i < total_i; i++) { prev = prevLayer.At(n * iStep + i); out = OutputLayer.At(total_i - i - 1); if(CheckPointer(prev) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) continue; delta += prev.getOutputVal() * out.getGradient(); } if(optimization == SGD) con.weight += con.deltaWeight = (delta != 0 ? lr*delta : 0) + (con.deltaWeight != 0 ? alpha*con.deltaWeight : 0); else { con.mt = b1 * con.mt + (1 - b1) * delta; con.vt = (float)(b2 * con.vt + (1 - b2) * pow(delta, 2) + 0.00000001); con.weight += con.deltaWeight = (float)(lt * con.mt / sqrt(con.vt)); t++; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { iWindow = window; iStep = step; if(!CNeuronBase::Init(window, myIndex, optimization_type)) ReturnFalse; OutputLayer = new CLayer(numOutputs); if(CheckPointer(OutputLayer) == POINTER_INVALID) ReturnFalse; if(OutputLayer.Reserve(units_count)) for(int i = 0; i < units_count; i++) { if(!OutputLayer.CreateElement(i)) ReturnFalse; OutputLayer.IncreaseTotal(); } //--- if(Type() == defNeuronProof) { if(CheckPointer(Connections) != POINTER_INVALID) Connections.Clear(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronProof::~CNeuronProof(void) { DeleteObj(OutputLayer); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- int total = prevLayer.Total() - iWindow + 1; CNeuron *temp; for(int i = 0; i <= total; i += iStep) { float sum = 0; for(int j = 0; j < iWindow; j++) { temp = prevLayer.At(i + j); if(CheckPointer(temp) == POINTER_INVALID) continue; sum += temp.getOutputVal(); } temp = OutputLayer.At(i / iStep); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setOutputVal(sum / iWindow); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(nextLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || OutputLayer.Total() <= 0) ReturnFalse; //--- gradient = 0; int total = OutputLayer.Total(); CNeuron *temp; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(temp.sumDOW(nextLayer)); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID || CheckPointer(prevLayer.At(0)) == POINTER_INVALID) ReturnFalse; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronProof *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; } //--- CNeuronBase *prevNeuron, *outputNeuron; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; float prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; float stop = (float)((i - i % iStep) / iStep + 1); for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) ReturnFalse; //--- if(prevNeuron.Type() != defNeuron) { CNeuronProof *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; float prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; float stop = (float)((index - index % iStep) / iStep + 1); for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); if(CheckPointer(outputNeuron) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() / iWindow; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) ReturnFalse; //--- if(prevLayer.At(0).Type() != defNeuron) { CNeuronProof *temp = prevLayer.At(m_myIndex); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; prevLayer = temp.getOutputLayer(); if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; } //--- CNeuronBase *prevNeuron, *outputNeuron; CConnection *con; int total = prevLayer.Total(); for(int i = 0; i < total; i++) { prevNeuron = prevLayer.At(i); if(CheckPointer(prevNeuron) == POINTER_INVALID) continue; float prev_gradient = 0; int start = i - iWindow + iStep; start = (start - start % iStep) / iStep; float stop = (float)((i - i % iStep) / iStep + 1); for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = ((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + i % iStep; con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * prevNeuron.activationFunctionDerivative(prevNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(OutputLayer) == POINTER_INVALID) ReturnFalse; //--- if(prevNeuron.Type() != defNeuron) { CNeuronProof *temp = prevNeuron; return calcInputGradients(temp.getOutputLayer()); } //--- CNeuronBase *outputNeuron; CConnection *con; float prev_gradient = 0; int start = (int)index - iWindow + iStep; start = (start - start % iStep) / iStep; float stop = (float)((index - index % iStep) / iStep + 1); for(int out = (int)fmax(0, start); out < (int)fmin(OutputLayer.Total(), stop); out++) { outputNeuron = OutputLayer.At(out); int c = (int)(((int)fmin(OutputLayer.Total(), stop) - out - 1) * iStep + index % iStep); con = Connections.At(c); if(CheckPointer(outputNeuron) == POINTER_INVALID || CheckPointer(con) == POINTER_INVALID) continue; prev_gradient += outputNeuron.getGradient() * activationFunctionDerivative(outputNeuron.getOutputVal()) * con.weight; } prevNeuron.setGradient(prev_gradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBase::Save(int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) ReturnFalse; //--- if(FileWriteInteger(file_handle, t, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return Connections.Save(file_handle); } //+------------------------------------------------------------------+ ///\class CLayerDescription /// Class of layer decription. Used to describe the structure of a neural network from the main program. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CLayerDescription : public CObject { public: /** Constructor */ CLayerDescription(void); /** Destructor */~CLayerDescription(void) {}; //--- uint type; ///< Type of neurons in layer (\ref ObjectTypes) uint count; ///< Number of neurons uint window; ///< Size of input window uint window_out; ///< Size of output window uint step; ///< Step size uint layers; ///< Layers count uint batch; ///< Batch Size ENUM_ACTIVATION activation; ///< Type of activation function (#ENUM_ACTIVATION) ENUM_OPTIMIZATION optimization; ///< Type of optimization method (#ENUM_OPTIMIZATION) float probability; ///< Probability of neurons shutdown, only Dropout used uint windows[]; uint units[]; uint heads[]; vector radius; uint variables; //--- virtual bool Copy(CLayerDescription *source); //--- virtual bool operator= (CLayerDescription *source) { return Copy(source); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription::CLayerDescription(void) : type(defNeuron), count(0), window(1), step(1), layers(1), activation(TANH), optimization(ADAM), probability((float)0.1), batch(1e+4), variables(1) {} //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayerDescription::Copy(CLayerDescription *source) { if(!source) ReturnFalse; //--- type = source.type; count = source.count; window = source.window; window_out = source.window_out; step = source.step; layers = source.layers; batch = source.batch; activation = source.activation; optimization = source.optimization; probability = source.probability; ArrayCopy(windows, source.windows); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CBufferFloat; //+------------------------------------------------------------------+ ///\class CNet /// The main class of the neural network. Contains basic methods for the functioning of a neural network. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNet : public CObject { protected: bool backPropOCL(CBufferFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, bool layer_by_layer = false, ENUM_ACTIVATION SecondActivation = None); ///< Back propagation method for GPU calculation. @param[in] targetVals Target values int backPropCount; float recentAverageError; ///< Average error CArrayLayer *layers; ///< Array of layers COpenCLMy *opencl; ///< Class for working with OpenCL virtual bool OpenCLInit(void); public: /** Constructor */ CNet(void) { Create(NULL); } CNet(CArrayObj *Description) { Create(Description); } bool Create(CArrayObj *Description); /** Destructor */ ~CNet(void); bool feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true, CBufferFloat *SecondInput = NULL); ///< Feed Forward method.@param[in] prevLayer Pointer to previos layer. @param[in] window Window of input data. @param[in] tem Use Time Embedding. bool feedForward(CNet *inputNet, int inputLayer = -1, CBufferFloat *SecondInput = NULL); bool feedForward(CNet *inputNet, int inputLayer = -1, CNet *secondNet = NULL, int secondLayer = -1); bool feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true, CNet *secondNet = NULL, int secondLayer = -1); bool backProp(CArrayFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL); ///< Back propagation method. @param[in] targetVals Target values bool backProp(CArrayFloat *targetVals, CNet *secondNet = NULL, int secondLayer = -1); ///< Back propagation method. @param[in] targetVals Target values bool backPropGradient(CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, int LastLayer = -1, bool load_weights = true); ///< Back propagation method for GPU calculation. @param[in] targetVals Target values bool backPropGradient(CNet *secondNet, int secondLayer = -1, int LastLayer = -1, bool load_weights = true); void getResults(CBufferFloat *&resultVals); ///< Method to get results of feed forward process.@param[out] resultVals Array of result values void getResults(vector &resultVals); ///< Method to get results of feed forward process.@param[out] resultVals Array of result values void getResults(float &resultVals[]); ///< Method to get results of feed forward process.@param[out] resultVals Array of result values float getRecentAverageError() { return recentAverageError; } ///< Method to check quality of study. @return Average error bool Save(string file_name, float error, float undefine, float forecast, datetime time, bool common = true); ///< Save method. @param[in] file_name File name to save @param[in] error Average error @param[in] undefine Undefined percent @param[in] Foecast percent @param[in] time Last study time @param[in] common Common flag virtual bool Save(const int file_handle); bool Load(string file_name, float &error, float &undefine, float &forecast, datetime &time, bool common = true); ///< Load method. @param[in] file_name File name to save @param[out] error Average error @param[out] undefine Undefined percent @param[out] Foecast percent @param[out] time Last study time @param[in] common Common flag virtual bool Load(const int file_handle); //--- static float recentAverageSmoothingFactor; ///< Smoothing factor of average error virtual int Type(void) const { return defNet; }///< Identificator of class.@return Type of class virtual bool TrainMode(bool flag); ///< Set Training Mode Flag virtual bool GetLayerOutput(uint layer, CBufferFloat *&result); ///< Retutn Output data of layer. @param[in] layer Number of layer @param[out] return Buffer with data virtual bool GetLayerOutput(uint layer, vector &result); ///< Retutn Output data of layer. @param[in] layer Number of layer @param[out] return Buffer with data //--- virtual void SetOpenCL(COpenCLMy *obj); virtual COpenCLMy* GetOpenCL(void) { return opencl; } //--- virtual bool WeightsUpdate(CNet *net, float tau); //--- Soft Actor-Critic virtual bool GetLogProbs(vector &log_probs); virtual bool AlphasGradient(CNet *PolicyNet); virtual bool CalcLogProbs(CBufferFloat *buffer); virtual bool SetResult(float &result[]); virtual bool Clear(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNet::recentAverageSmoothingFactor = 10000.0; // Number of training samples to average over //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Create(CArrayObj *Description) { recentAverageError = 0; backPropCount = 0; if(!Description) return false; //--- int total = Description.Total(); if(total <= 0) ReturnFalse; //--- if(!layers) layers = new CArrayLayer(); if(!layers) ReturnFalse; layers.FreeMode(true); layers.Clear(); //--- CLayer *temp; CLayerDescription *desc = NULL, *next = NULL, *prev = NULL; CNeuronBase *neuron = NULL; CNeuronProof *neuron_p = NULL; uint output_count = 0; uint temp_count = 0; //--- next = Description.At(1); if(next.type == defNeuron || next.type >= defNeuronBaseOCL) { if(!OpenCLInit()) DeleteObj(opencl); } else { DeleteObj(opencl); } //--- for(int i = 0; i < total; i++) { prev = desc; desc = Description.At(i); if((i + 1) < total) { next = Description.At(i + 1); if(!next) ReturnFalse; } else next = NULL; uint outputs = (next == NULL || (next.type != defNeuron && next.type != defNeuronBaseOCL && next.type != defNeuronBaseSAMOCL) ? 0 : next.count); if(!!next && (next.type == defNeuronFQF || next.type == defNeuronSoftActorCritic)) outputs = next.count * next.window_out; temp = new CLayer(outputs); uint neurons = (desc.count + (desc.type == defNeuron || desc.type == defNeuronBaseOCL || desc.type == defNeuronBaseSAMOCL ? 1 : 0)); if(!!opencl) { CNeuronBaseOCL *neuron_ocl = NULL; CNeuronConvOCL *neuron_conv_ocl = NULL; CNeuronProofOCL *neuron_proof_ocl = NULL; CNeuronAttentionOCL *neuron_attention_ocl = NULL; CNeuronMLMHAttentionOCL *neuron_mlattention_ocl = NULL; CNeuronMLMHSparseAttention *neuron_sparseattention = NULL; CNeuronMH2AttentionOCL *neuron_mh2attention_ocl = NULL; CNeuronDropoutOCL *dropout = NULL; CNeuronBatchNormOCL *batch = NULL; CVAE *vae = NULL; CNeuronLSTMOCL *lstm = NULL; CNeuronSoftMaxOCL *softmax = NULL; CNeuronFQF *fqf = NULL; CNeuronMultiModel *multi_model = NULL; CNeuronConcatenate *concat = NULL; CNeuronEmbeddingOCL *emb = NULL; CNeuronPositionEncoder *pe = NULL; CNeuronTransposeOCL *tr = NULL; CNeuronCGConvOCL *cgc = NULL; CNeuronXCiTOCL *xcit = NULL; CNeuronDOTOCL *dot = NULL; CNeuronFAQOCL *faq = NULL; CNeuronCrossAttention *cross = NULL; CNeuronGTE *gte = NULL; CNeuronCCMROCL *ccmr = NULL; CNeuronNODEOCL *node = NULL; CNeuronConformer *conf = NULL; CNeuronRevINDenormOCL *revin = NULL; CNeuronClientOCL *client = NULL; CNeuronLearnabledPE *lpe = NULL; CNeuronUShapeAttention *usa = NULL; CNeuronPatching *patch = NULL; CNeuronTiDEOCL *tide = NULL; CNeuronFEDW *fedw = NULL; CNeuronFITSOCL *fits = NULL; CNeuronFreDFOCL *fredf = NULL; CNeuronComplexMLMHAttention *comp_att = NULL; CNeuronATFNetOCL *atf = NULL; CNeuronS3 *s3 = NULL; CNeuronMLMHAttentionMLKV *mlkv = NULL; CNeuronMLCrossAttentionMLKV *crmlkv = NULL; CNeuronCSCMOCL *cscm = NULL; CNeuronSPyrAttentionOCL *spyr = NULL; CNeuronPLROCL *plr = NULL; CNeuronTPMEncoder *tpme = NULL; CNeuronTPM *tpm = NULL; CNeuronSTNNEncoder *stnne = NULL; CNeuronSTNNDecoder *stnnd = NULL; CNeuronSparseTSF *sptsf = NULL; CNeuronTEMPOOCL *tempo = NULL; CNeuronInjectTST *inject = NULL; CNeuronMambaOCL *mamba = NULL; CNeuronTrajLLMOCL *traj = NULL; CNeuronUniTraj *utraj = NULL; CNeuronHiVTOCL *hivt = NULL; CNeuronPointNetOCL *pn = NULL; CNeuronSceneSpecific *ss = NULL; CNeuronSEFormer *sef = NULL; CNeuronSPFormer *spf = NULL; CNeuronMAFT *maft = NULL; CNeuronGRES *gres = NULL; CNeuronRefMask *refm = NULL; CNeuronRelativeSelfAttention *rat = NULL; CNeuronRMAT *rmat = NULL; CNeuronMHAttentionPooling *atpool = NULL; CNeuronMotifs *motif = NULL; CNeuronMultiScaleAttention *msat = NULL; CNeuronMolformer *molf = NULL; CNeuronAMCT *amct = NULL; CNeuronNAFS *nafs = NULL; CNeuronHyperProjection *lorentz = NULL; CNeuronHyperboloids *logmap = NULL; CNeuronHypDiff *hypdiff = NULL; CNeuronBaseSAMOCL *sam_base = NULL; CNeuronConvSAMOCL *sam_conv = NULL; CNeuronPSBlock *ps_block = NULL; CNeuronPSformer *psformer = NULL; CNeuronMASA *masa = NULL; CNeuronMarketObserver *mo = NULL; CNeuronRLAgent *rl_agent = NULL; CNeuronControlAgent *contr_ag = NULL; CNeuronMASAAT *masaat = NULL; CNeuronMHConvOCL *mhconv = NULL; CNeuronMHFeedForward *mhff = NULL; CNeuronDMHAttention *dmhat = NULL; CNeuronCrossDMHAttention *crdmhat = NULL; CNeuronMultitaskStockformer *mtstockf = NULL; CNeuronFinMem *fin_mem = NULL; CNeuronFinAgent *fin_agent = NULL; CNeuronFinConManager *fin_con = NULL; CNeuronMacroHFT *mhft = NULL; CNeuronMacroHFTHyperAgent *mhft_hyp = NULL; CNeuronMacroHFTvsRiskManager *mhft_risk = NULL; CNeuronHidformer *hidf = NULL; CNeuronResNeXtBlock *resnext = NULL; CNeuronChimera *chimera = NULL; CNeuronMoT *mot = NULL; CNeuronHypridDecoder *hypridDecod = NULL; CNeuronAttraos *attraos = NULL; CNeuronDUET *duet = NULL; CNeuronMultiWindowsConvOCL *mwconv = NULL; CNeuronAdaBN *adabn = NULL; CNeuronTransposeRCDOCL *trcd = NULL; CNeuronTransposeVRCOCL *tvrc = NULL; CNeuronCATCH *catch = NULL; CNeuronSkillsEncoder *skills = NULL; CNeuronHiSSDLowLevelControler *hissd = NULL; CNeuronLinerAttention *lineatt = NULL; CNeuronCGLSTMOCL *cglstm = NULL; CNeuronMHProbAttention *probatt = NULL; CNeuronTSPositionEncoder *tspe = NULL; CMamba4CastEmbeding* m4cemb = NULL; CNeuronMultiWindowsConvWPadOCL *mwconvpad = NULL; CNeuronConcatDiff *condiff = NULL; CNeuronMantisPatching *mantpatch = NULL; CNeuronMantisAttentionUnit *mantatt = NULL; CNeuronTimeFoundPatching *tf_patch = NULL; CNeuronTimeFoundTransformerUnit *tf_tu = NULL; CNeuronSwiGLUOCL *swiglu = NULL; CNeuronTimeMoESparseExperts *timemoesp = NULL; CNeuronTimeMoEAttention *timemoeatt = NULL; CNeuronAdaptConv *ac = NULL; CNeuronRoPE *rope = NULL; CNeuronK2VAEEncoder *k2vae = NULL; CNeuronInterpolationAttention *ia = NULL; CNeuronGinARCell *ginar_cell = NULL; CNeuronGinAR *ginar = NULL; CNeuronPeriodNorm *per_norm = NULL; CNeuronAdaptSpatialNorm *adapt_norm = NULL; CNeuronSCNNEncoder* scnn_enc = NULL; CNeuronSCNN* scnn = NULL; CNeuronAttentNorm* atn = NULL; CNeuronSAttentNorm* satn = NULL; CNeuronPolynomialRegression* pol_regr = NULL; CNeuronSSCNNEncoder* sscnn_enc = NULL; CNeuronSSCNN* sscnn = NULL; CNeuronTQMHA* tqmha = NULL; CNeuronHimNetEncoder* hn_enc = NULL; CNeuronHimNetDecoder* hn_dec = NULL; CNeuronSAGDFN* sagdfn = NULL; CNeuronSpatialEmbedding* semb = NULL; CNeuronTempEmbedding* temb = NULL; CNeuronGlobalLocalAttention* glatt = NULL; CNeuronExtralonger* exlon = NULL; CNeuronExtralongerGraph* exlongr = NULL; CNeuronSpikeConv* spike = NULL; CNeuronGateLineAttention* glineatt = NULL; CNeuronSpikeSparseAttention* spikeatt = NULL; CNeuronMoEConv *moecon = NULL; CNeuronSpikingBrain* spikebrain = NULL; CNeuronSparseSoftMax* sparsoftmax = NULL; CNeuronSpikeActivation* spikeact = NULL; CNeuronSpikeResNeXtBlock* spikeresblock = NULL; CNeuronSpikeResNeXtBottleneck* spikeresbottleneck = NULL; CNeuronSpikeResNeXtResidual* spikeresresid = NULL; CNeuronSSAM* ssam = NULL; CNeuronSSAMResNeXtBlock* ssamblock = NULL; CNeuronSTFS* stfs = NULL; CNeuronSpikeConvGRU* sp_conv_gru = NULL; CNeuronAddToStack* stack = NULL; CNeuronStackCorrelation* st_correl = NULL; CNeuronSTEFlowNetEncoder* ste_encod = NULL; CNeuronSTEFlowNetResidualBlock* ste_resid = NULL; CNeuronSTEFlowNetDecoder* ste_decod = NULL; CNeuronSTEFlowNet* ste = NULL; CNeuronMultiScaleStackCorrelation* mul_correl = NULL; CNeuronSpikeConvGRU2D* sp_conv_gru2D = NULL; CNeuronRAFT* raft = NULL; CNeuronMultScalStackCorrelBySegments* segm_corr = NULL; CNeuronTMAMFE* mfe = NULL; CNeuronTMAMPA* mpa = NULL; CNeuronTMA* tma = NULL; CNeuronFERENet* fere = NULL; CNeuronSpikeSTConvGRU* st_convgru = NULL; CNeuronFGDModule* fgd = NULL; CNeuronEVMGRFlowNet* evmgr = NULL; CNeuronSpikeConvBlock* spike_block = NULL; CNeuronMSRes* msres = NULL; CNeuronSpikeQKAttention* qkatt = NULL; CNeuronSDSA* sdsa = NULL; CNeuronSDformerUBlock* sdublock = NULL; CNeuronSDformerFlow* sdflow = NULL; CNeuronSpikePatchStak* sp_patch_stak = NULL; CNeuronSpike2DSSMOCL* sp_ssm = NULL; CNeuronSpikeMamba2D* sp_mamba = NULL; CNeuronSpikeSTSSM* sp_stssm = NULL; CNeuronESTMEncoder* sp_estm = NULL; CNeuronCreateFlow* cr_flow = NULL; CNeuronBiDirectCorrelation* bi_corr = NULL; CNeuronSATMA* satma = NULL; CNeuronBAT* bat = NULL; CNeuronSpikeSCM* scm = NULL; CNeuronSpikeFAM* fam = NULL; CNeuronSpikeMDC* mdc = NULL; CNeuronMDS* mds = NULL; CNeuronSpikeADM* adm = NULL; CNeuronCDCSelfCorrector* selfcorrect = NULL; CNeuronSpikeCDC* cdc = NULL; CNeuronSpikeEEMFLow* eemflow = NULL; CNeuronSpikeDepthWiseConv* dw_conv = NULL; CNeuronMultiScaleExcitation* excit = NULL; CNeuronMultiScaleDifference* ms_diff = NULL; CNeuronCorrelationEncoder* corr_enc = NULL; CNeuronEDCFlow* edcflow = NULL; CNeuronCreateICEFlow* ice = NULL; CNeuronSpikeMFE* sp_mfe = NULL; CNeuronSpikeMHCrossAttention* sp_crosatt = NULL; CNeuronSpikeMixFusion* mix_fusion = NULL; CNeuronSTFlow* stflow = NULL; CNeuronPSSE* pse = NULL; CNeuronSpikeDepthWiseResidual* dwr = NULL; CNeuronSpikeSuperKernelBlock* sk_block = NULL; CNeuronSpikeGMA* gma = NULL; CNeuronETROF* etrof = NULL; CNeuronPSSEFlow* pseflow = NULL; CNeuronSpikeUSR* usr = NULL; CNeuronInitialFlow* init_flow = NULL; CNeuronSpikeSMR* smr = NULL; CNeuronEVAFlow* eva = NULL; CNeuronResFlowHTR* htr = NULL; CNeuronResFlow* resflow = NULL; CNeuronConfGateTailAwareMoE* tailaware = NULL; CNeuronResFlowLattice* lattice = NULL; CNeuronFieldAwareConv* faconv = NULL; CNeuronMHFAT* fat = NULL; CNeuronMHCrossFAT* cross_fat = NULL; CNeuronFieldPatternEmbedding* faemb = NULL; CNeuronAutoToken* autotoken = NULL; CNeuronMultiMixAttention* mixattention = NULL; CNeuronSparseMoEConv* sparsemoe = NULL; CNeuronMTmixAttBlock* mixattenblock = NULL; CCrossMHFlashAttention* flash_att = NULL; CNeuronMixFeedForward* mixff = NULL; CNeuronOneTrans* onetrans = NULL; CNeuronMHTHCrossAttention* thcatt = NULL; CNeuronSTCA* stca = NULL; //--- switch(desc.type) { case defNeuron: case defNeuronBase: case defNeuronBaseOCL: neuron_ocl = new CNeuronBaseOCL(); if(CheckPointer(neuron_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_ocl.Init(outputs, 0, opencl, desc.count, desc.optimization, desc.batch)) { DeleteObj(neuron_ocl); DeleteObj(temp); ReturnFalse; } neuron_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_ocl)) { DeleteObj(neuron_ocl); DeleteObj(temp); ReturnFalse; } neuron_ocl = NULL; break; case defNeuronBaseSAMOCL: sam_base = new CNeuronBaseSAMOCL(); if(CheckPointer(sam_base) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!sam_base.Init(outputs, 0, opencl, desc.count, desc.probability, desc.optimization, desc.batch)) { delete sam_base; DeleteObj(temp); ReturnFalse; } sam_base.SetActivationFunction(desc.activation); if(!temp.Add(sam_base)) { delete sam_base; DeleteObj(temp); ReturnFalse; } sam_base = NULL; break; //--- case defNeuronConvOCL: neuron_conv_ocl = new CNeuronConvOCL(); if(CheckPointer(neuron_conv_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_conv_ocl.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.layers, desc.optimization, desc.batch)) { delete neuron_conv_ocl; DeleteObj(temp); ReturnFalse; } neuron_conv_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_conv_ocl)) { delete neuron_conv_ocl; DeleteObj(temp); ReturnFalse; } neuron_conv_ocl = NULL; break; //--- case defNeuronConvSAMOCL: sam_conv = new CNeuronConvSAMOCL(); if(CheckPointer(sam_conv) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!sam_conv.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, 1, desc.probability, desc.optimization, desc.batch)) { delete sam_conv; DeleteObj(temp); ReturnFalse; } sam_conv.SetActivationFunction(desc.activation); if(!temp.Add(sam_conv)) { delete sam_conv; DeleteObj(temp); ReturnFalse; } sam_conv = NULL; break; //--- case defNeuronProofOCL: neuron_proof_ocl = new CNeuronProofOCL(); if(!neuron_proof_ocl) { DeleteObj(temp); ReturnFalse; } if(!neuron_proof_ocl.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, desc.optimization, desc.batch)) { delete neuron_proof_ocl; DeleteObj(temp); ReturnFalse; } neuron_proof_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_proof_ocl)) { delete neuron_proof_ocl; DeleteObj(temp); ReturnFalse; } neuron_proof_ocl = NULL; break; //--- case defNeuronAttentionOCL: neuron_attention_ocl = new CNeuronAttentionOCL(); if(CheckPointer(neuron_attention_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_attention_ocl.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { delete neuron_attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_attention_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_attention_ocl)) { delete neuron_attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_attention_ocl = NULL; break; //--- case defNeuronMHAttentionOCL: neuron_attention_ocl = new CNeuronMHAttentionOCL(); if(CheckPointer(neuron_attention_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_attention_ocl.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { delete neuron_attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_attention_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_attention_ocl)) { delete neuron_attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_attention_ocl = NULL; break; //--- case defNeuronMLMHAttentionOCL: neuron_mlattention_ocl = new CNeuronMLMHAttentionOCL(); if(CheckPointer(neuron_mlattention_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_mlattention_ocl.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { delete neuron_mlattention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mlattention_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_mlattention_ocl)) { delete neuron_mlattention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mlattention_ocl = NULL; break; //--- case defNeuronMLMHSparseAttentionOCL: neuron_sparseattention = new CNeuronMLMHSparseAttention(); if(CheckPointer(neuron_sparseattention) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_sparseattention.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { delete neuron_sparseattention; DeleteObj(temp); ReturnFalse; } neuron_sparseattention.SetActivationFunction(desc.activation); neuron_sparseattention.Sparse(desc.probability); if(!temp.Add(neuron_sparseattention)) { delete neuron_mlattention_ocl; DeleteObj(temp); ReturnFalse; } neuron_sparseattention = NULL; break; //--- case defNeuronMFTOCL: neuron_mlattention_ocl = new CNeuronMFTOCL(); if(CheckPointer(neuron_mlattention_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_mlattention_ocl.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { delete neuron_mlattention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mlattention_ocl.SetActivationFunction(desc.activation); if(!temp.Add(neuron_mlattention_ocl)) { delete neuron_mlattention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mlattention_ocl = NULL; break; case defNeuronMH2AttentionOCL: neuron_mh2attention_ocl = new CNeuronMH2AttentionOCL(); if(CheckPointer(neuron_mh2attention_ocl) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!neuron_mh2attention_ocl.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.optimization, desc.batch)) { delete neuron_mh2attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mh2attention_ocl.SetActivationFunction(None); if(!temp.Add(neuron_mh2attention_ocl)) { delete neuron_mh2attention_ocl; DeleteObj(temp); ReturnFalse; } neuron_mh2attention_ocl = NULL; break; //--- case defNeuronXCiTOCL: xcit = new CNeuronXCiTOCL(); if(CheckPointer(xcit) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!xcit.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { delete xcit; DeleteObj(temp); ReturnFalse; } if(!temp.Add(xcit)) { delete xcit; DeleteObj(temp); ReturnFalse; } xcit = NULL; break; //--- case defNeuronDropoutOCL: dropout = new CNeuronDropoutOCL(); if(CheckPointer(dropout) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!dropout.Init(outputs, 0, opencl, desc.count, desc.probability, desc.optimization, desc.batch)) { delete dropout; DeleteObj(temp); ReturnFalse; } if(!temp.Add(dropout)) { delete dropout; DeleteObj(temp); ReturnFalse; } dropout = NULL; break; //--- case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = (desc.type == defNeuronBatchNormWithNoise ? new CNeuronBatchNormWithNoise() : new CNeuronBatchNormOCL()); if(CheckPointer(batch) == POINTER_INVALID) { DeleteObj(temp); ReturnFalse; } if(!batch.Init(outputs, 0, opencl, desc.count, desc.batch, desc.optimization)) { delete batch; DeleteObj(temp); ReturnFalse; } batch.SetActivationFunction(desc.activation); if(!temp.Add(batch)) { delete batch; DeleteObj(temp); ReturnFalse; } batch = NULL; break; //--- case defNeuronVAEOCL: vae = new CVAE(); if(!vae) { DeleteObj(temp); ReturnFalse; } if(!vae.Init(outputs, 0, opencl, desc.count, desc.optimization, desc.batch)) { delete vae; DeleteObj(temp); ReturnFalse; } if(!temp.Add(vae)) { delete vae; DeleteObj(temp); ReturnFalse; } vae = NULL; break; case defNeuronLSTMOCL: lstm = new CNeuronLSTMOCL(); if(!lstm) { DeleteObj(temp); ReturnFalse; } desc.layers = MathMax(desc.layers, 1); if(!lstm.Init(outputs, 0, opencl, desc.count, desc.layers, desc.optimization, desc.batch)) { delete lstm; DeleteObj(temp); ReturnFalse; } if(!!prev) if(!lstm.SetInputs(prev.count / desc.layers)) { delete lstm; DeleteObj(temp); ReturnFalse; } if(!temp.Add(lstm)) { delete lstm; DeleteObj(temp); ReturnFalse; } lstm = NULL; break; //--- case defNeuronSoftMaxOCL: softmax = new CNeuronSoftMaxOCL(); if(!softmax) { DeleteObj(temp); ReturnFalse; } if(!softmax.Init(outputs, 0, opencl, desc.count * desc.step, desc.optimization, desc.batch)) { delete softmax; DeleteObj(temp); ReturnFalse; } softmax.SetHeads(desc.step); if(!temp.Add(softmax)) { delete softmax; DeleteObj(temp); ReturnFalse; } softmax = NULL; break; //--- case defNeuronFQF: fqf = new CNeuronFQF(); if(!fqf) { DeleteObj(temp); ReturnFalse; } if(!fqf.Init(outputs, 0, opencl, desc.count, desc.window_out, prev.count * (prev.type == defNeuronConv ? prev.window_out : 1), desc.optimization, desc.batch)) { delete fqf; DeleteObj(temp); ReturnFalse; } if(!temp.Add(fqf.AsObject())) { delete fqf; DeleteObj(temp); ReturnFalse; } fqf = NULL; break; case defNeuronMultiModels: multi_model = new CNeuronMultiModel(); if(!multi_model) { DeleteObj(temp); ReturnFalse; } if(!multi_model.Init(desc.window, 0, opencl, desc.count, ADAM, desc.step)) { delete multi_model; DeleteObj(temp); ReturnFalse; } multi_model.SetActivationFunction(desc.activation); if(!temp.Add(multi_model)) { delete multi_model; DeleteObj(temp); ReturnFalse; } multi_model = NULL; break; case defNeuronConcatenate: concat = new CNeuronConcatenate(); if(!concat) { DeleteObj(temp); ReturnFalse; } if(!concat.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, ADAM, desc.batch)) { delete concat; DeleteObj(temp); ReturnFalse; } concat.SetActivationFunction(desc.activation); if(!temp.Add(concat)) { delete concat; DeleteObj(temp); ReturnFalse; } concat = NULL; break; //--- case defNeuronSoftActorCritic: fqf = new CNeuronSoftActorCritic(); if(!fqf) { DeleteObj(temp); ReturnFalse; } if(!fqf.Init(outputs, 0, opencl, desc.count, desc.window_out, prev.count * (prev.type == defNeuronConv ? prev.window_out : 1), desc.optimization, desc.batch)) { delete fqf; DeleteObj(temp); ReturnFalse; } fqf.SetActivationFunction(desc.activation); if(!temp.Add(fqf.AsObject())) { delete fqf; DeleteObj(temp); ReturnFalse; } fqf = NULL; break; //--- case defNeuronEmbeddingOCL: emb = new CNeuronEmbeddingOCL(); if(!emb) { DeleteObj(temp); ReturnFalse; } if(!emb.Init(outputs, 0, opencl, desc.count, desc.window_out, desc.windows)) { DeleteObj(temp); delete emb; ReturnFalse; } if(!temp.Add(emb)) { DeleteObj(temp); delete emb; ReturnFalse; } break; //--- case defNeuronPEOCL: pe = new CNeuronPositionEncoder(); if(!pe) { DeleteObj(temp); ReturnFalse; } if(!pe.Init(outputs, 0, opencl, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete pe; ReturnFalse; } if(!temp.Add(pe)) { DeleteObj(temp); delete pe; ReturnFalse; } break; //--- case defNeuronTransposeOCL: tr = new CNeuronTransposeOCL(); if(!tr) { DeleteObj(temp); ReturnFalse; } if(!tr.Init(outputs, 0, opencl, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete tr; ReturnFalse; } if(!temp.Add(tr)) { DeleteObj(temp); delete tr; ReturnFalse; } break; case defNeuronCGConvOCL: cgc = new CNeuronCGConvOCL(); if(!cgc) { DeleteObj(temp); ReturnFalse; } if(!cgc.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete cgc; ReturnFalse; } if(!temp.Add(cgc)) { DeleteObj(temp); delete cgc; ReturnFalse; } break; //--- case defNeuronDOTOCL: dot = new CNeuronDOTOCL(); if(!dot) { DeleteObj(temp); ReturnFalse; } neurons = prev.count * MathMax((prev.type == defNeuronConv ? prev.window_out : prev.window), 1) / desc.count; if(!dot.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, neurons, desc.optimization, desc.batch)) { DeleteObj(temp); delete dot; ReturnFalse; } if(!temp.Add(dot)) { DeleteObj(temp); delete dot; ReturnFalse; } break; //--- case defNeuronFAQOCL: faq = new CNeuronFAQOCL(); if(!faq) { DeleteObj(temp); ReturnFalse; } if(!faq.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete faq; ReturnFalse; } if(!temp.Add(faq)) { DeleteObj(temp); delete faq; ReturnFalse; } break; //--- case defNeuronCrossAttenOCL: cross = new CNeuronCrossAttention(); if(!cross) { DeleteObj(temp); ReturnFalse; } if(!cross.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.step, desc.units[0], desc.windows[1], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete cross; ReturnFalse; } if(!temp.Add(cross)) { DeleteObj(temp); delete cross; ReturnFalse; } break; //--- case defNeuronGTE: gte = new CNeuronGTE(); if(!gte) { DeleteObj(temp); ReturnFalse; } if(!gte.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete gte; ReturnFalse; } if(!temp.Add(gte)) { DeleteObj(temp); delete gte; ReturnFalse; } break; case defNeuronCCMROCL: ccmr = new CNeuronCCMROCL(); if(!ccmr) { DeleteObj(temp); ReturnFalse; } if(!ccmr.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete ccmr; ReturnFalse; } if(!temp.Add(ccmr)) { DeleteObj(temp); delete ccmr; ReturnFalse; } break; case defNeuronNODEOCL: node = new CNeuronNODEOCL(); if(!node) { DeleteObj(temp); ReturnFalse; } if(!node.Init(outputs, 0, opencl, desc.window, desc.step, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete node; ReturnFalse; } if(!temp.Add(node)) { DeleteObj(temp); delete node; ReturnFalse; } break; case defNeuronConformerOCL: conf = new CNeuronConformer(); if(!conf) { DeleteObj(temp); ReturnFalse; } if(!conf.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.layers, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete conf; ReturnFalse; } if(!temp.Add(conf)) { DeleteObj(temp); delete conf; ReturnFalse; } break; case defNeuronRevInDenormOCL: if(int(desc.layers) >= layers.Total()) { DeleteObj(temp); ReturnFalse; } if(!(((CLayer *)layers.At(desc.layers)).At(0).Type() == defNeuronBatchNormOCL || ((CLayer *)layers.At(desc.layers)).At(0).Type() == defNeuronBatchNormWithNoise)) { DeleteObj(temp); ReturnFalse; } revin = new CNeuronRevINDenormOCL(); if(!revin) { DeleteObj(temp); ReturnFalse; } if(!revin.Init(outputs, 0, opencl, desc.count, desc.layers, ((CLayer *)layers.At(desc.layers)).At(0))) { DeleteObj(temp); DeleteObj(revin); ReturnFalse; } if(!temp.Add(revin)) { DeleteObj(temp); DeleteObj(revin); ReturnFalse; } break; case defNeuronClientOCL: client = new CNeuronClientOCL(); if(!client) { DeleteObj(temp); ReturnFalse; } if(!client.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.layers, desc.count, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); delete client; ReturnFalse; } if(!temp.Add(client)) { DeleteObj(temp); delete client; ReturnFalse; } break; case defNeuronLearnabledPE: lpe = new CNeuronLearnabledPE(); if(!lpe) { DeleteObj(temp); ReturnFalse; } if(!lpe.Init(outputs, 0, opencl, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete lpe; ReturnFalse; } if(!temp.Add(lpe)) { DeleteObj(temp); delete lpe; ReturnFalse; } break; case defNeuronUShapeAttention: usa = new CNeuronUShapeAttention(); if(!usa) { DeleteObj(temp); ReturnFalse; } if(!usa.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.batch, desc.optimization, 1)) { DeleteObj(temp); delete usa; ReturnFalse; } if(!temp.Add(usa)) { DeleteObj(temp); delete usa; ReturnFalse; } break; //--- case defNeuronPatchingOCL: patch = new CNeuronPatching(); if(!patch) { DeleteObj(temp); ReturnFalse; } if(!patch.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete patch; ReturnFalse; } patch.SetActivationFunction(desc.activation); if(!temp.Add(patch)) { DeleteObj(temp); delete patch; ReturnFalse; } break; //--- case defNeuronTiDEOCL: tide = new CNeuronTiDEOCL(); if(!tide) { DeleteObj(temp); ReturnFalse; } if(!tide.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); delete tide; ReturnFalse; } if(!temp.Add(tide)) { DeleteObj(temp); delete tide; ReturnFalse; } break; //--- case defNeuronFEDW: fedw = new CNeuronFEDW(); if(!fedw) { DeleteObj(temp); ReturnFalse; } if(!fedw.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); delete fedw; ReturnFalse; } if(!temp.Add(fedw)) { DeleteObj(temp); delete fedw; ReturnFalse; } break; //--- case defNeuronFITSOCL: fits = new CNeuronFITSOCL(); if(!fits) { DeleteObj(temp); ReturnFalse; } if(!fits.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); delete fits; ReturnFalse; } if(!temp.Add(fits)) { DeleteObj(temp); delete fits; ReturnFalse; } break; //--- case defNeuronFreDFOCL: fredf = new CNeuronFreDFOCL(); if(!fredf) { DeleteObj(temp); ReturnFalse; } if(!fredf.Init(outputs, 0, opencl, desc.window, desc.count, desc.probability, bool(desc.step), desc.optimization, desc.batch)) { DeleteObj(temp); delete fredf; ReturnFalse; } if(!temp.Add(fredf)) { DeleteObj(temp); delete fredf; ReturnFalse; } break; //--- case defNeuronComplexMLMHAttentionOCL: comp_att = new CNeuronComplexMLMHAttention(); if(!comp_att) { DeleteObj(temp); ReturnFalse; } if(!comp_att.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete comp_att; ReturnFalse; } if(!temp.Add(comp_att)) { DeleteObj(temp); delete comp_att; ReturnFalse; } break; //--- case defNeuronATFNetOCL: atf = new CNeuronATFNetOCL(); if(!atf) { DeleteObj(temp); ReturnFalse; } if(!atf.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.layers, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); delete atf; ReturnFalse; } if(!temp.Add(atf)) { DeleteObj(temp); delete atf; ReturnFalse; } break; //--- case defNeuronS3: s3 = new CNeuronS3(); if(!s3) { DeleteObj(temp); ReturnFalse; } if(!s3.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete s3; ReturnFalse; } if(!temp.Add(s3)) { DeleteObj(temp); delete s3; ReturnFalse; } break; case defNeuronMLMHAttentionMLKV: mlkv = new CNeuronMLMHAttentionMLKV(); if(!mlkv) { DeleteObj(temp); ReturnFalse; } if(!mlkv.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete mlkv; ReturnFalse; } if(!temp.Add(mlkv)) { DeleteObj(temp); delete mlkv; ReturnFalse; } break; case defNeuronMLMHSceneConditionAttention: mlkv = new CNeuronMLMHSceneConditionAttention(); if(!mlkv) { DeleteObj(temp); ReturnFalse; } if(!mlkv.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete mlkv; ReturnFalse; } if(!temp.Add(mlkv)) { DeleteObj(temp); delete mlkv; ReturnFalse; } break; //--- case defNeuronMLCrossAttentionMLKV: crmlkv = new CNeuronMLCrossAttentionMLKV(); if(!crmlkv) { DeleteObj(temp); ReturnFalse; } if(!crmlkv.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.heads[0], desc.windows[1], desc.heads[1], desc.units[0], desc.units[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete crmlkv; ReturnFalse; } if(!temp.Add(crmlkv)) { DeleteObj(temp); delete crmlkv; ReturnFalse; } break; //--- case defNeuronCSCMOCL: cscm = new CNeuronCSCMOCL(); if(!cscm) ReturnFalse; if(!cscm.Init(outputs, 0, opencl, desc.windows, desc.window, desc.count, (bool)desc.step, desc.optimization, desc.batch)) { delete cscm; DeleteObj(temp); ReturnFalse; } if(!temp.Add(cscm)) { DeleteObj(temp); delete cscm; ReturnFalse; } break; //--- case defNeuronSPyrAttentionMLKV: spyr = new CNeuronSPyrAttentionOCL(); if(!spyr) ReturnFalse; if(!spyr.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete spyr; ReturnFalse; } if(!temp.Add(spyr)) { DeleteObj(temp); delete spyr; ReturnFalse; } break; //--- case defNeuronPLROCL: plr = new CNeuronPLROCL(); if(!plr) ReturnFalse; if(!plr.Init(outputs, 0, opencl, desc.window, desc.count, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete plr; ReturnFalse; } if(!temp.Add(plr)) { DeleteObj(temp); delete plr; ReturnFalse; } break; //--- case defNeuronTPM: tpm = new CNeuronTPM(); if(!tpm) ReturnFalse; if(!tpm.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete tpm; ReturnFalse; } if(!temp.Add(tpm)) { DeleteObj(temp); delete tpm; ReturnFalse; } break; //--- case defNeuronTPMEncoder: tpme = new CNeuronTPMEncoder(); if(!tpme) ReturnFalse; if(!tpme.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete tpme; ReturnFalse; } if(!temp.Add(tpme)) { DeleteObj(temp); delete tpme; ReturnFalse; } break; //--- case defNeuronSTNNEncoder: stnne = new CNeuronSTNNEncoder(); if(!stnne) ReturnFalse; if(!stnne.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete stnne; ReturnFalse; } if(!temp.Add(stnne)) { DeleteObj(temp); delete stnne; ReturnFalse; } break; //--- case defNeuronSTNNDecoder: stnnd = new CNeuronSTNNDecoder(); if(!stnnd) ReturnFalse; if(!stnnd.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.heads[0], desc.windows[1], desc.heads[1], desc.units[0], desc.units[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete stnnd; ReturnFalse; } if(!temp.Add(stnnd)) { DeleteObj(temp); delete stnnd; ReturnFalse; } break; case defNeuronSparseTSF: sptsf = new CNeuronSparseTSF(); if(!sptsf) ReturnFalse; if(!sptsf.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); delete sptsf; ReturnFalse; } if(!temp.Add(sptsf)) { DeleteObj(temp); delete sptsf; ReturnFalse; } break; case defNeuronTEMPOOCL: tempo = new CNeuronTEMPOOCL(); if(!tempo) ReturnFalse; if(!tempo.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete tempo; ReturnFalse; } if(!temp.Add(tempo)) { DeleteObj(temp); delete tempo; ReturnFalse; } break; case defNeuronInjectTST: inject = new CNeuronInjectTST(); if(!inject) ReturnFalse; if(!inject.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.step, desc.heads[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete inject; ReturnFalse; } if(!temp.Add(inject)) { DeleteObj(temp); delete inject; ReturnFalse; } break; case defNeuronMambaOCL: mamba = new CNeuronMambaOCL(); if(!mamba) ReturnFalse; if(!mamba.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete mamba; ReturnFalse; } if(!temp.Add(mamba)) { DeleteObj(temp); delete mamba; ReturnFalse; } break; case defNeuronTrajLLMOCL: traj = new CNeuronTrajLLMOCL(); if(!traj) ReturnFalse; if(!traj.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete traj; ReturnFalse; } if(!temp.Add(traj)) { DeleteObj(temp); delete traj; ReturnFalse; } break; //--- case defNeuronUniTrajOCL: utraj = new CNeuronUniTraj(); if(!utraj) ReturnFalse; if(!utraj.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.layers, desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); delete utraj; ReturnFalse; } if(!temp.Add(utraj)) { DeleteObj(temp); delete utraj; ReturnFalse; } break; //--- case defNeuronHiVTOCL: hivt = new CNeuronHiVTOCL(); if(!hivt) ReturnFalse; if(!hivt.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.step, desc.count, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete hivt; ReturnFalse; } if(!temp.Add(hivt)) { DeleteObj(temp); delete hivt; ReturnFalse; } break; //--- case defNeuronPointNetOCL: pn = new CNeuronPointNetOCL(); if(!pn) ReturnFalse; if(!pn.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete pn; ReturnFalse; } if(!temp.Add(pn)) { DeleteObj(temp); delete pn; ReturnFalse; } break; //--- case defNeuronPointNet2OCL: pn = new CNeuronPointNet2OCL(); if(!pn) ReturnFalse; if(!pn.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete pn; ReturnFalse; } if(!temp.Add(pn)) { DeleteObj(temp); delete pn; ReturnFalse; } break; //--- case defNeuronPointFormer: pn = new CNeuronPointFormer(); if(!pn) ReturnFalse; if(!pn.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete pn; ReturnFalse; } if(!temp.Add(pn)) { DeleteObj(temp); delete pn; ReturnFalse; } break; case defNeuronHyperDet: pn = new CNeuronHyperDet(); if(!pn) ReturnFalse; if(!pn.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, (bool)desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete pn; ReturnFalse; } if(!temp.Add(pn)) { DeleteObj(temp); delete pn; ReturnFalse; } break; case defNeuronSceneSpecific: ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.units[0], desc.units[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete ss; ReturnFalse; } if(!temp.Add(ss)) { DeleteObj(temp); delete ss; ReturnFalse; } break; case defNeuronSEFormer: sef = new CNeuronSEFormer(); if(!sef) ReturnFalse; if(!sef.Init(outputs, 0, opencl, desc.windows[0], desc.units[0], desc.window_out, (bool)desc.step, desc.units[1], desc.windows[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete sef; ReturnFalse; } if(!temp.Add(sef)) { DeleteObj(temp); delete sef; ReturnFalse; } break; //--- case defNeuronSPFormer: spf = new CNeuronSPFormer(); if(!spf) ReturnFalse; if(!spf.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.heads[0], desc.windows[1], desc.units[1], desc.heads[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete spf; ReturnFalse; } if(!temp.Add(spf)) { DeleteObj(temp); delete spf; ReturnFalse; } break; //--- case defNeuronMAFT: maft = new CNeuronMAFT(); if(!maft) ReturnFalse; if(!maft.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.heads[0], desc.windows[1], desc.units[1], desc.heads[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete maft; ReturnFalse; } if(!temp.Add(maft)) { DeleteObj(temp); delete maft; ReturnFalse; } break; //--- case defNeuronGRES: gres = new CNeuronGRES(); if(!gres) ReturnFalse; if(!gres.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.heads[0], desc.windows[1], desc.units[1], desc.heads[1], desc.windows[2], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete gres; ReturnFalse; } if(!temp.Add(gres)) { DeleteObj(temp); delete gres; ReturnFalse; } break; //--- case defNeuronRefMask: refm = new CNeuronRefMask(); if(!refm) ReturnFalse; if(!refm.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.step, desc.windows[1], desc.units[1], desc.units[2], desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete refm; ReturnFalse; } if(!temp.Add(refm)) { DeleteObj(temp); delete refm; ReturnFalse; } break; //--- case defNeuronRelativeSelfAttention: rat = new CNeuronRelativeSelfAttention(); if(!rat) ReturnFalse; if(!rat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete rat; ReturnFalse; } if(!temp.Add(rat)) { DeleteObj(temp); delete rat; ReturnFalse; } break; //--- case defNeuronRMAT: rmat = new CNeuronRMAT(); if(!rmat) ReturnFalse; if(!rmat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete rmat; ReturnFalse; } if(!temp.Add(rmat)) { DeleteObj(temp); delete rmat; ReturnFalse; } break; //--- case defNeuronMHAttentionPooling: atpool = new CNeuronMHAttentionPooling(); if(!atpool) ReturnFalse; if(!atpool.Init(outputs, 0, opencl, desc.window, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete atpool; ReturnFalse; } if(!temp.Add(atpool)) { DeleteObj(temp); delete atpool; ReturnFalse; } break; //--- case defNeuronMotifs: motif = new CNeuronMotifs(); if(!motif) ReturnFalse; if(!motif.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete motif; ReturnFalse; } if(!temp.Add(motif)) { DeleteObj(temp); delete motif; ReturnFalse; } break; //--- case defNeuronMultiScaleAttention: msat = new CNeuronMultiScaleAttention(); if(!msat) ReturnFalse; if(!msat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete msat; ReturnFalse; } if(!temp.Add(msat)) { DeleteObj(temp); delete msat; ReturnFalse; } break; //--- case defNeuronMolformer: molf = new CNeuronMolformer(); if(!molf) ReturnFalse; if(!molf.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.heads[0], desc.layers, desc.windows[1], desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete molf; ReturnFalse; } if(!temp.Add(molf)) { DeleteObj(temp); delete molf; ReturnFalse; } break; //--- case defNeuronAMCT: amct = new CNeuronAMCT(); if(!amct) ReturnFalse; if(!amct.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.units[1], desc.units[0], desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete amct; ReturnFalse; } if(!temp.Add(amct)) { DeleteObj(temp); delete amct; ReturnFalse; } break; //--- case defNeuronNAFS: nafs = new CNeuronNAFS(); if(!nafs) ReturnFalse; if(!nafs.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete nafs; ReturnFalse; } if(!temp.Add(nafs)) { DeleteObj(temp); delete nafs; ReturnFalse; } break; //--- case defNeuronDiffusion: usa = new CNeuronDiffusion(); if(!usa) { DeleteObj(temp); ReturnFalse; } if(!usa.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.count, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete usa; ReturnFalse; } if(!temp.Add(usa)) { DeleteObj(temp); delete usa; ReturnFalse; } //--- case defNeuronHyperProjection: lorentz = new CNeuronHyperProjection(); if(!lorentz) { DeleteObj(temp); ReturnFalse; } if(!lorentz.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete lorentz; ReturnFalse; } if(!temp.Add(lorentz)) { DeleteObj(temp); delete lorentz; ReturnFalse; } //--- case defNeuronHyperboloids: logmap = new CNeuronHyperboloids(); if(!logmap) { DeleteObj(temp); ReturnFalse; } if(!logmap.Init(outputs, 0, opencl, desc.window, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete logmap; ReturnFalse; } if(!temp.Add(logmap)) { DeleteObj(temp); delete logmap; ReturnFalse; } //--- case defNeuronHypDiff: hypdiff = new CNeuronHypDiff(); if(!hypdiff) { DeleteObj(temp); ReturnFalse; } if(!hypdiff.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.heads[0], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete hypdiff; ReturnFalse; } if(!temp.Add(hypdiff)) { DeleteObj(temp); delete hypdiff; ReturnFalse; } break; //--- case defNeuronPSBlock: ps_block = new CNeuronPSBlock(); if(!ps_block) { DeleteObj(temp); ReturnFalse; } if(!ps_block.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); delete ps_block; ReturnFalse; } if(!temp.Add(ps_block)) { DeleteObj(temp); delete ps_block; ReturnFalse; } break; //--- case defNeuronPSformer: psformer = new CNeuronPSformer(); if(!psformer) { DeleteObj(temp); ReturnFalse; } if(!psformer.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); delete psformer; ReturnFalse; } if(!temp.Add(psformer)) { DeleteObj(temp); delete psformer; ReturnFalse; } break; //--- case defNeuronMASA: masa = new CNeuronMASA(); if(!masa) { DeleteObj(temp); ReturnFalse; } if(!masa.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.heads[0], desc.units[0], desc.windows[1], desc.window, desc.probability, desc.units[1], desc.windows[2], desc.heads[1], desc.units[2], desc.step, (desc.step >= 0 ? ((CLayer*)layers.At(desc.step)).At(0) : NULL), desc.optimization, desc.batch)) { DeleteObj(temp); delete masa; ReturnFalse; } if(!temp.Add(masa)) { DeleteObj(temp); delete masa; ReturnFalse; } masa.SetActivationFunction(desc.activation); break; //--- case defNeuronMarketObserver: mo = new CNeuronMarketObserver(); if(!mo) { DeleteObj(temp); ReturnFalse; } if(!mo.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete mo; ReturnFalse; } if(!temp.Add(mo)) { DeleteObj(temp); delete mo; ReturnFalse; } mo.SetActivationFunction(desc.activation); break; //--- case defNeuronRLAgent: rl_agent = new CNeuronRLAgent(); if(!rl_agent) { DeleteObj(temp); ReturnFalse; } if(!rl_agent.Init(outputs, 0, opencl, desc.windows[0], desc.count, desc.window_out, desc.probability, desc.layers, desc.windows[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete rl_agent; ReturnFalse; } if(!temp.Add(rl_agent)) { DeleteObj(temp); delete rl_agent; ReturnFalse; } break; //--- case defNeuronControlAgent: contr_ag = new CNeuronControlAgent(); if(!contr_ag) { DeleteObj(temp); ReturnFalse; } if(!contr_ag.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.step, desc.windows[1], desc.units[1], desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete contr_ag; ReturnFalse; } if(!temp.Add(contr_ag)) { DeleteObj(temp); delete contr_ag; ReturnFalse; } break; //--- case defNeuronMASAAT: masaat = new CNeuronMASAAT(); if(!masaat) { DeleteObj(temp); ReturnFalse; } if(!masaat.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.step, desc.count, desc.layers, desc.radius, desc.windows[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete masaat; ReturnFalse; } if(!temp.Add(masaat)) { DeleteObj(temp); delete masaat; ReturnFalse; } break; //--- case defNeuronMHConvOCL: mhconv = new CNeuronMHConvOCL(); if(!mhconv) { DeleteObj(temp); ReturnFalse; } if(!mhconv.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.heads[0], desc.heads[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete mhconv; ReturnFalse; } if(!temp.Add(mhconv)) { DeleteObj(temp); delete mhconv; ReturnFalse; } break; //--- case defNeuronMHFeedForward: mhff = new CNeuronMHFeedForward(); if(!mhff) { DeleteObj(temp); ReturnFalse; } if(!mhff.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.heads[0], desc.heads[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete mhff; ReturnFalse; } if(!temp.Add(mhff)) { DeleteObj(temp); delete mhff; ReturnFalse; } break; //--- case defNeuronDMHAttention: dmhat = new CNeuronDMHAttention(); if(!dmhat) { DeleteObj(temp); ReturnFalse; } if(!dmhat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete dmhat; ReturnFalse; } if(!temp.Add(dmhat)) { DeleteObj(temp); delete dmhat; ReturnFalse; } break; //--- case defNeuronCrossDMHAttention: crdmhat = new CNeuronCrossDMHAttention(); if(!crdmhat) { DeleteObj(temp); ReturnFalse; } if(!crdmhat.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.windows[1], desc.units[1], desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete crdmhat; ReturnFalse; } if(!temp.Add(crdmhat)) { DeleteObj(temp); delete crdmhat; ReturnFalse; } break; //--- case defNeuronMultitaskStockformer: mtstockf = new CNeuronMultitaskStockformer(); if(!mtstockf) { DeleteObj(temp); ReturnFalse; } if(!mtstockf.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[2], desc.windows[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete mtstockf; ReturnFalse; } if(!temp.Add(mtstockf)) { DeleteObj(temp); delete mtstockf; ReturnFalse; } break; //--- case defNeuronFinMem: fin_mem = new CNeuronFinMem(); if(!fin_mem) { DeleteObj(temp); ReturnFalse; } if(!fin_mem.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete fin_mem; ReturnFalse; } if(!temp.Add(fin_mem)) { DeleteObj(temp); delete fin_mem; ReturnFalse; } break; //--- case defNeuronFinAgent: fin_agent = new CNeuronFinAgent(); if(!fin_agent) { DeleteObj(temp); ReturnFalse; } if(!fin_agent.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete fin_agent; ReturnFalse; } if(!temp.Add(fin_agent)) { DeleteObj(temp); delete fin_agent; ReturnFalse; } break; //--- case defNeuronFinConManager: fin_con = new CNeuronFinConManager(); if(!fin_con) { DeleteObj(temp); ReturnFalse; } if(!fin_con.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete fin_con; ReturnFalse; } if(!temp.Add(fin_con)) { DeleteObj(temp); delete fin_con; ReturnFalse; } break; //--- case defNeuronMacroHFT: mhft = new CNeuronMacroHFT(); if(!mhft) { DeleteObj(temp); ReturnFalse; } if(!mhft.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete mhft; ReturnFalse; } if(!temp.Add(mhft)) { DeleteObj(temp); delete mhft; ReturnFalse; } break; case defNeuronMacroHFTHyperAgent: mhft_hyp = new CNeuronMacroHFTHyperAgent(); if(!mhft_hyp) { DeleteObj(temp); ReturnFalse; } if(!mhft_hyp.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete mhft_hyp; ReturnFalse; } if(!temp.Add(mhft_hyp)) { DeleteObj(temp); delete mhft_hyp; ReturnFalse; } break; case defNeuronMacroHFTvsRiskManager: mhft_risk = new CNeuronMacroHFTvsRiskManager(); if(!mhft_risk) { DeleteObj(temp); ReturnFalse; } if(!mhft_risk.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete mhft_risk; ReturnFalse; } if(!temp.Add(mhft_risk)) { DeleteObj(temp); delete mhft_risk; ReturnFalse; } break; case defNeuronMultiScaleRelativeSelfAttention: rat = new CNeuronMultiScaleRelativeSelfAttention(); if(!rat) ReturnFalse; if(!rat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete rat; ReturnFalse; } if(!temp.Add(rat)) { DeleteObj(temp); delete rat; ReturnFalse; } break; //--- case defNeuronHidformer: hidf = new CNeuronHidformer(); if(!hidf) { DeleteObj(temp); ReturnFalse; } if(!hidf.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete hidf; ReturnFalse; } if(!temp.Add(hidf)) { DeleteObj(temp); delete hidf; ReturnFalse; } break; //--- case defNeuronResNeXtBlock: resnext = new CNeuronResNeXtBlock(); if(!resnext) { DeleteObj(temp); ReturnFalse; } if(!resnext.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.window, desc.step, desc.units[0], desc.units[1], desc.units[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete resnext; ReturnFalse; } if(!temp.Add(resnext)) { DeleteObj(temp); delete resnext; ReturnFalse; } break; //--- case defNeuronChimera: chimera = new CNeuronChimera(); if(!chimera) { DeleteObj(temp); ReturnFalse; } if(!chimera.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); delete chimera; ReturnFalse; } if(!temp.Add(chimera)) { DeleteObj(temp); delete chimera; ReturnFalse; } break; //--- case defNeuronMoT: mot = new CNeuronMoT(); if(!mot) ReturnFalse; if(!mot.Init(outputs, 0, opencl, desc.window, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete mot; ReturnFalse; } if(!temp.Add(mot)) { DeleteObj(temp); delete mot; ReturnFalse; } break; //--- case defNeuronHypridDecoder: hypridDecod = new CNeuronHypridDecoder(); if(!hypridDecod) { DeleteObj(temp); ReturnFalse; } if(!hypridDecod.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.count, desc.step, desc.layers, desc.windows[1], desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete hypridDecod; ReturnFalse; } if(!temp.Add(hypridDecod)) { DeleteObj(temp); delete hypridDecod; ReturnFalse; } break; //--- case defNeuronAttraos: attraos = new CNeuronAttraos(); if(!attraos) { DeleteObj(temp); ReturnFalse; } if(!attraos.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete attraos; ReturnFalse; } if(!temp.Add(attraos)) { DeleteObj(temp); delete attraos; ReturnFalse; } break; //--- case defNeuronDUET: duet = new CNeuronDUET(); if(!duet) { DeleteObj(temp); ReturnFalse; } if(!duet.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.units[0], desc.step, desc.units[1], desc.units[2], desc.units[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete duet; ReturnFalse; } if(!temp.Add(duet)) { DeleteObj(temp); delete duet; ReturnFalse; } break; //--- case defNeuronMultiWindowsConvOCL: mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv) { DeleteObj(temp); ReturnFalse; } if(!mwconv.Init(outputs, 0, opencl, desc.windows, desc.window_out, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete mwconv; ReturnFalse; } if(!temp.Add(mwconv)) { DeleteObj(temp); delete mwconv; ReturnFalse; } break; //--- case defNeuronAdaBN: adabn = new CNeuronAdaBN(); if(!adabn) { DeleteObj(temp); ReturnFalse; } if(!adabn.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.windows, desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete adabn; ReturnFalse; } if(!temp.Add(adabn)) { DeleteObj(temp); delete adabn; ReturnFalse; } break; //--- case defNeuronTransposeRCDOCL: trcd = new CNeuronTransposeRCDOCL(); if(!trcd) { DeleteObj(temp); ReturnFalse; } if(!trcd.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete trcd; ReturnFalse; } if(!temp.Add(trcd)) { DeleteObj(temp); delete trcd; ReturnFalse; } break; //--- case defNeuronTransposeVRCOCL: tvrc = new CNeuronTransposeVRCOCL(); if(!tvrc) { DeleteObj(temp); ReturnFalse; } if(!tvrc.Init(outputs, 0, opencl, desc.step, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete tvrc; ReturnFalse; } if(!temp.Add(tvrc)) { DeleteObj(temp); delete tvrc; ReturnFalse; } break; //--- case defNeuronCATCH: catch = new CNeuronCATCH(); if(!catch) { DeleteObj(temp); ReturnFalse; } if(!catch.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.step, desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete catch; ReturnFalse; } if(!temp.Add(catch)) { DeleteObj(temp); delete catch; ReturnFalse; } break; //--- case defNeuronSkillsEncoder: skills = new CNeuronSkillsEncoder(); if(!skills) { DeleteObj(temp); ReturnFalse; } if(!skills.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.window, desc.step, desc.window_out, desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); delete skills; ReturnFalse; } if(!temp.Add(skills)) { DeleteObj(temp); delete skills; ReturnFalse; } break; //--- case defNeuronHiSSDLowLevelControler: hissd = new CNeuronHiSSDLowLevelControler(); if(!hissd) { DeleteObj(temp); ReturnFalse; } if(!hissd.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.windows[2], desc.windows[3], desc.window, desc.step, desc.window_out, desc.windows[4], desc.optimization, desc.batch)) { DeleteObj(temp); delete hissd; ReturnFalse; } hissd.SetActivationFunction(desc.activation); if(!temp.Add(hissd)) { DeleteObj(temp); delete hissd; ReturnFalse; } break; //--- case defNeuronLinerAttention: lineatt = new CNeuronLinerAttention(); if(!lineatt) { DeleteObj(temp); ReturnFalse; } if(!lineatt.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete lineatt; ReturnFalse; } lineatt.SetActivationFunction(None); if(!temp.Add(lineatt)) { DeleteObj(temp); delete lineatt; ReturnFalse; } break; //--- case defNeuronCGLSTMOCL: cglstm = new CNeuronCGLSTMOCL(); if(!cglstm) { DeleteObj(temp); ReturnFalse; } if(!cglstm.Init(outputs, 0, opencl, desc.count, desc.window, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete cglstm; ReturnFalse; } cglstm.SetActivationFunction(None); if(!temp.Add(cglstm)) { DeleteObj(temp); delete cglstm; ReturnFalse; } break; //--- case defNeuronMHProbAttention: probatt = new CNeuronMHProbAttention(); if(!probatt) { DeleteObj(temp); ReturnFalse; } if(!probatt.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.step, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); delete probatt; ReturnFalse; } if(!temp.Add(probatt)) { DeleteObj(temp); delete probatt; ReturnFalse; } break; //--- case defNeuronTSPositionEncoder: tspe = new CNeuronTSPositionEncoder(); if(!tspe) { DeleteObj(temp); ReturnFalse; } if(!tspe.Init(outputs, 0, opencl, desc.window, desc.count, desc.windows, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); delete tspe; ReturnFalse; } if(!temp.Add(tspe)) { DeleteObj(temp); delete tspe; ReturnFalse; } break; //--- case defMamba4CastEmbeding: m4cemb = new CMamba4CastEmbeding(); if(!m4cemb) { DeleteObj(temp); ReturnFalse; } if(!m4cemb.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); delete m4cemb; ReturnFalse; } if(!temp.Add(m4cemb)) { DeleteObj(temp); delete m4cemb; ReturnFalse; } break; case defNeuronMultiWindowsConvWPadOCL: mwconvpad = new CNeuronMultiWindowsConvWPadOCL(); if(!mwconvpad) { DeleteObj(temp); ReturnFalse; } if(!mwconvpad.Init(outputs, 0, opencl, desc.windows, desc.step, desc.window_out, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete mwconvpad; ReturnFalse; } if(!temp.Add(mwconvpad)) { DeleteObj(temp); delete mwconvpad; ReturnFalse; } break; case defNeuronConcatDiff: condiff = new CNeuronConcatDiff(); if(!condiff) { DeleteObj(temp); ReturnFalse; } if(!condiff.Init(outputs, 0, opencl, desc.count, desc.step, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete condiff; ReturnFalse; } if(!temp.Add(condiff)) { DeleteObj(temp); delete condiff; ReturnFalse; } break; case defNeuronMantisPatching: mantpatch = new CNeuronMantisPatching(); if(!mantpatch) { DeleteObj(temp); ReturnFalse; } if(!mantpatch.Init(outputs, 0, opencl, desc.count, desc.step, desc.layers, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); delete mantpatch; ReturnFalse; } if(!temp.Add(mantpatch)) { DeleteObj(temp); delete mantpatch; ReturnFalse; } break; //--- case defNeuronMantisAttentionUnit: mantatt = new CNeuronMantisAttentionUnit(); if(!mantatt) { DeleteObj(temp); ReturnFalse; } if(!mantatt.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.window_out, desc.step, desc.count, desc.layers, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete mantatt; ReturnFalse; } if(!temp.Add(mantatt)) { DeleteObj(temp); delete mantatt; ReturnFalse; } break; //--- case defNeuronTimeFoundPatching: tf_patch = new CNeuronTimeFoundPatching(); if(!tf_patch) { DeleteObj(temp); ReturnFalse; } if(!tf_patch.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete tf_patch; ReturnFalse; } if(!temp.Add(tf_patch)) { DeleteObj(temp); delete tf_patch; ReturnFalse; } break; //--- case defNeuronTimeFoundTransformerUnit: tf_tu = new CNeuronTimeFoundTransformerUnit(); if(!tf_tu) { DeleteObj(temp); ReturnFalse; } if(!tf_tu.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.heads[0], desc.heads[1], desc.count, desc.layers, desc.step, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete tf_tu; ReturnFalse; } if(!temp.Add(tf_tu)) { DeleteObj(temp); delete tf_tu; ReturnFalse; } break; //--- case defNeuronSwiGLUOCL: swiglu = new CNeuronSwiGLUOCL(); if(!swiglu) { DeleteObj(temp); ReturnFalse; } if(!swiglu.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete swiglu; ReturnFalse; } if(!temp.Add(swiglu)) { DeleteObj(temp); delete swiglu; ReturnFalse; } break; //--- case defNeuronTimeMoESparseExperts: timemoesp = new CNeuronTimeMoESparseExperts(); if(!timemoesp) { DeleteObj(temp); ReturnFalse; } if(!timemoesp.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); delete timemoesp; ReturnFalse; } if(!temp.Add(timemoesp)) { DeleteObj(temp); delete timemoesp; ReturnFalse; } break; //--- case defNeuronTimeMoEAttention: timemoeatt = new CNeuronTimeMoEAttention(); if(!timemoeatt) { DeleteObj(temp); ReturnFalse; } if(!timemoeatt.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.units[0], desc.windows[1], desc.units[1], desc.step, desc.layers, desc.units[2], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete timemoeatt; ReturnFalse; } if(!temp.Add(timemoeatt)) { DeleteObj(temp); delete timemoeatt; ReturnFalse; } break; //--- case defNeuronAdaptConv: ac = new CNeuronAdaptConv(); if(!ac) { DeleteObj(temp); ReturnFalse; } if(!ac.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete ac; ReturnFalse; } if(!temp.Add(ac)) { DeleteObj(temp); delete ac; ReturnFalse; } break; case defNeuronRoPE: rope = new CNeuronRoPE(); if(!rope) { DeleteObj(temp); ReturnFalse; } if(!rope.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete rope; ReturnFalse; } if(!temp.Add(rope)) { DeleteObj(temp); delete rope; ReturnFalse; } break; case defNeuronK2VAEEncoder: k2vae = new CNeuronK2VAEEncoder(); if(!k2vae) { DeleteObj(temp); ReturnFalse; } if(!k2vae.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.step, desc.layers, desc.units[1], desc.units[2], desc.windows[2], desc.units[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete k2vae; ReturnFalse; } if(!temp.Add(k2vae)) { DeleteObj(temp); delete k2vae; ReturnFalse; } break; case defNeuronInterpolationAttention: ia = new CNeuronInterpolationAttention(); if(!ia) { DeleteObj(temp); ReturnFalse; } if(!ia.Init(outputs, 0, opencl, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete ia; ReturnFalse; } if(!temp.Add(ia)) { DeleteObj(temp); delete ia; ReturnFalse; } break; case defNeuronGinARCell: ginar_cell = new CNeuronGinARCell(); if(!ginar_cell) { DeleteObj(temp); ReturnFalse; } if(!ginar_cell.Init(outputs, 0, opencl, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete ginar_cell; ReturnFalse; } if(!temp.Add(ginar_cell)) { DeleteObj(temp); delete ginar_cell; ReturnFalse; } break; case defNeuronGinAR: ginar = new CNeuronGinAR(); if(!ginar) { DeleteObj(temp); ReturnFalse; } if(!ginar.Init(outputs, 0, opencl, desc.count, desc.window, desc.optimization, desc.batch)) { DeleteObj(temp); delete ginar; ReturnFalse; } if(!temp.Add(ginar)) { DeleteObj(temp); delete ginar; ReturnFalse; } break; case defNeuronPeriodNorm: per_norm = new CNeuronPeriodNorm(); if(!per_norm) { DeleteObj(temp); ReturnFalse; } if(!per_norm.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete per_norm; ReturnFalse; } if(!temp.Add(per_norm)) { DeleteObj(temp); delete per_norm; ReturnFalse; } break; case defNeuronAdaptSpatialNorm: adapt_norm = new CNeuronAdaptSpatialNorm(); if(!adapt_norm) { DeleteObj(temp); ReturnFalse; } if(!adapt_norm.Init(outputs, 0, opencl, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete adapt_norm; ReturnFalse; } if(!temp.Add(adapt_norm)) { DeleteObj(temp); delete adapt_norm; ReturnFalse; } break; case defNeuronSCNNEncoder: scnn_enc = new CNeuronSCNNEncoder(); if(!scnn_enc) { DeleteObj(temp); ReturnFalse; } if(!scnn_enc.Init(outputs, 0, opencl, desc.windows[0], desc.variables, desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete scnn_enc; ReturnFalse; } if(!temp.Add(scnn_enc)) { DeleteObj(temp); delete scnn_enc; ReturnFalse; } break; case defNeuronSCNN: scnn = new CNeuronSCNN(); if(!scnn) { DeleteObj(temp); ReturnFalse; } if(!scnn.Init(outputs, 0, opencl, desc.windows[0], desc.variables, desc.windows[1], desc.windows[2], desc.windows[3], desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete scnn; ReturnFalse; } if(!temp.Add(scnn)) { DeleteObj(temp); delete scnn; ReturnFalse; } break; case defNeuronAttentNorm: atn = new CNeuronAttentNorm(); if(!atn) { DeleteObj(temp); ReturnFalse; } if(!atn.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete atn; ReturnFalse; } if(!temp.Add(atn)) { DeleteObj(temp); delete atn; ReturnFalse; } break; case defNeuronSAttentNorm: satn = new CNeuronSAttentNorm(); if(!satn) { DeleteObj(temp); ReturnFalse; } if(!satn.Init(outputs, 0, opencl, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); delete satn; ReturnFalse; } if(!temp.Add(satn)) { DeleteObj(temp); delete satn; ReturnFalse; } break; case defNeuronPolynomialRegression: pol_regr = new CNeuronPolynomialRegression(); if(!pol_regr) { DeleteObj(temp); ReturnFalse; } if(!pol_regr.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); delete pol_regr; ReturnFalse; } if(!temp.Add(pol_regr)) { DeleteObj(temp); delete pol_regr; ReturnFalse; } break; case defNeuronSSCNNEncoder: sscnn_enc = new CNeuronSSCNNEncoder(); if(!sscnn_enc) { DeleteObj(temp); ReturnFalse; } if(!sscnn_enc.Init(outputs, 0, opencl, desc.windows[0], desc.variables, desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); delete sscnn_enc; ReturnFalse; } if(!temp.Add(sscnn_enc)) { DeleteObj(temp); delete sscnn_enc; ReturnFalse; } break; case defNeuronSSCNN: sscnn = new CNeuronSSCNN(); if(!sscnn) { DeleteObj(temp); ReturnFalse; } if(!sscnn.Init(outputs, 0, opencl, desc.windows[0], desc.variables, desc.windows[1], desc.windows[2], desc.windows[3], desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); delete sscnn; ReturnFalse; } if(!temp.Add(sscnn)) { DeleteObj(temp); delete sscnn; ReturnFalse; } break; case defNeuronTQMHA: tqmha = new CNeuronTQMHA(); if(!tqmha) { DeleteObj(temp); ReturnFalse; } if(!tqmha.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.step, desc.count, desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(tqmha); ReturnFalse; } if(!temp.Add(tqmha)) { DeleteObj(temp); DeleteObj(tqmha); ReturnFalse; } break; case defNeuronHimNetEncoder: hn_enc = new CNeuronHimNetEncoder(); if(!hn_enc) { DeleteObj(temp); ReturnFalse; } if(!hn_enc.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.step, desc.layers, desc.windows[2], desc.windows[3], desc.windows[4], desc.windows[5], desc.windows[6], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(hn_enc); ReturnFalse; } if(!temp.Add(hn_enc)) { DeleteObj(temp); DeleteObj(hn_enc); ReturnFalse; } break; case defNeuronHimNetDecoder: hn_dec = new CNeuronHimNetDecoder(); if(!hn_dec) { DeleteObj(temp); ReturnFalse; } if(!hn_dec.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.step, desc.layers, desc.windows[2], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(hn_dec); ReturnFalse; } if(!temp.Add(hn_dec)) { DeleteObj(temp); DeleteObj(hn_dec); ReturnFalse; } break; case defNeuronSAGDFN: sagdfn = new CNeuronSAGDFN(); if(!sagdfn) { DeleteObj(temp); ReturnFalse; } if(!sagdfn.Init(outputs, 0, opencl, desc.count, desc.window, desc.windows[0], desc.units[0], desc.windows[1], desc.step, desc.probability, desc.units[1], desc.windows[2], desc.units[2], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sagdfn); ReturnFalse; } if(!temp.Add(sagdfn)) { DeleteObj(temp); DeleteObj(sagdfn); ReturnFalse; } break; case defNeuronSpatialEmbedding: semb = new CNeuronSpatialEmbedding(); if(!semb) { DeleteObj(temp); ReturnFalse; } if(!semb.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(semb); ReturnFalse; } if(!temp.Add(semb)) { DeleteObj(temp); DeleteObj(semb); ReturnFalse; } break; case defNeuronTempEmbedding: temb = new CNeuronTempEmbedding(); if(!temb) { DeleteObj(temp); ReturnFalse; } if(!temb.Init(outputs, 0, opencl, desc.count, desc.window, desc.windows[0], desc.units[0], desc.windows[1], desc.windows[2], desc.units[1], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(temb); ReturnFalse; } if(!temp.Add(temb)) { DeleteObj(temp); DeleteObj(temb); ReturnFalse; } break; case defNeuronGlobalLocalAttention: glatt = new CNeuronGlobalLocalAttention(); if(!glatt) { DeleteObj(temp); ReturnFalse; } if(!glatt.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.step, desc.variables, desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(glatt); ReturnFalse; } if(!temp.Add(glatt)) { DeleteObj(temp); DeleteObj(glatt); ReturnFalse; } break; case defNeuronExtralonger: exlon = new CNeuronExtralonger(); if(!exlon) { DeleteObj(temp); ReturnFalse; } if(!exlon.Init(outputs, 0, opencl, desc.units[0], desc.units[1], desc.window, desc.window_out, desc.windows[0], desc.units[2], desc.windows[1], desc.units[3], desc.windows[2], desc.layers, desc.step, desc.windows[3], desc.units[4], desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(exlon); ReturnFalse; } if(!temp.Add(exlon)) { DeleteObj(temp); DeleteObj(exlon); ReturnFalse; } break; case defNeuronExtralongerGraph: exlongr = new CNeuronExtralongerGraph(); if(!exlongr) { DeleteObj(temp); ReturnFalse; } if(!exlongr.Init(outputs, 0, opencl, desc.units[0], desc.units[1], desc.window, desc.window_out, desc.windows[0], desc.units[2], desc.windows[1], desc.units[3], desc.windows[2], desc.layers, desc.step, desc.units[4], desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(exlongr); ReturnFalse; } if(!temp.Add(exlongr)) { DeleteObj(temp); DeleteObj(exlongr); ReturnFalse; } break; //--- case defNeuronSpikeConv: spike = new CNeuronSpikeConv(); if(!spike) { DeleteObj(temb); ReturnFalse; } if(!spike.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spike); ReturnFalse; } if(!temp.Add(spike)) { DeleteObj(temp); DeleteObj(spike); ReturnFalse; } break; //--- case defNeuronGateLineAttention: glineatt = new CNeuronGateLineAttention(); if(!glineatt) { DeleteObj(temp); ReturnFalse; } if(!glineatt.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(glineatt); ReturnFalse; } if(!temp.Add(glineatt)) { DeleteObj(temp); DeleteObj(glineatt); ReturnFalse; } break; //--- case defNeuronSpikeSparseAttention: spikeatt = new CNeuronSpikeSparseAttention(); if(!spikeatt) { DeleteObj(temp); ReturnFalse; } if(!spikeatt.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.probability, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikeatt); ReturnFalse; } if(!temp.Add(spikeatt)) { DeleteObj(temp); DeleteObj(spikeatt); ReturnFalse; } break; //--- case defNeuronMoEConv: moecon = new CNeuronMoEConv(); if(!moecon) { DeleteObj(temp); ReturnFalse; } if(!moecon.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.probability, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(moecon); ReturnFalse; } if(!temp.Add(moecon)) { DeleteObj(temp); DeleteObj(moecon); ReturnFalse; } break; //--- case defNeuronSpikingBrain: spikebrain = new CNeuronSpikingBrain(); if(!spikebrain) { DeleteObj(temp); ReturnFalse; } if(!spikebrain.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.probability, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikebrain); ReturnFalse; } if(!temp.Add(spikebrain)) { DeleteObj(temp); DeleteObj(spikebrain); ReturnFalse; } break; case defNeuronSparseSoftMax: sparsoftmax = new CNeuronSparseSoftMax(); if(!sparsoftmax) { DeleteObj(temp); ReturnFalse; } if(!sparsoftmax.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sparsoftmax); ReturnFalse; } if(!temp.Add(sparsoftmax)) { DeleteObj(temp); DeleteObj(sparsoftmax); ReturnFalse; } break; case defNeuronSpikeActivation: spikeact = new CNeuronSpikeActivation(); if(!spikeact) { DeleteObj(temp); ReturnFalse; } if(!spikeact.Init(outputs, 0, opencl, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikeact); ReturnFalse; } if(!temp.Add(spikeact)) { DeleteObj(temp); DeleteObj(spikeact); ReturnFalse; } break; case defNeuronSpikeResNeXtBlock: spikeresblock = new CNeuronSpikeResNeXtBlock(); if(!spikeresblock) { DeleteObj(temp); ReturnFalse; } if(!spikeresblock.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.windows[2], desc.units[2], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikeresblock); ReturnFalse; } if(!temp.Add(spikeresblock)) { DeleteObj(temp); DeleteObj(spikeresblock); ReturnFalse; } break; case defNeuronSpikeResNeXtBottleneck: spikeresbottleneck = new CNeuronSpikeResNeXtBottleneck(); if(!spikeresbottleneck) { DeleteObj(temp); ReturnFalse; } if(!spikeresbottleneck.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.windows[2], desc.units[2], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikeresbottleneck); ReturnFalse; } if(!temp.Add(spikeresbottleneck)) { DeleteObj(temp); DeleteObj(spikeresbottleneck); ReturnFalse; } break; case defNeuronSpikeResNeXtResidual: spikeresresid = new CNeuronSpikeResNeXtResidual(); if(!spikeresresid) { DeleteObj(temp); ReturnFalse; } if(!spikeresresid.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spikeresresid); ReturnFalse; } if(!temp.Add(spikeresresid)) { DeleteObj(temp); DeleteObj(spikeresresid); ReturnFalse; } break; case defNeuronSSAM: ssam = new CNeuronSSAM(); if(!ssam) { DeleteObj(temp); ReturnFalse; } if(!ssam.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, (bool)desc.probability, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ssam); ReturnFalse; } if(!temp.Add(ssam)) { DeleteObj(temp); DeleteObj(ssam); ReturnFalse; } break; case defNeuronSSAMResNeXtBlock: ssamblock = new CNeuronSSAMResNeXtBlock(); if(!ssamblock) { DeleteObj(temp); ReturnFalse; } if(!ssamblock.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.windows[2], desc.units[2], desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ssamblock); ReturnFalse; } if(!temp.Add(ssamblock)) { DeleteObj(temp); DeleteObj(ssamblock); ReturnFalse; } break; case defNeuronSTFS: stfs = new CNeuronSTFS(); if(!stfs) { DeleteObj(temp); ReturnFalse; } if(!stfs.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(stfs); ReturnFalse; } stfs.SetActivationFunction(desc.activation); if(!temp.Add(stfs)) { DeleteObj(temp); DeleteObj(stfs); ReturnFalse; } break; case defNeuronSpikeConvGRU: sp_conv_gru = new CNeuronSpikeConvGRU(); if(!sp_conv_gru) { DeleteObj(temp); ReturnFalse; } if(!sp_conv_gru.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_conv_gru); ReturnFalse; } if(!temp.Add(sp_conv_gru)) { DeleteObj(temp); DeleteObj(sp_conv_gru); ReturnFalse; } break; case defNeuronAddToStack: stack = new CNeuronAddToStack(); if(!stack) { DeleteObj(temp); ReturnFalse; } if(!stack.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(stack); ReturnFalse; } if(!temp.Add(stack)) { DeleteObj(temp); DeleteObj(stack); ReturnFalse; } break; case defNeuronStackCorrelation: st_correl = new CNeuronStackCorrelation(); if(!st_correl) { DeleteObj(temp); ReturnFalse; } if(!st_correl.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(st_correl); ReturnFalse; } st_correl.SetActivationFunction(desc.activation); if(!temp.Add(st_correl)) { DeleteObj(temp); DeleteObj(st_correl); ReturnFalse; } break; case defNeuronSTEFlowNetEncoder: ste_encod = new CNeuronSTEFlowNetEncoder(); if(!ste_encod) { DeleteObj(temp); ReturnFalse; } if(!ste_encod.Init(outputs, 0, opencl, desc.windows, desc.units, desc.window, desc.variables, desc.step, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ste_encod); ReturnFalse; } if(!temp.Add(ste_encod)) { DeleteObj(temp); DeleteObj(ste_encod); ReturnFalse; } break; case defNeuronSTEFlowNet: ste = new CNeuronSTEFlowNet(); if(!ste) { DeleteObj(temp); ReturnFalse; } if(!ste.Init(outputs, 0, opencl, desc.windows, desc.units, desc.window, desc.variables, desc.step, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ste); ReturnFalse; } if(!temp.Add(ste)) { DeleteObj(temp); DeleteObj(ste); ReturnFalse; } break; case defNeuronSTEFlowNetDecoder: ste_decod = new CNeuronSTEFlowNetDecoder(); if(!ste_decod) { DeleteObj(temp); ReturnFalse; } if(!ste_decod.Init(outputs, 0, opencl, desc.windows, desc.units, desc.window, desc.variables, desc.step, desc.window_out, ((CLayer*)layers.At(desc.layers))[0], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ste_decod); ReturnFalse; } if(!temp.Add(ste_decod)) { DeleteObj(temp); DeleteObj(ste_decod); ReturnFalse; } break; case defNeuronSTEFlowNetResidualBlock: ste_resid = new CNeuronSTEFlowNetResidualBlock(); if(!ste_resid) { DeleteObj(temp); ReturnFalse; } if(!ste_resid.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ste_resid); ReturnFalse; } if(!temp.Add(ste_resid)) { DeleteObj(temp); DeleteObj(ste_resid); ReturnFalse; } break; case defNeuronMultiScaleStackCorrelation : mul_correl = new CNeuronMultiScaleStackCorrelation(); if(!mul_correl) { DeleteObj(temp); ReturnFalse; } if(!mul_correl.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(mul_correl); ReturnFalse; } if(!temp.Add(mul_correl)) { DeleteObj(temp); DeleteObj(mul_correl); ReturnFalse; } break; case defNeuronSpikeConvGRU2D: sp_conv_gru2D = new CNeuronSpikeConvGRU2D(); if(!sp_conv_gru2D) { DeleteObj(temp); ReturnFalse; } if(!sp_conv_gru2D.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_conv_gru2D); ReturnFalse; } if(!temp.Add(sp_conv_gru2D)) { DeleteObj(temp); DeleteObj(sp_conv_gru2D); ReturnFalse; } break; case defNeuronRAFT: raft = new CNeuronRAFT(); if(!raft) { DeleteObj(temp); ReturnFalse; } if(!raft.Init(outputs, 0, opencl, desc.units, desc.windows, desc.window, desc.step, desc.count, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(raft); ReturnFalse; } if(!temp.Add(raft)) { DeleteObj(temp); DeleteObj(raft); ReturnFalse; } break; case defNeuronMultScalStackCorrelBySegments: segm_corr = new CNeuronMultScalStackCorrelBySegments(); if(!segm_corr) { DeleteObj(temp); ReturnFalse; } if(!segm_corr.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.variables, desc.layers, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(segm_corr); ReturnFalse; } if(!temp.Add(segm_corr)) { DeleteObj(temp); DeleteObj(segm_corr); ReturnFalse; } break; case defNeuronTMAMFE: mfe = new CNeuronTMAMFE(); if(!mfe) { DeleteObj(temp); ReturnFalse; } if(!mfe.Init(outputs, 0, opencl, desc.count, desc.windows, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(mfe); ReturnFalse; } if(!temp.Add(mfe)) { DeleteObj(temp) DeleteObj(mfe) ReturnFalse; } break; case defNeuronTMAMPA: mpa = new CNeuronTMAMPA(); if(!mpa) { DeleteObj(temp) ReturnFalse; } if(!mpa.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp) DeleteObj(mpa) ReturnFalse; } if(!temp.Add(mpa)) { DeleteObj(temp) DeleteObj(mpa) ReturnFalse; } break; case defNeuronTMA: tma = new CNeuronTMA(); if(!tma) { DeleteObj(temp); ReturnFalse; } if(!tma.Init(outputs, 0, opencl, desc.units, desc.windows, desc.window, desc.count, desc.heads[0], desc.heads[1], desc.layers, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(tma); ReturnFalse; } if(!temp.Add(tma)) { DeleteObj(temp); DeleteObj(tma); ReturnFalse; } break; case defNeuronSpikeSTConvGRU: st_convgru = new CNeuronSpikeSTConvGRU(); if(!st_convgru) { DeleteObj(temp); ReturnFalse; } if(!st_convgru.Init(outputs, 0, opencl, desc.count, desc.windows, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(st_convgru); ReturnFalse; } if(!temp.Add(st_convgru)) { DeleteObj(temp); DeleteObj(st_convgru); ReturnFalse; } break; case defNeuronFERENet: fere = new CNeuronFERENet(); if(!fere) { DeleteObj(temp); ReturnFalse; } if(!fere.Init(outputs, 0, opencl, desc.count, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(fere); ReturnFalse; } if(!temp.Add(fere)) { DeleteObj(temp); DeleteObj(fere); ReturnFalse; } break; case defNeuronFGDModule: fgd = new CNeuronFGDModule(); if(!fgd) { DeleteObj(temp); ReturnFalse; } if(!fgd.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(fgd); ReturnFalse; } if(!temp.Add(fgd)) { DeleteObj(temp); DeleteObj(fgd); ReturnFalse; } break; case defNeuronEVMGRFlowNet: evmgr = new CNeuronEVMGRFlowNet(); if(!evmgr) { DeleteObj(temp); ReturnFalse; } if(!evmgr.Init(outputs, 0, opencl, desc.count, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(evmgr); ReturnFalse; } if(!temp.Add(evmgr)) { DeleteObj(temp); DeleteObj(evmgr); ReturnFalse; } break; case defNeuronSpikeConvBlock: spike_block = new CNeuronSpikeConvBlock(); if(!spike_block) { DeleteObj(temp); ReturnFalse; } if(!spike_block.Init(outputs, 0, opencl, desc.window, desc.step, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(spike_block); ReturnFalse; } if(!temp.Add(spike_block)) { DeleteObj(temp); DeleteObj(spike_block); ReturnFalse; } break; case defNeuronMSRes: msres = new CNeuronMSRes(); if(!msres) { DeleteObj(temp); ReturnFalse; } if(!msres.Init(outputs, 0, opencl, desc.count, desc.window, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(msres); ReturnFalse; } if(!temp.Add(msres)) { DeleteObj(temp); DeleteObj(msres); ReturnFalse; } break; case defNeuronSpikeQKAttention: qkatt = new CNeuronSpikeQKAttention(); if(!qkatt) { DeleteObj(temp); ReturnFalse; } if(!qkatt.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(qkatt); ReturnFalse; } if(!temp.Add(qkatt)) { DeleteObj(temp); DeleteObj(qkatt); ReturnFalse; } break; case defNeuronSDSA: sdsa = new CNeuronSDSA(); if(!sdsa) { DeleteObj(temp); ReturnFalse; } if(!sdsa.Init(outputs, 0, opencl, desc.count, desc.window, desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sdsa); ReturnFalse; } if(!temp.Add(sdsa)) { DeleteObj(temp); DeleteObj(sdsa); ReturnFalse; } break; case defNeuronSDformerUBlock: sdublock = new CNeuronSDformerUBlock(); if(!sdublock) { DeleteObj(temp); ReturnFalse; } if(!sdublock.Init(outputs, 0, opencl, desc.count, desc.windows, desc.heads, desc.units, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sdublock); ReturnFalse; } if(!temp.Add(sdublock)) { DeleteObj(temp); DeleteObj(sdublock); ReturnFalse; } break; case defNeuronSDformerFlow: sdflow = new CNeuronSDformerFlow(); if(!sdflow) { DeleteObj(temp); ReturnFalse; } if(!sdflow.Init(outputs, 0, opencl, desc.count, desc.windows, desc.heads, desc.units, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sdflow); ReturnFalse; } if(!temp.Add(sdflow)) { DeleteObj(temp); DeleteObj(sdflow); ReturnFalse; } case defNeuronSpikePatchStak: sp_patch_stak = new CNeuronSpikePatchStak(); if(!sp_patch_stak) { DeleteObj(temp); ReturnFalse; } if(!sp_patch_stak.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_patch_stak); ReturnFalse; } if(!temp.Add(sp_patch_stak)) { DeleteObj(temp); DeleteObj(sp_patch_stak); ReturnFalse; } break; case defNeuronSpike2DSSMOCL: sp_ssm = new CNeuronSpike2DSSMOCL(); if(!sp_ssm) { DeleteObj(temp); ReturnFalse; } if(!sp_ssm.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_ssm); ReturnFalse; } if(!temp.Add(sp_ssm)) { DeleteObj(temp); DeleteObj(sp_ssm); ReturnFalse; } break; case defNeuronSpikeMamba2D: sp_mamba = new CNeuronSpikeMamba2D(); if(!sp_mamba) { DeleteObj(temp); ReturnFalse; } if(!sp_mamba.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_mamba); ReturnFalse; } if(!temp.Add(sp_mamba)) { DeleteObj(temp); DeleteObj(sp_mamba); ReturnFalse; } break; case defNeuronSpikeSTSSM: sp_stssm = new CNeuronSpikeSTSSM(); if(!sp_stssm) { DeleteObj(temp); ReturnFalse; } if(!sp_stssm.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.windows[2], desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_stssm); ReturnFalse; } if(!temp.Add(sp_stssm)) { DeleteObj(temp); DeleteObj(sp_stssm); ReturnFalse; } break; case defNeuronESTMEncoder: sp_estm = new CNeuronESTMEncoder(); if(!sp_estm) { DeleteObj(temp); ReturnFalse; } if(!sp_estm.Init(outputs, 0, opencl, desc.heads, desc.windows, desc.window, desc.window_out, desc.units, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(sp_estm); ReturnFalse; } if(!temp.Add(sp_estm)) { DeleteObj(temp); DeleteObj(sp_estm); ReturnFalse; } break; case defNeuronCreateFlow: cr_flow = new CNeuronCreateFlow(); if(!cr_flow) { DeleteObj(temp); ReturnFalse; } if(!cr_flow.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(cr_flow); ReturnFalse; } if(!temp.Add(cr_flow)) { DeleteObj(temp); DeleteObj(cr_flow); ReturnFalse; } break; case defNeuronBiDirectCorrelation: bi_corr = new CNeuronBiDirectCorrelation(); if(!bi_corr) { DeleteObj(temp); ReturnFalse; } if(!bi_corr.Init(outputs, 0, opencl, desc.window, desc.count, desc.variables, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(bi_corr); ReturnFalse; } if(!temp.Add(bi_corr)) { DeleteObj(temp); DeleteObj(bi_corr); ReturnFalse; } break; case defNeuronSATMA: satma = new CNeuronSATMA(); if(!satma) { DeleteObj(temp); ReturnFalse; } if(!satma.Init(outputs, 0, opencl, desc.window, desc.count, desc.variables, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(satma); ReturnFalse; } if(!temp.Add(satma)) { DeleteObj(temp); DeleteObj(satma); ReturnFalse; } break; case defNeuronBAT: bat = new CNeuronBAT(); if(!bat) { DeleteObj(temp); ReturnFalse; } if(!bat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(bat); ReturnFalse; } if(!temp.Add(bat)) { DeleteObj(temp); DeleteObj(bat); ReturnFalse; } break; case defNeuronBATvADM: bat = new CNeuronBATvADM(); if(!bat) { DeleteObj(temp); ReturnFalse; } if(!bat.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(bat); ReturnFalse; } if(!temp.Add(bat)) { DeleteObj(temp); DeleteObj(bat); ReturnFalse; } break; case defNeuronCDCSelfCorrector: selfcorrect = new CNeuronCDCSelfCorrector(); if(!selfcorrect) { DeleteObj(temp); ReturnFalse; } if(!selfcorrect.Init(outputs, 0, opencl, desc.window, desc.windows, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(selfcorrect); ReturnFalse; } if(!temp.Add(selfcorrect)) { DeleteObj(temp); DeleteObj(selfcorrect); ReturnFalse; } break; case defNeuronSpikeCDC: cdc = new CNeuronSpikeCDC(); if(!cdc) { DeleteObj(temp); ReturnFalse; } if(!cdc.Init(outputs, 0, opencl, desc.window, desc.windows, desc.window_out, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(cdc); ReturnFalse; } if(!temp.Add(cdc)) { DeleteObj(temp); DeleteObj(cdc); ReturnFalse; } break; case defNeuronSpikeEEMFLow: eemflow = new CNeuronSpikeEEMFLow(); if(!eemflow) { DeleteObj(temp); ReturnFalse; } if(!eemflow.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(eemflow); ReturnFalse; } if(!temp.Add(eemflow)) { DeleteObj(temp); DeleteObj(eemflow); ReturnFalse; } break; case defNeuronSpikeDepthWiseConv: dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv) { DeleteObj(temp); ReturnFalse; } if(!dw_conv.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.layers, desc.step, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(dw_conv); ReturnFalse; } if(!temp.Add(dw_conv)) { DeleteObj(temp); DeleteObj(dw_conv); ReturnFalse; } break; case defNeuronMultiScaleExcitation: excit = new CNeuronMultiScaleExcitation(); if(!excit) { DeleteObj(temp); ReturnFalse; } if(!excit.Init(outputs, 0, opencl, desc.window, desc.count, desc.step, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(excit); ReturnFalse; } if(!temp.Add(excit)) { DeleteObj(temp); DeleteObj(excit); ReturnFalse; } break; case defNeuronMultiScaleDifference: ms_diff = new CNeuronMultiScaleDifference(); if(!ms_diff) { DeleteObj(temp); ReturnFalse; } if(!ms_diff.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.windows, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(ms_diff); ReturnFalse; } if(!temp.Add(ms_diff)) { DeleteObj(temp); DeleteObj(ms_diff); ReturnFalse; } break; case defNeuronCorrelationEncoder: corr_enc = new CNeuronCorrelationEncoder(); if(!corr_enc) { DeleteObj(temp); ReturnFalse; } if(!corr_enc.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(corr_enc); ReturnFalse; } if(!temp.Add(corr_enc)) { DeleteObj(temp); DeleteObj(corr_enc); ReturnFalse; } break; case defNeuronEDCFlow: edcflow = new CNeuronEDCFlow(); if(!edcflow) { DeleteObj(temp); ReturnFalse; } if(!edcflow.Init(outputs, 0, opencl, desc.windows[0], desc.units[0], desc.window_out, desc.windows[1], desc.units[1], desc.heads, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObj(edcflow); ReturnFalse; } if(!temp.Add(edcflow)) { DeleteObj(temp); DeleteObj(edcflow); ReturnFalse; } break; case defNeuronCreateICEFlow: ice = new CNeuronCreateICEFlow(); if(!ice) DeleteObjAndFalse(temp); if(!ice.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(ice); } if(!temp.Add(ice)) { DeleteObj(temp); DeleteObjAndFalse(ice); } break; case defNeuronSpikeMFE: sp_mfe = new CNeuronSpikeMFE(); if(!sp_mfe) DeleteObjAndFalse(temp); if(!sp_mfe.Init(outputs, 0, opencl, desc.count, desc.windows, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(sp_mfe); } if(!temp.Add(sp_mfe)) { DeleteObj(temp); DeleteObjAndFalse(sp_mfe); } break; case defNeuronSpikeMHCrossAttention: sp_crosatt = new CNeuronSpikeMHCrossAttention(); if(!sp_crosatt) DeleteObjAndFalse(temp); if(!sp_crosatt.Init(outputs, 0, opencl, desc.windows[0], desc.window_out, desc.step, desc.units[0], desc.windows[1], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(sp_crosatt); } if(!temp.Add(sp_crosatt)) { DeleteObj(temp); DeleteObjAndFalse(sp_crosatt); } break; case defNeuronSpikeMixFusion: mix_fusion = new CNeuronSpikeMixFusion(); if(!mix_fusion) DeleteObjAndFalse(temp); if(!mix_fusion.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(mix_fusion); } if(!temp.Add(mix_fusion)) { DeleteObj(temp); DeleteObjAndFalse(mix_fusion); } break; case defNeuronSTFlow: stflow = new CNeuronSTFlow(); if(!stflow) DeleteObjAndFalse(temp); if(!stflow.Init(outputs, 0, opencl, desc.windows[0], desc.units[0], desc.window_out, desc.windows[1], desc.units[1], desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(stflow); } if(!temp.Add(stflow)) { DeleteObj(temp); DeleteObjAndFalse(stflow); } break; case defNeuronPSSE: pse = new CNeuronPSSE(); if(!pse) DeleteObjAndFalse(temp); if(!pse.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(pse); } if(!temp.Add(pse)) { DeleteObj(temp); DeleteObjAndFalse(pse); } break; case defNeuronSpikeDepthWiseResidual: dwr = new CNeuronSpikeDepthWiseResidual(); if(!dwr) DeleteObjAndFalse(temp); if(!dwr.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.layers, desc.step, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(dwr); } if(!temp.Add(dwr)) { DeleteObj(temp); DeleteObjAndFalse(dwr); } break; case defNeuronSpikeSuperKernelBlock: sk_block = new CNeuronSpikeSuperKernelBlock(); if(!sk_block) DeleteObjAndFalse(temp); if(!sk_block.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.windows, desc.heads, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(sk_block); } if(!temp.Add(sk_block)) { DeleteObj(temp); DeleteObjAndFalse(sk_block); } break; case defNeuronSpikeGMA: gma = new CNeuronSpikeGMA(); if(!gma) DeleteObjAndFalse(temp); if(!gma.Init(outputs, 0, opencl, desc.window, desc.count, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(gma); } if(!temp.Add(gma)) { DeleteObj(temp); DeleteObjAndFalse(gma); } break; case defNeuronETROF: etrof = new CNeuronETROF(); if(!etrof) DeleteObjAndFalse(temp); if(!etrof.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(etrof); } if(!temp.Add(etrof)) { DeleteObj(temp); DeleteObjAndFalse(etrof); } break; case defNeuronPSSEFlow: pseflow = new CNeuronPSSEFlow(); if(!pseflow) DeleteObjAndFalse(temp); if(!pseflow.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(pseflow); } if(!temp.Add(pseflow)) { DeleteObj(temp); DeleteObjAndFalse(pseflow); } break; case defNeuronSpikeUSR: usr = new CNeuronSpikeUSR(); if(!usr) DeleteObjAndFalse(temp); if(!usr.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(usr); } if(!temp.Add(usr)) { DeleteObj(temp); DeleteObjAndFalse(usr); } break; case defNeuronInitialFlow: init_flow = new CNeuronInitialFlow(); if(!init_flow) DeleteObjAndFalse(temp); if(!init_flow.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.variables, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(init_flow); } if(!temp.Add(init_flow)) { DeleteObj(temp); DeleteObjAndFalse(init_flow); } break; case defNeuronSpikeSMR: smr = new CNeuronSpikeSMR(); if(!smr) DeleteObjAndFalse(temp); if(!smr.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.window_out, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(smr); } if(!temp.Add(smr)) { DeleteObj(temp); DeleteObjAndFalse(smr); } break; case defNeuronEVAFlow: eva = new CNeuronEVAFlow(); if(!eva) DeleteObjAndFalse(temp); if(!eva.Init(outputs, 0, opencl, desc.count, desc.window, desc.window_out, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(eva); } if(!temp.Add(eva)) { DeleteObj(temp); DeleteObjAndFalse(eva); } break; case defNeuronResFlowHTR: htr = new CNeuronResFlowHTR(); if(!htr) DeleteObjAndFalse(temp); if(!htr.Init(outputs, 0, opencl, desc.count, desc.windows[0], desc.windows[1], desc.windows[2], desc.windows[3], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(htr); } if(!temp.Add(htr)) { DeleteObj(temp); DeleteObjAndFalse(htr); } break; case defNeuronResFlow: resflow = new CNeuronResFlow(); if(!resflow) DeleteObjAndFalse(temp); if(!resflow.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(resflow); } if(!temp.Add(resflow)) { DeleteObj(temp); DeleteObjAndFalse(resflow); } break; //--- case defNeuronConfGateTailAwareMoE: tailaware = new CNeuronConfGateTailAwareMoE(); if(!tailaware) DeleteObjAndFalse(temp); if(!tailaware.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(tailaware); } if(!temp.Add(tailaware)) { DeleteObj(temp); DeleteObjAndFalse(tailaware); } break; case defNeuronResFlowLattice: lattice = new CNeuronResFlowLattice(); if(!lattice) DeleteObjAndFalse(temp); if(!lattice.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.variables, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(lattice); } if(!temp.Add(lattice)) { DeleteObj(temp); DeleteObjAndFalse(lattice); } break; case defNeuronFieldAwareConv: faconv = new CNeuronFieldAwareConv(); if(!faconv) DeleteObjAndFalse(temp); if(!faconv.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.count, desc.windows[2], desc.variables, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(faconv); } if(!temp.Add(faconv)) { DeleteObj(temp); DeleteObjAndFalse(faconv); } faconv.SetActivationFunction(desc.activation); break; case defNeuronMHFAT: fat = new CNeuronMHFAT(); if(!fat) DeleteObjAndFalse(temp); if(!fat.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.step, desc.count, desc.windows[2], desc.layers, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(fat); } if(!temp.Add(fat)) { DeleteObj(temp); DeleteObjAndFalse(fat); } break; case defNeuronMHCrossFAT: cross_fat = new CNeuronMHCrossFAT(); if(!cross_fat) DeleteObjAndFalse(temp); if(!cross_fat.Init(outputs, 0, opencl, desc.windows[0], desc.windows[1], desc.step, desc.units[0], desc.windows[2], desc.units[1], desc.windows[3], desc.layers, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(cross_fat); } if(!temp.Add(cross_fat)) { DeleteObj(temp); DeleteObjAndFalse(cross_fat); } break; case defNeuronFieldPatternEmbedding: faemb = new CNeuronFieldPatternEmbedding(); if(!faemb) DeleteObjAndFalse(temp); if(!faemb.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(faemb); } if(!temp.Add(faemb)) { DeleteObj(temp); DeleteObjAndFalse(faemb); } faemb.SetActivationFunction(desc.activation); break; case defNeuronAutoToken: autotoken = new CNeuronAutoToken(); if(!autotoken) DeleteObjAndFalse(temp); if(!autotoken.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.units[0], desc.units[1], desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(autotoken); } if(!temp.Add(autotoken)) { DeleteObj(temp); DeleteObjAndFalse(autotoken); } break; case defNeuronMultiMixAttention: mixattention = new CNeuronMultiMixAttention(); if(!mixattention) DeleteObjAndFalse(temp); if(!mixattention.Init(outputs, 0, opencl, desc.window, desc.count, desc.step, desc.window_out, desc.layers, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(mixattention); } if(!temp.Add(mixattention)) { DeleteObj(temp); DeleteObjAndFalse(mixattention); } break; case defNeuronSparseMoEConv: sparsemoe = new CNeuronSparseMoEConv(); if(!sparsemoe) DeleteObjAndFalse(temp); if(!sparsemoe.Init(outputs, 0, opencl, desc.window, desc.window_out, desc.count, desc.layers, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(sparsemoe); } if(!temp.Add(sparsemoe)) { DeleteObj(temp); DeleteObjAndFalse(sparsemoe); } sparsemoe.SetActivationFunction(desc.activation); break; case defNeuronMTmixAttBlock: mixattenblock = new CNeuronMTmixAttBlock(); if(!mixattenblock) DeleteObjAndFalse(temp); if(!mixattenblock.Init(outputs, 0, opencl, desc.window, desc.count, desc.step, desc.window_out, desc.layers, desc.variables, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(mixattenblock); } if(!temp.Add(mixattenblock)) { DeleteObj(temp); DeleteObjAndFalse(mixattenblock); } break; case defCrossMHFlashAttention: flash_att = new CCrossMHFlashAttention(); if(!flash_att) DeleteObjAndFalse(temp); if(!flash_att.Init(outputs, 0, opencl, desc.window, desc.step, (bool)desc.probability, desc.units[0], desc.units[1], desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(flash_att); } if(!temp.Add(flash_att)) { DeleteObj(temp); DeleteObjAndFalse(flash_att); } break; case defNeuronMixFeedForward: mixff = new CNeuronMixFeedForward(); if(!mixff) DeleteObjAndFalse(temp); if(!mixff.Init(outputs, 0, opencl, desc.window, desc.count, desc.window_out, desc.variables, desc.step, desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(mixff); } if(!temp.Add(mixff)) { DeleteObj(temp); DeleteObjAndFalse(mixff); } break; case defNeuronOneTrans: onetrans = new CNeuronOneTrans(); if(!onetrans) DeleteObjAndFalse(temp); if(!onetrans.Init(outputs, 0, opencl, desc.windows, desc.units[0], desc.units[1], desc.step, desc.count, desc.layers, desc.window_out, desc.variables, uint(desc.probability), desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(onetrans); } if(!temp.Add(onetrans)) { DeleteObj(temp); DeleteObjAndFalse(onetrans); } break; case defNeuronMHTHCrossAttention: thcatt = new CNeuronMHTHCrossAttention(); if(!thcatt) DeleteObjAndFalse(temp); if(!thcatt.Init(outputs, 0, opencl, desc.windows[0], desc.units[0], desc.step, desc.windows[1], desc.units[1], desc.window_out, desc.variables, uint(desc.probability), desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(thcatt); } if(!temp.Add(thcatt)) { DeleteObj(temp); DeleteObjAndFalse(thcatt); } break; case defNeuronSTCA: stca = new CNeuronSTCA(); if(!stca) DeleteObjAndFalse(temp); if(!stca.Init(outputs, 0, opencl, desc.windows, desc.units[0], desc.units[1], desc.step, desc.units[2], desc.count, desc.layers, desc.window_out, desc.variables, uint(desc.probability), desc.optimization, desc.batch)) { DeleteObj(temp); DeleteObjAndFalse(stca); } if(!temp.Add(stca)) { DeleteObj(temp); DeleteObjAndFalse(stca); } break; //--- default: ReturnFalse; break; } } else for(uint n = 0; n < neurons; n++) { switch(desc.type) { case defNeuron: neuron = new CNeuron(); if(CheckPointer(neuron) == POINTER_INVALID) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } neuron.Init(outputs, n, desc.optimization); neuron.SetActivationFunction(desc.activation); break; case defNeuronConv: neuron_p = new CNeuronConv(); if(CheckPointer(neuron_p) == POINTER_INVALID) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronProof: neuron_p = new CNeuronProof(); if(CheckPointer(neuron_p) == POINTER_INVALID) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } if(CheckPointer(prev) != POINTER_INVALID) { if(prev.type == defNeuron) { temp_count = (int)((prev.count - desc.window) % desc.step); output_count = (int)((prev.count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } else if(n == 0) { temp_count = (int)((output_count - desc.window) % desc.step); output_count = (int)((output_count - desc.window - temp_count) / desc.step + (temp_count == 0 ? 1 : 2)); } } if(neuron_p.Init(outputs, n, desc.window, desc.step, output_count, desc.optimization)) neuron = neuron_p; break; case defNeuronLSTM: neuron_p = new CNeuronLSTM(); if(CheckPointer(neuron_p) == POINTER_INVALID) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } output_count = (next != NULL ? next.window : desc.step); if(neuron_p.Init(outputs, n, desc.window, 1, output_count, desc.optimization)) neuron = neuron_p; break; } if(!temp.Add(neuron)) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } neuron = NULL; } if(!layers.Add(temp)) { DeleteObj(temp); DeleteObj(layers); ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CArrayFloat *inputVals, int window = 1, bool tem = true, CBufferFloat *SecondInput = NULL) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1) ReturnFalse; //--- CLayer *previous = NULL; CLayer *current = layers.At(0); int total = MathMin(current.Total(), inputVals.Total()); CNeuronBase *neuron = NULL; if(CheckPointer(opencl) == POINTER_INVALID) { for(int i = 0; i < total; i++) { neuron = current.At(i); if(CheckPointer(neuron) == POINTER_INVALID) ReturnFalse; int pos = i; int d = 0; if(window > 1) { d = i % window; pos = (i - d) / window; } neuron.setOutputVal(inputVals.At(i) + (float)(tem ? (d % 2 == 0 ? sin(pos / pow(10000, (2 * d + 1) / (window + 1))) : cos(pos / pow(10000, (2 * d + 1) / (window + 1)))) : 0)); if(tem) inputVals.Update(i, neuron.getOutputVal()); } } else { CNeuronBaseOCL *neuron_ocl = current.At(0); CBufferFloat *inputs = neuron_ocl.getOutput(); if(tem) { int total_data = inputVals.Total(); for(int d = 0; d < total_data; d++) { int pos = d; int dim = 0; if(window > 1) { dim = d % window; pos = (d - dim) / window; } float value = pos / pow(10000, (2 * dim + 1) / (float)(window + 1)); value = (float)(inputVals.At(d) + (dim % 2 == 0 ? sin(value) : cos(value))); if(!inputVals.Update(d, value)) ReturnFalse; } } if(!inputs.Fill(inputVals)) ReturnFalse; //--- if(!!SecondInput) { if(SecondInput.GetIndex() < 0) if(!SecondInput.BufferCreate(opencl)) ReturnFalse; } } //--- CObject *temp = NULL; #ifdef _DEBUG vector res; #endif for(int l = 1; l < layers.Total(); l++) { previous = current; current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID) ReturnFalse; //--- if(CheckPointer(opencl) != POINTER_INVALID) { CNeuronBaseOCL *current_ocl = current.At(0); if(!current_ocl.FeedForward(previous.At(0), SecondInput)) ReturnFalse; #ifdef _DEBUG if(!current_ocl.getOutputVal(res)) break; #endif continue; } //--- total = current.Total() % 5; if(current.At(0).Type() == defNeuron) total--; //--- for(int n = 0; n < total; n++) { neuron = current.At(n); if(CheckPointer(neuron) == POINTER_INVALID) ReturnFalse; if(previous.At(0).Type() == defNeuron) { temp = previous; if(!neuron.feedForward(temp)) ReturnFalse; continue; } if(neuron.Type() == defNeuron) { if(n == 0) { CLayer *temp_l = new CLayer(total); if(CheckPointer(temp_l) == POINTER_INVALID) ReturnFalse; CNeuronProof *proof = NULL; for(int p = 0; p < previous.Total(); p++) { proof = previous.At(p); if(CheckPointer(proof) == POINTER_INVALID) ReturnFalse; temp_l.AddArray(proof.getOutputLayer()); } temp = temp_l; } if(!neuron.feedForward(temp)) ReturnFalse; if(n == total - 1) { CLayer *temp_l = temp; temp_l.FreeMode(false); temp_l.Shutdown(); DeleteObj(temp_l); } continue; } temp = previous.At(n); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!neuron.feedForward(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::backProp(CArrayFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID) ReturnFalse; if(CheckPointer(opencl) != POINTER_INVALID) { if(!backPropOCL(targetVals, SecondInput, SecondGradient, false)) ReturnFalse; return true; } //--- CLayer *outputLayer = layers.At(layers.Total() - 1); if(CheckPointer(outputLayer) == POINTER_INVALID) ReturnFalse; //--- float error = 0.0; int total = outputLayer.Total() - 1; for(int n = 0; n < total && !IsStopped(); n++) { CNeuron *neuron = outputLayer.At(n); float target = targetVals.At(n); float delta = (target > 1 ? 1 : target < -1 ? -1 : target) - neuron.getOutputVal(); error += delta * delta; neuron.calcOutputGradients(targetVals.At(n)); } error /= (float)total; error = (float)sqrt(error); backPropCount++; recentAverageError += (error - recentAverageError) / fmin(recentAverageSmoothingFactor, backPropCount); //--- CNeuronBase *neuron = NULL; CObject *temp = NULL; for(int layerNum = layers.Total() - 2; layerNum > 0; layerNum--) { CLayer *hiddenLayer = layers.At(layerNum); CLayer *nextLayer = layers.At(layerNum + 1); total = hiddenLayer.Total(); for(int n = 0; n < total && !IsStopped(); ++n) { neuron = hiddenLayer.At(n); if(nextLayer.At(0).Type() == defNeuron) { temp = nextLayer; neuron.calcHiddenGradients(temp); continue; } if(neuron.Type() == defNeuron) { float g = 0; for(int i = 0; i < nextLayer.Total(); i++) { temp = nextLayer.At(i); neuron.calcHiddenGradients(temp); g += neuron.getGradient(); } neuron.setGradient(g); continue; } temp = nextLayer.At(n); neuron.calcHiddenGradients(temp); } } //--- for(int layerNum = layers.Total() - 1; layerNum > 0; layerNum--) { CLayer *layer = layers.At(layerNum); CLayer *prevLayer = layers.At(layerNum - 1); total = layer.Total() - (layer.At(0).Type() == defNeuron ? 1 : 0); int n_conv = 0; for(int n = 0; n < total && !IsStopped(); n++) { neuron = layer.At(n); if(CheckPointer(neuron) == POINTER_INVALID) ReturnFalse; if(neuron.Type() == defNeuronProof) continue; switch(prevLayer.At(0).Type()) { case defNeuron: temp = prevLayer; neuron.updateInputWeights(temp); break; case defNeuronConv: case defNeuronProof: case defNeuronLSTM: if(neuron.Type() == defNeuron) { for(n_conv = 0; n_conv < prevLayer.Total(); n_conv++) { temp = prevLayer.At(n_conv); neuron.updateInputWeights(temp); } } else { temp = prevLayer.At(n); neuron.updateInputWeights(temp); } break; default: temp = NULL; break; } } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::backPropOCL(CBufferFloat *targetVals, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, bool layer_by_layer = false, ENUM_ACTIVATION SecondActivation = None) { if(CheckPointer(targetVals) == POINTER_INVALID || CheckPointer(layers) == POINTER_INVALID || CheckPointer(opencl) == POINTER_INVALID) ReturnFalse; CLayer *currentLayer = layers.At(layers.Total() - 1); if(CheckPointer(currentLayer) == POINTER_INVALID) ReturnFalse; //--- float error = 0.0; int total = targetVals.Total(); CNeuronBaseOCL *neuron = currentLayer.At(0); if(neuron.Type() != defNeuronSoftMaxOCL && neuron.Type() != defNeuronMantisAttentionUnit ) { vector target, output; if(neuron.getOutputVal(output) < neuron.Neurons()) ReturnFalse; if(output.HasNan() > 0) { printf("%s - %d -> Output has NaN", __FUNCTION__, __LINE__); ReturnFalse; } targetVals.GetData(target); if(target.Size() > output.Size()) { if(!target.Resize(output.Size())) ReturnFalse; } error = output.Loss(target, LOSS_MSE); } //--- if(!neuron.calcOutputGradients(targetVals, error)) ReturnFalse; //--- if(neuron.TrainMode()) { if(!MathIsValidNumber(error)) error = getRecentAverageError(); backPropCount++; recentAverageError += (error - recentAverageError) / fmin(recentAverageSmoothingFactor, (float)backPropCount); } //--- if(layer_by_layer) { total = layers.Total(); for(int layer_update = 1; layer_update < total; layer_update++) { for(int layerNum = total - 2; layerNum > layer_update - 1 && !IsStopped(); layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); neuron = currentLayer.At(0); if(!neuron.CalcHiddenGradients(nextLayer.At(0), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; } //--- CLayer *prevLayer = layers.At(layer_update - 1); currentLayer = layers.At(layer_update); neuron = currentLayer.At(0); if(!neuron.UpdateInputWeights(prevLayer.At(0), SecondInput)) ReturnFalse; //--- for(int layerNum = layer_update; layerNum < total; layerNum++) { currentLayer = layers.At(layerNum); if(CheckPointer(currentLayer) == POINTER_INVALID) ReturnFalse; //--- CNeuronBaseOCL *current_ocl = currentLayer.At(0); if(!current_ocl.FeedForward(prevLayer.At(0), SecondInput)) ReturnFalse; prevLayer = currentLayer; } } return true; } //--- Calc Hidden Gradients total = layers.Total(); for(int layerNum = total - 2; layerNum >= 0 && !IsStopped(); layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); neuron = currentLayer.At(0); if(!neuron.CalcHiddenGradients(nextLayer.At(0), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; #ifdef _DEBUG if(!neuron.getGradient().BufferRead()) ReturnFalse; #endif } //--- CLayer *prevLayer = layers.At(total - 1); for(int layerNum = total - 1; layerNum > 0 && !IsStopped(); layerNum--) { currentLayer = prevLayer; prevLayer = layers.At(layerNum - 1); neuron = currentLayer.At(0); if(neuron.TrainMode()) if(!neuron.UpdateInputWeights(prevLayer.At(0), SecondInput)) ReturnFalse; } //--- bool result = false; for(int layerNum = 0; (layerNum < layers.Total() && !result && !IsStopped()); layerNum++) { currentLayer = layers.At(layerNum); CNeuronBaseOCL *temp = currentLayer.At(0); CNeuronConvOCL *conv = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; if(!temp.TrainMode()) { if(!!temp.getGradient()) temp.getGradient().BufferRead(); if(layerNum == layers.Total() - 1) result = true; continue; } switch(temp.Type()) { case defNeuronConvOCL: conv = temp; if(!!conv.GetWeightsConv() && conv.GetWeightsConv().BufferRead()) result = true; else if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; if(!!batch.getBatchOptions() && batch.getBatchOptions().BufferRead()) result = true; if(!result && !!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronLSTMOCL: lstm = temp; if(!!lstm.getLSTMWeights() && lstm.getLSTMWeights().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; default: if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; } } //--- if(!!SecondGradient) SecondGradient.BufferRead(); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::getResults(CBufferFloat *&resultVals) { if(CheckPointer(resultVals) == POINTER_INVALID) { resultVals = new CBufferFloat(); if(CheckPointer(resultVals) == POINTER_INVALID) return; } //--- resultVals.Clear(); if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 0) return; //--- CLayer *output = layers.At(layers.Total() - 1); if(CheckPointer(output) == POINTER_INVALID) return; //--- if(CheckPointer(opencl) != POINTER_INVALID && output.At(0).Type() >= defNeuronBaseOCL) { CNeuronBaseOCL *temp = output.At(0); temp.getOutputVal(resultVals); return; } CNeuronBase *neuron = NULL; CLayer *temp = NULL; int total = output.Total(); if(output.At(0).Type() == defNeuron) total--; //--- for(int i = 0; i < total; i++) { neuron = output.At(i); if(CheckPointer(neuron) == POINTER_INVALID) continue; if(neuron.Type() == defNeuron) { resultVals.Add(neuron.getOutputVal()); continue; } CNeuronProof *n = neuron; temp = n.getOutputLayer(); for(int ii = 0; ii < temp.Total(); ii++) { neuron = temp.At(ii); if(CheckPointer(neuron) == POINTER_INVALID) continue; resultVals.Add(neuron.getOutputVal()); } } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Save(string file_name, float error, float undefine, float forecast, datetime time, bool common = true) { //if(MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD) || MQLInfoInteger(MQL_OPTIMIZATION)) // return true; if(file_name == NULL || !layers) ReturnFalse; //--- int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_WRITE); if(handle == INVALID_HANDLE) ReturnFalse; //--- if(FileWriteDouble(handle, error) <= 0 || FileWriteDouble(handle, undefine) <= 0 || FileWriteDouble(handle, forecast) <= 0 || FileWriteLong(handle, (long)time) <= 0) { FileClose(handle); ReturnFalse; } bool result = layers.Save(handle); FileFlush(handle); FileClose(handle); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Save(const int file_handle) { if(file_handle == INVALID_HANDLE || !layers) ReturnFalse; //--- if(FileWriteDouble(file_handle, (double)recentAverageError) <= 0 || FileWriteDouble(file_handle, 0) <= 0 || FileWriteDouble(file_handle, 0) <= 0 || FileWriteLong(file_handle, (long)0) <= 0) ReturnFalse; //--- return layers.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Load(string file_name, float &error, float &undefine, float &forecast, datetime &time, bool common = true) { //if(MQLInfoInteger(MQL_OPTIMIZATION) || MQLInfoInteger(MQL_TESTER) || MQLInfoInteger(MQL_FORWARD) || MQLInfoInteger(MQL_OPTIMIZATION)) // ReturnFalse; //--- if(file_name == NULL) ReturnFalse; //--- Print(file_name); int handle = FileOpen(file_name, (common ? FILE_COMMON : 0) | FILE_BIN | FILE_READ | FILE_SHARE_READ); if(handle == INVALID_HANDLE) ReturnFalse; //--- error = (float)FileReadDouble(handle); undefine = (float)FileReadDouble(handle); forecast = (float)FileReadDouble(handle); time = (datetime)FileReadLong(handle); recentAverageError = fmax(error, 0); //--- if(CheckPointer(layers) != POINTER_INVALID) layers.Clear(); else layers = new CArrayLayer(); int i = 0, num; //--- if(!opencl) OpenCLInit(); //--- check //--- read and check start marker - 0xFFFFFFFFFFFFFFFF long temp = FileReadLong(handle); if(temp == -1) { //--- read and check array type if(FileReadInteger(handle, INT_VALUE) != layers.Type()) { FileClose(handle); return(false); } } else { FileClose(handle); return(false); } //--- read array length num = FileReadInteger(handle, INT_VALUE); //--- read array if(num != 0) { for(i = 0; i < num; i++) { //--- create new element CLayer *Layer = new CLayer(0, handle, opencl); if(!Layer.Load(handle)) { DeleteObj(Layer); break; } if(Layer.At(0).Type() == defNeuronRevInDenormOCL) { CNeuronRevINDenormOCL *revin = Layer.At(0); int l = revin.GetNormLayer(); if(!layers.At(l)) { DeleteObj(Layer); break; } CNeuronBaseOCL *neuron = ((CLayer *)layers.At(l)).At(0); if(!(neuron.Type() == defNeuronBatchNormOCL || neuron.Type() == defNeuronBatchNormWithNoise)) { DeleteObj(Layer); break; } if(!revin.Init(revin.getConnections(), 0, opencl, revin.Neurons(), l, neuron)) { DeleteObj(Layer); break; } } if(Layer.At(0).Type() == defNeuronMASA) { CNeuronMASA *masa = Layer.At(0); int l = masa.GetNormLayer(); if(!layers.At(l)) { DeleteObj(Layer); break; } CNeuronBaseOCL *neuron = ((CLayer *)layers.At(l)).At(0); if(!(neuron.Type() == defNeuronBatchNormOCL) || (neuron.Type() == defNeuronBatchNormWithNoise)) { DeleteObj(Layer); break; } if(!masa.SetNormLayer(l, neuron)) { DeleteObj(Layer); break; } } if(!layers.Add(Layer)) { DeleteObj(Layer); break; } } } FileClose(handle); //--- result return (layers.Total() == num); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Load(const int file_handle) { if(file_handle == INVALID_HANDLE || FileIsEnding(file_handle)) ReturnFalse; //--- { recentAverageError = (float)FileReadDouble(file_handle); float temp = (float)fmax(FileReadDouble(file_handle), 0); temp = (float)FileReadDouble(file_handle); temp = (float)FileReadLong(file_handle); } //--- if(CheckPointer(layers) != POINTER_INVALID) layers.Clear(); else layers = new CArrayLayer(); int i = 0, num; //--- if(CheckPointer(opencl) == POINTER_INVALID) { if(!OpenCLInit()) ReturnFalse; } //--- check //--- read and check start marker - 0xFFFFFFFFFFFFFFFF long temp = FileReadLong(file_handle); if(temp == -1) { //--- read and check array type if(FileReadInteger(file_handle, INT_VALUE) != layers.Type()) return(false); } else { while((temp = FileReadLong(file_handle)) != -1 && !FileIsEnding(file_handle)) { Sleep(0); } if(temp != -1) return(false); } //--- read array length num = FileReadInteger(file_handle, INT_VALUE); //--- read array if(num != 0) { for(i = 0; i < num; i++) { //--- create new element CLayer *Layer = new CLayer(0, file_handle, opencl); if(!Layer.Load(file_handle)) break; if(Layer.At(0).Type() == defNeuronRevInDenormOCL) { CNeuronRevINDenormOCL *revin = Layer.At(0); int l = revin.GetNormLayer(); if(!layers.At(l)) { DeleteObj(Layer); break; } CNeuronBaseOCL *neuron = ((CLayer *)layers.At(l)).At(0); if(!(neuron.Type() == defNeuronBatchNormOCL) || (neuron.Type() == defNeuronBatchNormWithNoise)) { DeleteObj(Layer); break; } if(!revin.Init(revin.getConnections(), 0, opencl, revin.Neurons(), l, neuron)) { DeleteObj(Layer); break; } } if(!layers.Add(Layer)) break; } } FileClose(file_handle); //--- result return (layers.Total() == num); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::TrainMode(bool flag) { if(CheckPointer(layers) == POINTER_INVALID || layers.Total() <= 1) ReturnFalse; //--- CLayer *current = NULL; //--- CObject *temp = NULL; for(int l = 0; l < layers.Total(); l++) { current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID || CheckPointer(current.At(0)) == POINTER_INVALID) ReturnFalse; //--- if(CheckPointer(opencl) != POINTER_INVALID) { CNeuronBaseOCL *current_ocl = current.At(0); current_ocl.TrainMode(flag); } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::Save(const int file_handle) { if(!CNeuronBase::Save(file_handle) || !OutputLayer.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProof::Load(const int file_handle) { if(!CNeuronBase::Load(file_handle) || !OutputLayer.Load(file_handle)) ReturnFalse; iWindow = FileReadInteger(file_handle, INT_VALUE); iStep = FileReadInteger(file_handle, INT_VALUE); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Save(const int file_handle) { if(!CNeuronProof::Save(file_handle)) ReturnFalse; if(FileWriteDouble(file_handle, param) < 8) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConv::Load(const int file_handle) { if(!CNeuronProof::Load(file_handle)) ReturnFalse; param = (float)FileReadDouble(file_handle); //--- return true; } //+------------------------------------------------------------------+ ///\class CNeuronLSTM /// Class of recurrent LSTM unit ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronLSTM : public CNeuronProof { protected: CLayer *ForgetGate; ///< Object of forget gate CLayer *InputGate; ///< Object of input gate CLayer *OutputGate; ///< Object of output gate CLayer *NewContent; ///< Object of new content CArrayFloat *Memory; ///< Memory array CArrayFloat *PrevMemory; ///< Ravious iteration memory array CArrayFloat *Input; ///< Input data CArrayFloat *InputGradient; ///< Gradient on previous layer //--- virtual bool feedForward(CLayer *prevLayer); ///< Feed Forward method. Detailed description on the link.@param prevLayer Pointer to previos layer. virtual bool calcHiddenGradients(CLayer *&nextLayer); ///< Method to transfer gradient to previous layer. @param nextLayer Pointer to next layer. virtual bool updateInputWeights(CLayer *&prevLayer); ///< Method for updating weights.@param prevLayer Pointer to previos layer. virtual bool updateInputWeights(CLayer *gate, CArrayFloat *input_data); ///< Method for updating gates' weights.@param gate Pointer to gate. @param input_data Pointer to tensor with input data. virtual bool InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type); ///< Method of gate initialization @param[in] layer Pointer to gate @param[in] numUnits Number of units in gate @param[in] numOutputs Number of outputs @param[in] optimization_type Type of optimization (#ENUM_OPTIMIZATION) virtual CArrayFloat *CalculateGate(CLayer *gate, CArrayFloat *sequence); ///< Method of calculation gate iteration @param[in] gate Pointer to gate @param[in] sequence Input data @return Array of output data public: /** Constructor */ CNeuronLSTM(void); /** Destructor */~CNeuronLSTM(void); virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type); ///< Unit initialization method. Detailed description on the link. //--- virtual CLayer *getOutputLayer(void) { return OutputLayer; } ///< Method for getting a pointer to the resulting neural layer. Not used in fully connected neural networks.@return Pointer to layer. virtual bool calcInputGradients(CLayer *prevLayer) ; virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ; //--- methods for working with files virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation virtual int Type(void) const { return defNeuronLSTM; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::CNeuronLSTM(void) { ForgetGate = new CLayer(); InputGate = new CLayer(); OutputGate = new CLayer(); NewContent = new CLayer(); Memory = new CArrayFloat(); PrevMemory = new CArrayFloat(); Input = new CArrayFloat(); InputGradient = new CArrayFloat(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTM::~CNeuronLSTM(void) { DeleteObj(ForgetGate); DeleteObj(InputGate); DeleteObj(OutputGate); DeleteObj(NewContent); DeleteObj(Memory); DeleteObj(PrevMemory); DeleteObj(Input); DeleteObj(InputGradient); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type) { if(units_count <= 0) ReturnFalse; //--- Init Layers if(!CNeuronProof::Init(numOutputs, myIndex, window, step, units_count, optimization_type)) ReturnFalse; if(!InitLayer(ForgetGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(InputGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(OutputGate, units_count, window + units_count, optimization_type)) ReturnFalse; if(!InitLayer(NewContent, units_count, window + units_count, optimization_type)) ReturnFalse; if(!Memory.Reserve(units_count)) ReturnFalse; if(!PrevMemory.Reserve(units_count)) ReturnFalse; CNeuron *temp; for(int i = 0; i < units_count; i++) { if(!Memory.Add(0)) ReturnFalse; if(!PrevMemory.Add(0)) ReturnFalse; temp = OutputLayer.At(i); temp.setOutputVal(0); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::InitLayer(CLayer *layer, int numUnits, int numOutputs, ENUM_OPTIMIZATION optimization_type) { if(CheckPointer(layer) == POINTER_INVALID) { layer = new CLayer(numOutputs); if(CheckPointer(layer) == POINTER_INVALID) ReturnFalse; } else layer.Clear(); if(!layer.Reserve(numUnits)) ReturnFalse; //--- CNeuron *temp; for(int i = 0; i < numUnits; i++) { temp = new CNeuron(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Init(numOutputs + 1, i, optimization_type)) ReturnFalse; if(!layer.Add(temp)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::feedForward(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || prevLayer.Total() <= 0) ReturnFalse; CNeuronBase *temp; CConnection *temp_con; if(CheckPointer(Input) == POINTER_INVALID) { Input = new CArrayFloat(); if(CheckPointer(Input) == POINTER_INVALID) ReturnFalse; } else Input.Clear(); //--- Concatenate input sequence int total = prevLayer.Total(); if(!Input.Reserve(total + OutputLayer.Total())) ReturnFalse; for(int i = 0; i < total; i++) { temp = prevLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) ReturnFalse; } total = OutputLayer.Total(); for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID || !Input.Add(temp.getOutputVal())) ReturnFalse; } int total_data = Input.Total(); //--- Calculated forget gate CArrayFloat *forget_gate = CalculateGate(ForgetGate, Input); if(CheckPointer(forget_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated input gate CArrayFloat *input_gate = CalculateGate(InputGate, Input); if(CheckPointer(input_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated output gate CArrayFloat *output_gate = CalculateGate(OutputGate, Input); if(CheckPointer(output_gate) == POINTER_INVALID) ReturnFalse; //--- Calculated new content CArrayFloat *new_content = new CArrayFloat(); if(CheckPointer(new_content) == POINTER_INVALID) ReturnFalse; total = NewContent.Total(); for(int i = 0; i < total; i++) { temp = NewContent.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; float val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) ReturnFalse; val += temp_con.weight * Input.At(c); } val = TanhFunction(val); temp.setOutputVal(val); if(!new_content.Add(val)) ReturnFalse; } //--- Calculated output sequences for(int i = 0; i < total; i++) { if(PrevMemory.Total() <= i) PrevMemory.Add(Memory.At(i)); else PrevMemory.Update(i, Memory.At(i)); float value = Memory.At(i) * forget_gate.At(i) + new_content.At(i) * input_gate.At(i); if(!Memory.Update(i, value)) ReturnFalse; temp = OutputLayer.At(i); value = TanhFunction(value) * output_gate.At(i); temp.setOutputVal(value); } //--- DeleteObj(forget_gate); DeleteObj(input_gate); DeleteObj(new_content); DeleteObj(output_gate); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CArrayFloat *CNeuronLSTM::CalculateGate(CLayer *gate, CArrayFloat *sequence) { CNeuronBase *temp; CConnection *temp_con; CArrayFloat *result = new CArrayFloat(); if(CheckPointer(gate) == POINTER_INVALID) return NULL; int total = gate.Total(); int total_data = sequence.Total(); for(int i = 0; i < total; i++) { temp = gate.At(i); if(CheckPointer(temp) == POINTER_INVALID) { DeleteObj(result); return NULL; } float val = 0; for(int c = 0; c < total_data; c++) { temp_con = temp.Connections.At(c); if(CheckPointer(temp_con) == POINTER_INVALID) { DeleteObj(result); return NULL; } val += temp_con.weight * (sequence.At(c) == DBL_MAX ? 1 : sequence.At(c)); } val = SigmoidFunction(val); temp.setOutputVal(val); if(!result.Add(val)) { DeleteObj(result); return NULL; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcHiddenGradients(CLayer *&nextLayer) { if(CheckPointer(InputGradient) == POINTER_INVALID) { InputGradient = new CArrayFloat(); if(CheckPointer(InputGradient) == POINTER_INVALID) ReturnFalse; } else InputGradient.Clear(); //--- int total = OutputLayer.Total(); CNeuron *temp; CArrayFloat *MemoryGradient = new CArrayFloat(); CNeuron *gate; CConnection *con; //--- if(nextLayer != OutputLayer) for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(temp.sumDOW(nextLayer)); } //--- Calculated memory and output gate gradients if(CheckPointer(MemoryGradient) == POINTER_INVALID) ReturnFalse; if(!MemoryGradient.Reserve(total)) ReturnFalse; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; float value = temp.getGradient() * gate.getOutputVal(); value = TanhFunctionDerivative(Memory.At(i)) * value; if(i >= MemoryGradient.Total()) { if(!MemoryGradient.Add(value)) ReturnFalse; } else { value = MemoryGradient.At(i) + value; if(!MemoryGradient.Update(i, value)) ReturnFalse; } gate.setGradient(gate.getOutputVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) / gate.getOutputVal() : 0); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getOutputVal()) : 0); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getOutputVal() != 0 && value != 0 ? value * temp.getOutputVal()*SigmoidFunctionDerivative(gate.getOutputVal()) : 0); temp.setGradient(temp.getOutputVal() != 0 && value != 0 ? value * gate.getOutputVal()*TanhFunctionDerivative(temp.getOutputVal()) : 0); } //--- Calculated input gradients int total_inp = temp.getConnections().Total(); for(int n = 0; n < total_inp; n++) { float value = 0; for(int i = 0; i < total; i++) { temp = ForgetGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = InputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = OutputGate.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; //--- temp = NewContent.At(i); con = temp.getConnections().At(n); value += temp.getGradient() * con.weight; } if(InputGradient.Total() >= n) { if(!InputGradient.Add(value)) ReturnFalse; } else if(!InputGradient.Update(n, value)) ReturnFalse; } //--- Calculated gradients for prev. state int shift = total_inp - total; for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; temp.setGradient(InputGradient.At(shift + i)); } //--- Calculated memory and output gate gradients for(int i = 0; i < total; i++) { temp = OutputLayer.At(i); gate = OutputGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; float value = temp.getGradient() * gate.getPrevVal(); value = MemoryGradient.At(i) + TanhFunctionDerivative(PrevMemory.At(i)) * value; if(!MemoryGradient.Update(i, value)) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && temp.getGradient() != 0 ? temp.getGradient()*temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) / gate.getPrevVal() : 0)); //--- Calcculated gates and new content gradients gate = ForgetGate.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); gate = InputGate.At(i); temp = NewContent.At(i); if(CheckPointer(gate) == POINTER_INVALID) ReturnFalse; gate.setGradient(gate.getGradient() + (gate.getPrevVal() != 0 && value != 0 ? value * temp.getPrevVal()*SigmoidFunctionDerivative(gate.getPrevVal()) : 0)); temp.setGradient(temp.getGradient() + (temp.getPrevVal() != 0 && value != 0 ? value * gate.getPrevVal()*TanhFunctionDerivative(temp.getPrevVal()) : 0)); } //--- DeleteObj(MemoryGradient); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *&prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID || CheckPointer(Input) == POINTER_INVALID) ReturnFalse; //--- if(!updateInputWeights(ForgetGate, Input) || !updateInputWeights(InputGate, Input) || !updateInputWeights(OutputGate, Input) || !updateInputWeights(NewContent, Input)) { ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::updateInputWeights(CLayer *gate, CArrayFloat *input_data) { if(CheckPointer(gate) == POINTER_INVALID || CheckPointer(input_data) == POINTER_INVALID) ReturnFalse; CNeuronBase *neuron; CConnection *con; int total_n = gate.Total(); int total_data = input_data.Total(); float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); for(int n = 0; n < total_n; n++) { neuron = gate.At(n); if(CheckPointer(neuron) == POINTER_INVALID) ReturnFalse; for(int i = 0; i < total_data; i++) { con = neuron.getConnections().At(i); if(CheckPointer(con) == POINTER_INVALID) ReturnFalse; float data = input_data.At(i); float g = neuron.getGradient(); if(optimization == SGD) con.weight += con.deltaWeight = (g != 0 && data != 0 ? lr * g * (data != DBL_MAX ? data : 1) : 0) + alpha * con.deltaWeight; else { con.mt = b1 * con.mt + (1 - b1) * g; con.vt = (float)(b2 * con.vt + (1 - b2) * pow(g, 2) + 0.00000001); con.weight += con.deltaWeight = (float)(lt * con.mt / sqrt(con.vt)); t++; } } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CNeuronBase *prevNeuron, uint index) { if(CheckPointer(prevNeuron) == POINTER_INVALID || CheckPointer(InputGradient) == POINTER_INVALID || InputGradient.Total() <= (int)index) ReturnFalse; //--- prevNeuron.setGradient(InputGradient.At(index)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::calcInputGradients(CLayer *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- int total = prevLayer.Total(); if(total <= 0) ReturnFalse; CNeuronBase *neuron; bool result = true; for(int i = 0; (i < total && result); i++) { neuron = prevLayer.At(i); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } result = calcInputGradients(neuron, i); } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Save(const int file_handle) { if(!CNeuronProof::Save(file_handle)) ReturnFalse; if(!ForgetGate.Save(file_handle)) ReturnFalse; if(!InputGate.Save(file_handle)) ReturnFalse; if(!OutputGate.Save(file_handle)) ReturnFalse; if(!NewContent.Save(file_handle)) ReturnFalse; if(!Memory.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTM::Load(const int file_handle) { if(!CNeuronProof::Load(file_handle)) ReturnFalse; if(!ForgetGate.Load(file_handle)) ReturnFalse; if(!InputGate.Load(file_handle)) ReturnFalse; if(!OutputGate.Load(file_handle)) ReturnFalse; if(!NewContent.Load(file_handle)) ReturnFalse; if(!Memory.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuronBase::activationFunction(float x) { switch(activation) { case TANH: return TanhFunction(x); break; case SIGMOID: return SigmoidFunction(x); break; } //--- return x; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuronBase::activationFunctionDerivative(float x) { switch(activation) { case TANH: return TanhFunctionDerivative(x); break; case SIGMOID: return SigmoidFunctionDerivative(x); break; } //--- return 1; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags) { int result = INVALID_HANDLE; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result == INVALID_HANDLE) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags)) return INVALID_HANDLE; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ int COpenCLMy::AddBuffer(uint size_in_bytes, const uint flags) { int result = INVALID_HANDLE; for(int i = 0; i < m_buffers_total; i++) { if(m_buffers[i] != INVALID_HANDLE) continue; result = i; break; } //--- if(result < 0) { if(ArrayResize(m_buffers, m_buffers_total + 1) > 0) { m_buffers_total = ArraySize(m_buffers); result = m_buffers_total - 1; m_buffers[result] = INVALID_HANDLE; } else return result; } //--- if(!BufferCreate(result, size_in_bytes, flags)) return INVALID_HANDLE; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayer::CLayer(uint outputs = 0, int handle = -1, COpenCLMy *opencl = NULL) { iOutputs = outputs; iFileHandle = handle; OpenCL = opencl; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::Load(const int file_handle) { iFileHandle = file_handle; if(!CArrayObj::Load(file_handle)) { for(int i = 0; i < int(m_data.Size()); i++) { DeleteObj(m_data[i]); } ReturnFalse; } if(CheckPointer(m_data[0]) == POINTER_INVALID) ReturnFalse; //--- CNeuronBaseOCL *ocl = NULL; CNeuronBase *cpu = NULL; if(m_data[0].Type() >= defNeuronBaseOCL) { ocl = m_data[0]; iOutputs = ocl.getConnections(); } else { cpu = m_data[0]; iOutputs = cpu.getConnections().Total(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CLayer::SetOpenCL(COpenCLMy *obj) { if(OpenCL == obj) return; DeleteObj(OpenCL); OpenCL = obj; //--- if(!obj || Total() <= 0) return; for(int i = 0; i < Total(); i++) { CNeuronBaseOCL *neuron = m_data[i]; if(!neuron) continue; neuron.SetOpenCL(obj); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CLayer::TrainMode(bool flag) { for(int i = 0; i < Total(); i++) { CNeuronBaseOCL *neuron = m_data[i]; if(!neuron) continue; neuron.TrainMode(flag); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNet::~CNet(void) { DeleteObj(layers); if(CheckPointer(opencl) != POINTER_INVALID) { opencl.Shutdown(); delete opencl; } } //+------------------------------------------------------------------+ ///\class CBufferFloat /// Class of OpenCL buffer data. Used for transfer data from CPU to GPU and back. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CBufferFloat : public CArrayFloat { protected: COpenCLMy *OpenCL; ///< Object for working with OpenCL int m_myIndex; ///< Index of buffer public: /** Constructor */ CBufferFloat(void); /** Destructor */~CBufferFloat(void); //--- virtual bool BufferInit(uint count, float value); ///< Method for buffer initialization @param[in] count Number of items @param[in] value Initialization value virtual bool BufferInitLike(CBufferFloat* master); ///< Method for buffer initialization @param[in] count Number of items @param[in] value Initialization value virtual bool BufferCreate(COpenCL *opencl); ///< Method for creating new buffer @param[in] opencl Pointer to #COpenCLMy object virtual bool BufferFree(void); ///< Method for deleting buffer from GPU virtual bool BufferRead(void); ///< Method for reading buffer data from GPU virtual bool BufferWrite(void); ///< Method for writing buffer data to GPU virtual bool BufferSet(int index) { if(!OpenCL.BufferFree(m_myIndex)) ReturnFalse; m_myIndex = index; return true; } ///< Change buffer index number @param index New index template int GetData(vector &values); ///< Read data from buffer to vector @param[out] values Vector to read data template int GetData(T &values[]); ///< Read data from buffer to array @param[out] values Array to read data template int GetData(matrix &values); ///< Read data from buffer to vector @param[out] values Vector to read data virtual int GetData(CArrayFloat *values); ///< Read data from buffer to array @param[out] values Array to read data virtual int GetIndex(void) const { return m_myIndex; } ///< Get buffer index @return Index virtual float Maximum(void) { return m_data[Argmax()];} virtual float Minimum(void) { return m_data[Argmin()];} virtual int Argmax(void) { return ArrayMaximum(m_data, 0, m_data_total);} virtual int Argmin(void) { return ArrayMinimum(m_data, 0, m_data_total);} //--- template bool AssignArray(const T &src[]); virtual bool AssignArray(const matrix &src); virtual bool AssignArray(const vector &src); virtual bool AssignArray(const CArrayFloat *obj); virtual bool AssignArray(const CBufferFloat *obj) { return AssignArray((CArrayFloat*)obj); } virtual bool AddArray(const CBufferFloat *src) { return CArrayFloat::AddArray((CArrayFloat*)src); } virtual bool AddArray(const vector &src); template bool AddArray(const T &src[]); //--- virtual bool Fill(const float value); virtual bool Fill(const CBufferFloat *obj); template bool Fill(const matrix &obj); template bool Fill(const vector &obj); virtual bool Random(const float min_value, const float max_value); //--- virtual int Type(void) const { return defBufferDouble; }///< Identificator of class.@return Type of class virtual void BufferToCSV(const string file_name); ///< Save buffer data to CSV file @param[in] file_name File name to write data //--- methods for working with files virtual bool Save(const int file_handle); virtual bool Load(const int file_handle); //--- virtual COpenCL *GetOpenCL(void) const { return OpenCL; } bool operator=(const CBufferFloat *obj) {return AssignArray(obj);} float operator[](const int index) const { if(index < 0) return(At(m_data_total + index)); return(At(index)); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat::CBufferFloat(void) : m_myIndex(INVALID_HANDLE) { OpenCL = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat::~CBufferFloat(void) { if(CheckPointer(OpenCL) != POINTER_INVALID && m_myIndex >= 0) { if(OpenCL.BufferFree(m_myIndex)) { m_myIndex = -1; OpenCL = NULL; } } Shutdown(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferCreate(COpenCL *opencl) { if(CheckPointer(opencl) == POINTER_INVALID) ReturnFalse; //--- if(opencl == OpenCL && m_myIndex >= 0) if(BufferWrite()) return true; //--- if(CheckPointer(OpenCL) != POINTER_INVALID && m_myIndex >= 0) { OpenCL.BufferFree(m_myIndex); m_myIndex = -1; OpenCL = NULL; } //--- if((m_myIndex = ((COpenCLMy*)opencl).AddBufferFromArray(m_data, 0, m_data_total, CL_MEM_READ_WRITE)) < 0) ReturnFalse; OpenCL = opencl; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferFree(void) { if(CheckPointer(OpenCL) != POINTER_INVALID && m_myIndex >= 0) if(OpenCL.BufferFree(m_myIndex)) { m_myIndex = INVALID_HANDLE; OpenCL = NULL; return true; } //--- return false; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferRead(void) { if(CheckPointer(OpenCL) == POINTER_INVALID || m_myIndex < 0) ReturnFalse; //--- if(!OpenCL.BufferRead(m_myIndex, m_data, 0, 0, m_data_total)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferWrite(void) { if(CheckPointer(OpenCL) == POINTER_INVALID || m_myIndex < 0) ReturnFalse; //--- return OpenCL.BufferWrite(m_myIndex, m_data, 0, 0, m_data_total); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferInit(uint count, float value) { if(m_data_total != count && !!OpenCL) BufferFree(); if(m_data_max < (int)count && !Reserve((int)count - m_data_total)) ReturnFalse; m_data_total = (int)fmin(ArrayInitialize(m_data, value), count); //--- return m_data_total == count; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int CBufferFloat::GetData(T &values[]) { if(!!OpenCL && !BufferRead()) ReturnFalse; return ArrayCopy(values, m_data, 0, 0, m_data_total); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int CBufferFloat::GetData(vector &values) { if(!!OpenCL && !BufferRead()) ReturnFalse; values.Assign(m_data); if(!values.Resize(m_data_total)) ReturnFalse; return (int)values.Size(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template int CBufferFloat::GetData(matrix &values) { if(!!OpenCL && !BufferRead()) ReturnFalse; ulong r = values.Rows(); ulong c = values.Cols(); for(ulong ir = 0; ir < r; ir++) { for(ulong ic = 0; ic < c; ic++) { int pos = int(ir * c + ic); if(pos >= m_data_total) break; values[ir, ic] = (T)m_data[pos]; } } return MathMin(int(r * c), m_data_total); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ int CBufferFloat::GetData(CArrayFloat *values) { if(!values) return -1; if(!BufferRead()) return -1; if(!values.AssignArray(GetPointer(this))) return -1; return m_data_total; } //+------------------------------------------------------------------+ //| Assignment (copying) of another array | //+------------------------------------------------------------------+ template bool CBufferFloat::AssignArray(const T &src[]) { int num = ArraySize(src); //--- check/reserve elements of array Clear(); if(m_data_max < num) { if(!Reserve(num)) return(false); } else Resize(num); //--- copy array for(int i = 0; i < num; i++) { m_data[i] = float(src[i]); m_data_total++; } m_sort_mode = -1; //--- successful return(true); } //+------------------------------------------------------------------+ //| Assignment (copying) of another array | //+------------------------------------------------------------------+ bool CBufferFloat::AssignArray(const CArrayFloat *obj) { if(!obj) ReturnFalse; int num = obj.Total(); //--- check/reserve elements of array Clear(); if(m_data_max < num) { if(!Reserve(num)) return(false); } else Resize(num); //--- copy array for(int i = 0; i < num; i++) { m_data[i] = (float)obj.At(i); m_data_total++; } m_sort_mode = -1; //--- successful return(true); } //+------------------------------------------------------------------+ //| Assignment (copying) of another array | //+------------------------------------------------------------------+ bool CBufferFloat::AssignArray(const matrix &src) { int num = (int)(src.Rows() * src.Cols()); //--- check/reserve elements of array Clear(); if(m_data_max < num) { if(!Reserve(num)) return(false); } else Resize(num); //--- copy array for(int i = 0; i < num; i++) { m_data[i] = src.Flat(i); m_data_total++; } m_sort_mode = -1; //--- //--- successful return(true); } //+------------------------------------------------------------------+ //| Assignment (copying) of another array | //+------------------------------------------------------------------+ bool CBufferFloat::AssignArray(const vector &src) { int num = (int)(src.Size()); //--- check/reserve elements of array Clear(); if(m_data_max < num) { if(!Reserve(num)) return(false); } else Resize(num); //--- copy array for(int i = 0; i < num; i++) { m_data[i] = src[i]; m_data_total++; } m_sort_mode = -1; //--- if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- successful return(true); } //+------------------------------------------------------------------+ //| Adding (copying) of another array | //+------------------------------------------------------------------+ bool CBufferFloat::AddArray(const vector &src) { int num = (int)(src.Size()); //--- check/reserve elements of array if(m_data_max < num + m_data_total) { if(!Reserve(num + m_data_total)) return(false); } else Resize(num + m_data_total); //--- copy array for(int i = 0; i < num; i++) m_data[m_data_total + i] = src[i]; m_data_total += num; m_sort_mode = -1; //--- successful return(true); } //+------------------------------------------------------------------+ //| Adding (copying) of another array | //+------------------------------------------------------------------+ template bool CBufferFloat::AddArray(const T &src[]) { int num = ArraySize(src); //--- check/reserve elements of array if(m_data_max < num + m_data_total) { if(!Reserve(num + m_data_total)) return(false); } else Resize(num + m_data_total); //--- copy array for(int i = 0; i < num; i++) m_data[m_data_total + i] = (float)src[i]; m_data_total += num; m_sort_mode = -1; //--- successful return(true); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::Save(const int file_handle) { if(CheckPointer(OpenCL) != POINTER_INVALID && m_myIndex >= 0) if(!BufferRead()) ReturnFalse; //--- return CArrayFloat::Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::Load(const int file_handle) { if(!CArrayFloat::Load(file_handle)) ReturnFalse; if(m_data_total == 0 || !OpenCL) return true; //--- for(int i = 0; i < m_data_total; i++) { switch(MathClassify(m_data[i])) { case FP_INFINITE: case FP_NAN: m_data[i] = 0; break; } } COpenCLMy *temp = OpenCL; BufferFree(); return BufferCreate(temp); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::Fill(const float value) { ArrayFill(m_data, 0, m_data_total, value); if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::Fill(const CBufferFloat *obj) { if(!obj) ReturnFalse; //--- int total = obj.Total(); for(int i = 0; i < fmin(total, m_data_total); i++) m_data[i] = obj.At(i); for(int i = total; i < m_data_total; i++) m_data[i] = 0; if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template bool CBufferFloat::Fill(const matrix &matr) { //--- int total = int(matr.Rows() * matr.Cols()); for(int i = 0; i < fmin(total, m_data_total); i++) m_data[i] = float(matr.Flat(i)); for(int i = total; i < m_data_total; i++) m_data[i] = 0; if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ template bool CBufferFloat::Fill(const vector &vec) { //--- int total = int(vec.Size()); for(int i = 0; i < fmin(total, m_data_total); i++) m_data[i] = float(vec[i]); for(int i = total; i < m_data_total; i++) m_data[i] = 0; if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::Random(const float min_value, const float max_value) { //--- const float delta = max_value - min_value; for(int i = 0; i < m_data_total; i++) m_data[i] = float(::MathRand()) / 32767.0f * delta + min_value; if(!!OpenCL) { if(!BufferWrite()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ ///\ingroup neuron_base ///\class CNeuronBaseOCL ///\brief The base class of neuron for GPU calculation. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronBaseOCL : public CObject { protected: bool bTrain; ///< Training Mode Flag COpenCLMy *OpenCL; ///< Object for working with OpenCL CBufferFloat *Output; ///< Buffer of Output tensor CBufferFloat *PrevOutput; ///< Buffer of previous iteration Output tensor CBufferFloat *Weights; ///< Buffer of weights matrix CBufferFloat *DeltaWeights; ///< Buffer of last delta weights matrix (#SGD) CBufferFloat *Gradient; ///< Buffer of gradient tensor CBufferFloat *FirstMomentum; ///< Buffer of first momentum matrix (#ADAM) CBufferFloat *SecondMomentum; ///< Buffer of second momentum matrix (#ADAM) vector prev_output; //--- //const float lr; float alpha; ///< Multiplier to momentum in #SGD optimization uint iBatch; ///< Batch size used in LS optimization uint t; ///< Count of iterations //--- int m_myIndex; ///< Index of neuron in layer ENUM_ACTIVATION activation; ///< Activation type (#ENUM_ACTIVATION) ENUM_OPTIMIZATION optimization; ///< Optimization method (#ENUM_OPTIMIZATION) //--- ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { return feedForward(NeuronOCL); } ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { return updateInputWeights(NeuronOCL); } ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { return calcInputGradients(NeuronOCL); } virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { ReturnFalse; } virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. virtual float GenerateWeight(void); virtual bool WeightsUpdateAdam(CNeuronBaseOCL *source, float tau); virtual bool SumAndNormilize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f); virtual bool Normilize(CBufferFloat *tensor, int dimension); virtual bool Different(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f); virtual bool DifferentGrad(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f); //--- virtual bool IdentSum(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f); virtual bool IdentDifferent(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f); virtual bool IdentDifferentGrad(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f); //--- virtual bool SumVecMatrix(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f); virtual bool SumVecMatrixGrad(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f); virtual bool MatMul(const CBufferFloat *matr1, const CBufferFloat *matr2, CBufferFloat *result, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true); virtual bool MatMulGrad(const CBufferFloat *matr1, CBufferFloat *matr1_gr, const CBufferFloat *matr2, CBufferFloat *matr2_gr, const CBufferFloat *result_gr, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true); virtual bool DiagMatMul(const CBufferFloat *diag, const CBufferFloat *matr, CBufferFloat *result, const int rows, const int cols, const int variables = 1, const int activation = None); virtual bool DiagMatMulGrad(const CBufferFloat *diag, CBufferFloat *diag_gr, const CBufferFloat *matr, CBufferFloat *matr_gr, const CBufferFloat *result_gr, const int rows, const int cols, const int variables = 1); virtual bool SparseMatMul(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols); virtual bool SparseMatMulGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols); virtual bool SparseConcatenate(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, const CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols); virtual bool SparseConcatenateGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols); virtual bool ScalarToVector(CBufferFloat *scalar, CBufferFloat *vector_in, CBufferFloat *vector_out, int dimension); virtual bool ScalarToVectorGrad(CBufferFloat *scalar, CBufferFloat *scalar_gr, CBufferFloat *vector_in, CBufferFloat *vector_in_gr, CBufferFloat *vector_out_gr, int dimension); ///< \brief Method sum and normilize 2 tensors by calling 2 kernels ::SumMatrix() and ::Normalize(). virtual bool LoadInsideLayer(int file_handle, CNeuronBaseOCL *neuron); virtual bool Activation(CBufferFloat *inputs, CBufferFloat *outputs, int activat); virtual bool DeActivation(CBufferFloat *inputs, CBufferFloat *inputs_gr, CBufferFloat *output_gr, int activat); //--- virtual bool Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *output, int window1, int window2, int count); virtual bool Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *output, int window1, int window2, int window3, int count); virtual bool Concat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *input4, CBufferFloat *output, int window1, int window2, int window3, int window4, int count); virtual bool Concat(int input1, int input2, int input3, int input4, int output, int window1, int window2, int window3, int window4, int count); virtual bool DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *output, int window1, int window2, int count); virtual bool DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *output, int window1, int window2, int window3, int count); virtual bool DeConcat(CBufferFloat *input1, CBufferFloat *input2, CBufferFloat *input3, CBufferFloat *input4, CBufferFloat *output, int window1, int window2, int window3, int window4, int count); //--- virtual bool ElementMult(const CBufferFloat *input1, const CBufferFloat *input2, CBufferFloat *output); virtual bool ElementMultGrad(const CBufferFloat *input1, CBufferFloat *input1_gr, const CBufferFloat *input2, CBufferFloat *input2_gr, const CBufferFloat *output, const int activation1, const int activation2); virtual bool GateElementMult(const CBufferFloat *input1, const CBufferFloat *input2, const CBufferFloat *gate, CBufferFloat *output); virtual bool GateElementMultGrad(const CBufferFloat *input1, CBufferFloat *input1_gr, const CBufferFloat *input2, CBufferFloat *input2_gr, const CBufferFloat *gate, CBufferFloat *gate_gr, const CBufferFloat *output, const int activation1, const int activation2, const int activation_gate); int RND(int total) { xor128; return (int)((float)(total - 1) / UINT_MAX * rnd_w); } ///< Generates a random neuron position to turn off //--- virtual bool DiversityLoss(CNeuronBaseOCL *neuron, const int units, const int dimension, const bool add = false); //--- virtual bool SwapBuffers(CBufferFloat *&buffer1, CBufferFloat *&buffer2, bool check_size = true); //--- virtual bool DilatedCorrelation(CNeuronBaseOCL* feature, CNeuronBaseOCL* correlations, uint shifts_index, uint dimension, uint units); virtual bool DilatedCorrelationGrad(CNeuronBaseOCL* feature, CNeuronBaseOCL* correlations, uint shifts_index, uint dimension, uint units); virtual bool DilatedDifference(CNeuronBaseOCL* feature, CNeuronBaseOCL* difference, uint shifts_index, uint dimension, uint units); virtual bool DilatedDifferenceGrad(CNeuronBaseOCL* feature, CNeuronBaseOCL* difference, uint shifts_index, uint dimension, uint units); virtual bool PerturbedMatrix(CNeuronBaseOCL* inputs, CNeuronBaseOCL* perturb, CNeuronBaseOCL* outputs, float perturb_mult, uint dimension, uint units); virtual bool PerturbedMatrixGrad(CNeuronBaseOCL* inputs, CNeuronBaseOCL* perturb, CNeuronBaseOCL* outputs, float perturb_mult, uint dimension, uint units); virtual bool LinearUpsample(CNeuronBaseOCL* data, CNeuronBaseOCL* upsample, uint upsampl_mult, uint units, uint variables); virtual bool LinearUpsampleGrad(CNeuronBaseOCL* data, CNeuronBaseOCL* upsample, uint upsampl_mult, uint units, uint variables); //--- virtual int CreateShifts(const uint max_displacement, int &count); public: /** Constructor */ CNeuronBaseOCL(void); /** Destructor */~CNeuronBaseOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; } ///< Set the type of activation function (#ENUM_ACTIVATION) //--- virtual int getOutputIndex(void) const { return Output.GetIndex(); } ///< Get index of output buffer @return Index virtual int getPrevOutIndex(void) const { return PrevOutput.GetIndex(); } ///< Get index of previous iteration output buffer @return Index virtual int getGradientIndex(void) const { return Gradient.GetIndex(); } ///< Get index of gradient buffer @return Index virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true); virtual bool SetOutput(CBufferFloat *buffer, bool delete_prev = true); virtual bool SetGradientIndex(int index) { return Gradient.BufferSet(index); } ///< Method for change index of gradient buffer.@param[in] New index of buffer virtual int getWeightsIndex(void) const { if(!Weights) return INVALID_HANDLE; return Weights.GetIndex(); } ///< Get index of weights matrix buffer @return Index virtual int getDeltaWeightsIndex(void) const { return DeltaWeights.GetIndex(); } ///< Get index of delta weights matrix buffer (SGD)@return Index virtual int getFirstMomentumIndex(void) const { return FirstMomentum.GetIndex(); } ///< Get index of first momentum matrix buffer (Adam)@return Index virtual int getSecondMomentumIndex(void) const { return SecondMomentum.GetIndex();} ///< Get index of Second momentum matrix buffer (Adam)@return Index virtual int getWeightsSAMIndex(void) const { return -1; } //--- virtual int getOutputVal(float &values[]) { return Output.GetData(values); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(CArrayFloat *values) { return Output.GetData(values); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(vector &values) { return Output.GetData(values); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getPrevVal(float &values[]) { return PrevOutput.GetData(values); } ///< Get values of previous iteration output buffer @param[out] values Array of data @return number of items virtual int getGradient(float &values[]) { return Gradient.GetData(values); } ///< Get values of gradient buffer @param[out] values Array of data @return number of items virtual int getGradient(vector &values) { return Gradient.GetData(values); } ///< Get values of gradient buffer @param[out] values Array of data @return number of items virtual bool calcAlphaGradients(CNeuronBaseOCL *NeuronOCL) { return true; } virtual bool GetAlphaLogProbs(vector &log_probs) { ReturnFalse; } virtual bool ReCalcLogProbs(void) { ReturnFalse; } virtual CBufferFloat *getOutput(void) const { return Output; } ///< Get pointer of output buffer @return Pointer to object virtual CBufferFloat *getGradient(void) const { return Gradient; } ///< Get pointer of gradient buffer @return Pointer to object virtual CBufferFloat *getPrevOutput(void) const { return PrevOutput; } ///< Get pointer of gradient buffer @return Pointer to object virtual int getWeights(float &values[]) { return Weights.GetData(values); } ///< Get values of weights matrix buffer @param[out] values Array of data @return number of items virtual CBufferFloat *getWeights(void) { return Weights; } ///< Get pointer of gradient buffer @return Pointer to object virtual int Neurons(void) const { return Output.Total(); } ///< Get number of neurons in layer @return Number of neurons virtual int Activation(void) const { return (int)activation; } ///< Get type of activation function @return Type (#ENUM_ACTIVATION) virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID ? Weights.Total() / (Output.Total() + 1) : 0); } ///< Get number of connections 1 neuron to next layer @return Number of connections virtual ENUM_OPTIMIZATION Optimization(void) { return optimization; } //--- virtual bool FeedForward(CObject *SourceObject, CBufferFloat *SecondInput = NULL); ///< Dispatch method for defining the subroutine for feed forward process. @param SourceObject Pointer to the previous layer. virtual bool UpdateInputWeights(CObject *SourceObject, CBufferFloat *SecondInput = NULL); ///< Dispatch method for defining the subroutine for updating weights.@param SourceObject Pointer to previos layer. ///\ingroup neuron_base_gr ///@{ virtual bool CalcHiddenGradients(CObject *TargetObject, CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, ENUM_ACTIVATION SecondActivation = None); ///< Dispatch method for defining the subroutine for transferring the gradient to the previous layer. @param TargetObject Pointer to the next layer. virtual bool calcOutputGradients(CBufferFloat *Target, float &error); ///< Method of output gradients calculation by calling kernel ::CalcOutputGradient().@param Target Traget value virtual bool calcAlignmentGradient(const CNeuronBaseOCL *neuron, const bool add = false); ///@} //--- virtual bool Save(const int file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(const int file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronBaseOCL; }///< Identificator of class.@return Type of class virtual void TrainMode(bool flag) { bTrain = flag; } ///< Set Training Mode Flag virtual bool TrainMode(void) { return bTrain; }///< Get Training Mode Flag virtual CLayerDescription* GetLayerInfo(void); virtual bool numOutputs(const uint outputs, ENUM_OPTIMIZATION optimization_type); virtual void SetOpenCL(COpenCLMy *obj); virtual CObject* AsObject(void) { return GetPointer(this); } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); //--- virtual bool Clear(void) { return !(!Output || !Output.Fill(0) || !Gradient || !Gradient.Fill(0)); } //--- virtual bool SwapOutputs(void) { return SwapBuffers(Output, PrevOutput, true); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::CNeuronBaseOCL(void) : alpha(momentum), activation(None), optimization(ADAM), t(1), bTrain(true) { OpenCL = NULL; Output = new CBufferFloat(); PrevOutput = new CBufferFloat(); Weights = new CBufferFloat(); DeltaWeights = new CBufferFloat(); Gradient = new CBufferFloat(); FirstMomentum = new CBufferFloat(); SecondMomentum = new CBufferFloat(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL::~CNeuronBaseOCL(void) { DeleteObj(Output); DeleteObj(PrevOutput); DeleteObj(Weights); DeleteObj(DeltaWeights); DeleteObj(Gradient); DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); OpenCL = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) ReturnFalse; OpenCL = open_cl; optimization = optimization_type; iBatch = batch; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferFloat(); if(CheckPointer(Output) == POINTER_INVALID) ReturnFalse; } if(!Output.BufferInit(numNeurons, 1.0)) ReturnFalse; if(!Output.BufferCreate(OpenCL)) ReturnFalse; prev_output.Init(numNeurons); //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferFloat(); if(CheckPointer(PrevOutput) == POINTER_INVALID) ReturnFalse; } if(!PrevOutput.BufferInit(numNeurons, 1.0)) ReturnFalse; if(!PrevOutput.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferFloat(); if(CheckPointer(Gradient) == POINTER_INVALID) ReturnFalse; } if(!Gradient.BufferInit(numNeurons, 0.0)) ReturnFalse; if(!Gradient.BufferCreate(OpenCL)) ReturnFalse; //--- if(numOutputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) ReturnFalse; } int count = (int)((numNeurons + 1) * numOutputs); if(!Weights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(numNeurons + 1)); for(int i = 0; i < count; i++) { if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!Weights.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferFloat(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) ReturnFalse; } if(!DeltaWeights.BufferInit(count, 0)) ReturnFalse; if(!DeltaWeights.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); } else { DeleteObj(DeltaWeights); //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!FirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentum.BufferInit((optimization == ADAM_MINI ? numOutputs : count), 0)) ReturnFalse; if(!SecondMomentum.BufferCreate(OpenCL)) ReturnFalse; } } else { DeleteObj(Weights); DeleteObj(DeltaWeights); DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::FeedForward(CObject *SourceObject, CBufferFloat *SecondInput = NULL) { if(CheckPointer(SourceObject) == POINTER_INVALID) ReturnFalse; //--- CNeuronBaseOCL *temp = SourceObject; return feedForward(temp, SecondInput); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- const int kernel = def_k_FeedForward; setBuffer(kernel, def_k_ff_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(kernel, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ff_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ff_inputs, NeuronOCL.Neurons()) setArgument(kernel, def_k_ff_activation, (int)activation) //--- uint global_work_offset[2] = { 0, 0 }; uint global_work_size[2] = { Output.Total(), (uint)MathMin((NeuronOCL.Neurons() + 3) / 4, OpenCL.GetMaxLocalSize(1)) }; uint local_work_size[2] = { 1, global_work_size[1] }; kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = (uint)MathMin((NeuronOCL.Neurons() + 3) / 4, OpenCL.GetMaxLocalSize(1)); uint local_work_size[2] = { 1, global_work_size[1] };; setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_w, getWeightsIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_g, NeuronOCL.getGradientIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_o, getOutputIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_ig, getGradientIndex()) setArgument(def_k_CalcHiddenGradient, def_k_chg_outputs, NeuronOCL.Neurons()) setArgument(def_k_CalcHiddenGradient, def_k_chg_activation, Activation()) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecuteLoc(def_k_CalcHiddenGradient, global_work_offset, global_work_size, local_work_size) //if(!Gradient.BufferRead()) //ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = NeuronOCL.Neurons(); setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_g, getGradientIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(def_k_CalcHiddenGradient, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(def_k_CalcHiddenGradient, def_k_chg_outputs, Neurons()) setArgument(def_k_CalcHiddenGradient, def_k_chg_activation, NeuronOCL.Activation()) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_CalcHiddenGradient, global_work_offset, global_work_size) //if(!NeuronOCL.Gradient.BufferRead()) // ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcOutputGradients(CBufferFloat *target, float &error) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(target) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1]; if(error <= 0) error = 1; global_work_size[0] = Neurons(); if(target.GetOpenCL() != OpenCL) { if(!Gradient.Fill(target)) ReturnFalse; setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_t, getGradientIndex()); } else { setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_t, target.GetIndex()); } setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_o, getOutputIndex()); setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_ig, getGradientIndex()); setArgument(def_k_CalcOutputGradient, def_k_cog_activation, (int)activation); setArgument(def_k_CalcOutputGradient, def_k_cog_error, error); ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_CalcOutputGradient, global_work_offset, global_work_size) #ifdef _DEBUG Gradient.BufferRead(); #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::calcAlignmentGradient(const CNeuronBaseOCL *neuron, const bool add = false) { if(!OpenCL || !neuron || neuron.Activation() != activation) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; int kernel = def_k_CalcAlignmentGradient; ResetLastError(); setBuffer(kernel, def_k_aliggr_matrix_o1, getOutputIndex()) setBuffer(kernel, def_k_aliggr_matrix_o2, neuron.getOutputIndex()) setBuffer(kernel, def_k_aliggr_matrix_g1, getGradientIndex()) setBuffer(kernel, def_k_aliggr_matrix_g2, neuron.getGradientIndex()) setArgument(kernel, def_k_aliggr_activation, Activation()) setArgument(kernel, def_k_aliggr_add, (int)add) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2], local_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons() + 1; uint rest = 0; float lt = lr; switch(NeuronOCL.Optimization()) { case SGD: setBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsMomentum, def_k_uwm_matrix_dw, NeuronOCL.getDeltaWeightsIndex()) setArgument(def_k_UpdateWeightsMomentum, def_k_uwm_inputs, NeuronOCL.Neurons()) setArgument(def_k_UpdateWeightsMomentum, def_k_uwm_learning_rates, lr) setArgument(def_k_UpdateWeightsMomentum, def_k_uwm_momentum, alpha) ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsMomentum, global_work_offset, global_work_size) break; case ADAM: setBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_m, NeuronOCL.getFirstMomentumIndex()) setBuffer(def_k_UpdateWeightsAdam, def_k_uwa_matrix_v, NeuronOCL.getSecondMomentumIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_UpdateWeightsAdam, def_k_uwa_inputs, NeuronOCL.Neurons()) setArgument(def_k_UpdateWeightsAdam, def_k_uwa_l, lt) setArgument(def_k_UpdateWeightsAdam, def_k_uwa_b1, b1) setArgument(def_k_UpdateWeightsAdam, def_k_uwa_b2, b2) ////Comment(com+"\n UpdateWeightsAdam"); ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsAdam, global_work_offset, global_work_size) t++; break; case LS: setBuffer(def_k_UpdateWeightsLS, def_k_uwls_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(def_k_UpdateWeightsLS, def_k_uwls_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsLS, def_k_uwls_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsLS, def_k_uwls_matrix_xg, NeuronOCL.getFirstMomentumIndex()) setBuffer(def_k_UpdateWeightsLS, def_k_uwls_matrix_xx, NeuronOCL.getSecondMomentumIndex()) setArgument(def_k_UpdateWeightsLS, def_k_uwls_inputs, NeuronOCL.Neurons()) setArgument(def_k_UpdateWeightsLS, def_k_uwls_l, lr) setArgument(def_k_UpdateWeightsLS, def_k_uwls_update, (int)(t >= iBatch)) rest = global_work_size[1] % 4; global_work_size[1] = (global_work_size[1] - rest) / 4 + (rest > 0 ? 1 : 0); ////Comment(com+"\n UpdateWeightsAdam"); ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsLS, global_work_offset, global_work_size) if(t >= iBatch) t = 0; else t++; break; case ADAM_MINI: setBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_m, NeuronOCL.getFirstMomentumIndex()) setBuffer(def_k_UpdateWeightsAdamMini, def_k_wuam_matrix_v, NeuronOCL.getSecondMomentumIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_l, lt) setArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_b1, b1) setArgument(def_k_UpdateWeightsAdamMini, def_k_wuam_b2, b2) global_work_size[0] = NeuronOCL.Neurons() + 1; global_work_size[1] = Neurons(); local_work_size[0] = global_work_size[0]; local_work_size[1] = 1; ////Comment(com+"\n UpdateWeightsAdam"); ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecuteLoc(def_k_UpdateWeightsAdamMini, global_work_offset, global_work_size, local_work_size) t++; break; default: ReturnFalse; break; } //--- { uint global_work_offset_n[] = {0}; uint global_work_size_n[] = {Neurons()}; setBuffer(def_k_NormilizeWeights, def_k_norm_buffer, NeuronOCL.getWeightsIndex()) setArgument(def_k_NormilizeWeights, def_k_norm_dimension, int(NeuronOCL.Neurons() + 1)) kernelExecute(def_k_NormilizeWeights, global_work_offset_n, global_work_size_n) } //--- return true;//NeuronOCL.Weights.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::CalcHiddenGradients(CObject *TargetObject, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { //--- if(CheckPointer(TargetObject) == POINTER_INVALID) ReturnFalse; //--- CNeuronBaseOCL *temp = TargetObject; //--- switch(TargetObject.Type()) { case defNeuronBaseOCL: return calcHiddenGradients(temp); case defNeuronMultiModels: return temp.CalcHiddenGradients(GetPointer(this)); default: return temp.calcInputGradients(GetPointer(this), SecondInput, SecondGradient, SecondActivation); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::UpdateInputWeights(CObject *SourceObject, CBufferFloat *SecondInput = NULL) { //--- if(CheckPointer(SourceObject) == POINTER_INVALID) ReturnFalse; //--- CNeuronBaseOCL *temp = SourceObject; if(!bTrain && !temp.TrainMode()) return true; if(Type() == defNeuronSoftMaxOCL) return true; //--- return updateInputWeights(temp, SecondInput); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Save(const int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; if(FileWriteInteger(file_handle, Type()) < INT_VALUE) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)activation, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)optimization, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iBatch, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)t, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)bTrain, INT_VALUE) < INT_VALUE) ReturnFalse; //--- if(CheckPointer(Output) == POINTER_INVALID || !Output.Save(file_handle)) ReturnFalse; if(CheckPointer(PrevOutput) == POINTER_INVALID || !PrevOutput.Save(file_handle)) ReturnFalse; if(CheckPointer(Gradient) == POINTER_INVALID || !Gradient.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Weights) == POINTER_INVALID) { FileWriteInteger(file_handle, 0); return true; } else FileWriteInteger(file_handle, 1); //--- if(CheckPointer(Weights) == POINTER_INVALID || !Weights.Save(file_handle)) ReturnFalse; if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID || !DeltaWeights.Save(file_handle)) ReturnFalse; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID || !FirstMomentum.Save(file_handle)) ReturnFalse; if(CheckPointer(SecondMomentum) == POINTER_INVALID || !SecondMomentum.Save(file_handle)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Load(const int file_handle) { if(file_handle == INVALID_HANDLE) ReturnFalse; //--- activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE); optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE); iBatch = (uint)FileReadInteger(file_handle, INT_VALUE); t = (uint)FileReadInteger(file_handle, INT_VALUE); bTrain = (bool)FileReadInteger(file_handle, INT_VALUE); if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferFloat(); if(CheckPointer(Output) == POINTER_INVALID) ReturnFalse; } if(Output.GetIndex() >= 0) Output.BufferFree(); if(!Output.Load(file_handle)) ReturnFalse; if(Output.Total() > 0 && !Output.BufferCreate(OpenCL)) ReturnFalse; prev_output.Init(Output.Total()); //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferFloat(); if(CheckPointer(PrevOutput) == POINTER_INVALID) ReturnFalse; } if(PrevOutput.GetIndex() >= 0) PrevOutput.BufferFree(); if(!PrevOutput.Load(file_handle)) ReturnFalse; if(PrevOutput.Total() > 0 && !PrevOutput.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferFloat(); if(CheckPointer(Gradient) == POINTER_INVALID) ReturnFalse; } if(Gradient.GetIndex() >= 0) Gradient.BufferFree(); if(!Gradient.Load(file_handle)) ReturnFalse; if(Gradient.Total() > 0 && !Gradient.BufferCreate(OpenCL)) ReturnFalse; //--- if(FileReadInteger(file_handle) == 0) { DeleteObj(Weights); DeleteObj(DeltaWeights); DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); return true; } //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) ReturnFalse; } if(Weights.GetIndex() >= 0) Weights.BufferFree(); if(!Weights.Load(file_handle)) ReturnFalse; if(Weights.Total() > 0 && !Weights.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferFloat(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) ReturnFalse; } if(DeltaWeights.GetIndex() >= 0) DeltaWeights.BufferFree(); if(!DeltaWeights.Load(file_handle)) ReturnFalse; if(DeltaWeights.Total() > 0 && !DeltaWeights.BufferCreate(OpenCL)) ReturnFalse; } else { if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) ReturnFalse; } if(FirstMomentum.GetIndex() >= 0) FirstMomentum.BufferFree(); if(!FirstMomentum.Load(file_handle)) ReturnFalse; if(FirstMomentum.Total() > 0 && !FirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) ReturnFalse; } if(SecondMomentum.GetIndex() >= 0) SecondMomentum.BufferFree(); if(!SecondMomentum.Load(file_handle)) ReturnFalse; if(SecondMomentum.Total() > 0 && !SecondMomentum.BufferCreate(OpenCL)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronBaseOCL::GetLayerInfo(void) { CLayerDescription* result = new CLayerDescription(); if(!result) return result; //--- result.type = Type(); result.count = Output.Total(); result.optimization = optimization; result.activation = activation; result.batch = (int)(optimization == LS ? iBatch : 1); result.layers = 1; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::numOutputs(const uint outputs, ENUM_OPTIMIZATION optimization_type) { if(outputs > 0) { if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) ReturnFalse; } Weights.BufferFree(); Weights.Clear(); int count = (int)((Output.Total() + 1) * outputs); if(!Weights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(Output.Total() + 1)); for(int i = 0; i < count; i++) { if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!Weights.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization_type == SGD) { if(CheckPointer(DeltaWeights) == POINTER_INVALID) { DeltaWeights = new CBufferFloat(); if(CheckPointer(DeltaWeights) == POINTER_INVALID) ReturnFalse; } DeltaWeights.BufferFree(); if(!DeltaWeights.BufferInit(count, 0)) ReturnFalse; if(!DeltaWeights.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); } else { DeleteObj(DeltaWeights); //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) ReturnFalse; } FirstMomentum.BufferFree(); if(!FirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!FirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) ReturnFalse; } SecondMomentum.BufferFree(); if(!SecondMomentum.BufferInit(count, 0)) ReturnFalse; if(!SecondMomentum.BufferCreate(OpenCL)) ReturnFalse; } } else { DeleteObj(Weights); DeleteObj(DeltaWeights); DeleteObj(FirstMomentum); DeleteObj(SecondMomentum); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBaseOCL::SetOpenCL(COpenCLMy *obj) { if(OpenCL == obj) return; DeleteObj(OpenCL); if(!obj) return; OpenCL = obj; if(!!Output) Output.BufferCreate(OpenCL); ///< Buffer of Output tensor if(!!PrevOutput) PrevOutput.BufferCreate(OpenCL); ///< Buffer of previous iteration Output tensor if(!!Weights) Weights.BufferCreate(OpenCL); ///< Buffer of weights matrix if(!!DeltaWeights) DeltaWeights.BufferCreate(OpenCL); ///< Buffer of last delta weights matrix (#SGD) if(!!Gradient) Gradient.BufferCreate(OpenCL); ///< Buffer of gradient tensor if(!!FirstMomentum) FirstMomentum.BufferCreate(OpenCL); ///< Buffer of first momentum matrix (#ADAM) if(!!SecondMomentum) SecondMomentum.BufferCreate(OpenCL); ///< Buffer of second momentum matrix (#ADAM) } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SetGradient(CBufferFloat *buffer, bool delete_prev = true) { if(Gradient == buffer) return true; //--- if(!!Gradient) { if(Output.Total() > buffer.Total()) ReturnFalse; if(delete_prev) DeleteObj(Gradient); } Gradient = buffer; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SetOutput(CBufferFloat *buffer, bool delete_prev = true) { if(Output == buffer) return true; //--- if(!!Output) { if(Output.Total() > buffer.Total()) ReturnFalse; if(delete_prev) DeleteObj(Output); } Output = buffer; //--- return true; } //+------------------------------------------------------------------+ ///\class CNeuronProofOCL /// Class of pooling layer GPU calculation //+------------------------------------------------------------------+ class CNeuronProofOCL : public CNeuronBaseOCL { protected: uint iWindow; ///< Input window size uint iStep; ///< Size of step virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< Feed Forward method.@param NeuronOCL Pointer to previos layer. //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true;}; ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. public: /** Constructor */ CNeuronProofOCL(void) : iWindow(2), iStep(1) {}; /** Destructor */~CNeuronProofOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of input window @param[in] step Step size.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- methods for working with files virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation virtual int Type(void) const { return defNeuronProofOCL; }///< Identificator of class.@return Type of class virtual CLayerDescription* GetLayerInfo(void); virtual uint GetWindow(void) const { return iWindow; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProofOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iStep = step; activation = None; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronProofOCL::~CNeuronProofOCL(void) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProofOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); setBuffer(def_k_FeedForwardProof, def_k_ffp_matrix_i, NeuronOCL.getOutputIndex()); setBuffer(def_k_FeedForwardProof, def_k_ffp_matrix_o, Output.GetIndex()); setArgument(def_k_FeedForwardProof, def_k_ffp_inputs, NeuronOCL.Neurons()); setArgument(def_k_FeedForwardProof, def_k_ffp_window, (int)iWindow); setArgument(def_k_FeedForwardProof, def_k_ffp_step, (int)iStep); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_FeedForwardProof, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProofOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = NeuronOCL.Neurons(); setBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_i, NeuronOCL.getOutputIndex()); setBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_o, Output.GetIndex()); setBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_g, Gradient.GetIndex()); setBuffer(def_k_CalcInputGradientProof, def_k_cigp_matrix_ig, NeuronOCL.getGradientIndex()); setArgument(def_k_CalcInputGradientProof, def_k_cigp_outputs, Output.Total()); setArgument(def_k_CalcInputGradientProof, def_k_cigp_window, (int)iWindow); setArgument(def_k_CalcInputGradientProof, def_k_cigp_step, (int)iStep); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_CalcInputGradientProof, global_work_offset, global_work_size) //NeuronOCL.getGradient().BufferRead(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronProofOCL::GetLayerInfo(void) { CLayerDescription *result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; result.window = (int)iWindow; result.step = (int)iStep; //--- return result; } //+------------------------------------------------------------------+ ///\class CNeuronConvOCL /// Class of convolution layer GPU calculation ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronConvOCL : public CNeuronProofOCL { protected: uint iWindowOut; ///< Size of out window uint iVariables; ///< Size of out window //--- CBufferFloat *WeightsConv; ///< Matrix of weights to previous layer CBufferFloat *DeltaWeightsConv; ///< Matrix of delta weights to previous layer (#SGD) CBufferFloat *FirstMomentumConv; ///< Matrix of first momentum to previous layer (#ADAM) CBufferFloat *SecondMomentumConv; ///< Matrix of second momentum to previous layer (#ADAM) //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< Feed Forward method.@param NeuronOCL Pointer to previos layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; ///< Method for updating weights.@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; ///< Method to transfer gradients to previous layer @param[in] NeuronOCL Pointer to previous layer. public: /** Constructor */ CNeuronConvOCL(void) : iWindowOut(1), iVariables(1) { activation = SoftPlus; } /** Destructor */~CNeuronConvOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of input window @param[in] step Step size.@param[in] window_out Size of output window @param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual CBufferFloat* GetWeightsConv(void) { return WeightsConv; } //--- virtual int Type(void) override const { return defNeuronConvOCL; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual CLayerDescription* GetLayerInfo(void) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetFilters(void) const { return iWindowOut; } virtual uint GetVariables(void) const { return iVariables; } virtual uint GetUnits(void) const { return Output.Total() / (iWindowOut * iVariables); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronConvOCL::~CNeuronConvOCL(void) { DeleteObj(WeightsConv); DeleteObj(DeltaWeightsConv); DeleteObj(FirstMomentumConv); DeleteObj(SecondMomentumConv); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { return CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window_in, step, window_out, units_count, 1, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if((optimization == LS && batch <= 0)) ReturnFalse; if(!CNeuronProofOCL::Init(numOutputs, myIndex, open_cl, window_in, step, units_count * window_out * variables, optimization_type, batch)) ReturnFalse; //--- iWindowOut = window_out; iVariables = variables; //--- if(CheckPointer(WeightsConv) == POINTER_INVALID) { WeightsConv = new CBufferFloat(); if(CheckPointer(WeightsConv) == POINTER_INVALID) ReturnFalse; } int count = (int)((iWindow + 1) * iWindowOut * iVariables); if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) { DeltaWeightsConv = new CBufferFloat(); if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) ReturnFalse; } if(!DeltaWeightsConv.BufferInit(count, 0.0)) ReturnFalse; if(!DeltaWeightsConv.BufferCreate(OpenCL)) ReturnFalse; } else { if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) { FirstMomentumConv = new CBufferFloat(); if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) { SecondMomentumConv = new CBufferFloat(); if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = { Output.Total() / (iWindowOut * iVariables), iWindowOut, iVariables }; setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_w, WeightsConv.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_o, Output.GetIndex()); setArgument(def_k_FeedForwardConv, def_k_ffc_inputs, NeuronOCL.Neurons() / iVariables); setArgument(def_k_FeedForwardConv, def_k_ffc_step, (int)iStep); setArgument(def_k_FeedForwardConv, def_k_ffc_window_in, (int)iWindow); setArgument(def_k_FeedForwardConv, def_k_ffс_window_out, (int)iWindowOut); setArgument(def_k_FeedForwardConv, def_k_ffc_activation, (int)activation); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_FeedForwardConv, global_work_offset, global_work_size) //vector temp; //Output.GetData(temp); //float delta = MathAbs((temp - prev_output)).Sum(); //prev_output = temp; //string error; //CLGetInfoString(OpenCL.GetContext(), CL_ERROR_DESCRIPTION, error); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iVariables; global_work_size[1] = iVariables; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, WeightsConv.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, Gradient.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, Neurons() / iVariables); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)iStep); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)iWindow); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)iWindowOut); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)NeuronOCL.Activation()); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)0); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { WeightsConv.Total() }; uint global_work_offset_am[3] = { 0, 0, 0 }; uint global_work_size_am[3] = { iWindow + 1, iWindowOut, iVariables }; uint local_work_size_am[3] = { global_work_size_am[0], 1, 1 }; float lt = 0; switch(optimization) { case SGD: setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_w, WeightsConv.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_g, getGradientIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_i, NeuronOCL.getOutputIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_dw, DeltaWeightsConv.GetIndex()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_inputs, NeuronOCL.Neurons() / iVariables); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_learning_rates, lr); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_momentum, alpha); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_in, (int)iWindow); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_out, (int)iWindowOut); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_step, (int)iStep); ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsConvMomentum, global_work_offset, global_work_size) break; case ADAM: setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_inputs, NeuronOCL.Neurons() / iVariables) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_in, (int)iWindow) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_out, (int)iWindowOut) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_step, (int)iStep) ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsConvAdam, global_work_offset, global_work_size) t++; break; case LS: global_work_size[0] = iWindow + 1; setBuffer(def_k_UpdateWeightsConvLS, def_k_uwcls_matrix_w, WeightsConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvLS, def_k_uwcls_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsConvLS, def_k_uwcls_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsConvLS, def_k_uwcls_matrix_xg, FirstMomentumConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvLS, def_k_uwcls_matrix_xx, SecondMomentumConv.GetIndex()) setArgument(def_k_UpdateWeightsConvLS, def_k_uwls_inputs, NeuronOCL.Neurons()) setArgument(def_k_UpdateWeightsConvLS, def_k_uwcls_l, lr) setArgument(def_k_UpdateWeightsConvLS, def_k_uwcls_update, (int)(t >= iBatch)) setArgument(def_k_UpdateWeightsConvLS, def_k_uwcls_window_in, (int)iWindow) setArgument(def_k_UpdateWeightsConvLS, def_k_uwcls_window_out, (int)iWindowOut) setArgument(def_k_UpdateWeightsConvLS, def_k_uwcls_step, (int)iStep) ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_UpdateWeightsConvLS, global_work_offset, global_work_size) if(t >= iBatch) t = 0; else t++; break; case ADAM_MINI: setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_w, WeightsConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_g, getGradientIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_v, SecondMomentumConv.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_inputs, NeuronOCL.Neurons() / iVariables) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_l, lt) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b1, b1) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b2, b2) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_step, (int)iStep) ResetLastError(); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecuteLoc(def_k_UpdateWeightsConvAdamMini, global_work_offset_am, global_work_size_am, local_work_size_am) t++; break; default: ReturnFalse; break; } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronConvOCL::GetLayerInfo(void) { CLayerDescription *result = CNeuronProofOCL::GetLayerInfo(); if(!result) return result; result.window_out = (int)iWindowOut; result.count /= (int)iWindowOut; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConvOCL::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!WeightsConv) WeightsConv.BufferCreate(obj); ///< Matrix of weights to previous layer if(!!DeltaWeightsConv) DeltaWeightsConv.BufferCreate(obj); ///< Matrix of delta weights to previous layer (#SGD) if(!!FirstMomentumConv) FirstMomentumConv.BufferCreate(obj); ///< Matrix of first momentum to previous layer (#ADAM) if(!!SecondMomentumConv) SecondMomentumConv.BufferCreate(obj); ///< Matrix of second momentum to previous layer (#ADAM) } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConvSAMOCL : public CNeuronConvOCL { protected: float fRho; //--- CBufferFloat cWeightsSAM; CBufferFloat cWeightsSAMConv; //--- virtual bool calcEpsilonWeights(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardSAM(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); //--- virtual bool InitBufferLike(CBufferFloat *&buffer, CBufferFloat *master); virtual void ReplaceBuffer(CBufferFloat *&buffer, CBufferFloat *master); public: CNeuronConvSAMOCL(void) { activation = GELU; } ~CNeuronConvSAMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool InitPS(CNeuronConvSAMOCL *master); //--- virtual int Type(void) const { return defNeuronConvSAMOCL; } virtual int Activation(void) const { return (fRho == 0 ? (int)None : (int)activation); } virtual int getWeightsSAMIndex(void) const override { return cWeightsSAM.GetIndex(); } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); //--- }; //+------------------------------------------------------------------+ ///\class CNeuronAttentionOCL /// Class of Self-Attention layer GPU calculation ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronAttentionOCL : public CNeuronBaseOCL { protected: CNeuronConvOCL *Querys; ///< Convolution layer for Querys CNeuronConvOCL *Values; ///< Convolution layer for Values CBufferFloat *Scores; ///< Buffer for Scores matrix CNeuronBaseOCL *AttentionOut; ///< Layer of Self-Attention Out CNeuronConvOCL *FF1; ///< Convolution layer for first layer of Feed Forward block CNeuronConvOCL *FF2; ///< Convolution layer for second layer of Feed Forward block //--- uint iWindow; ///< Window size uint iUnits; ///< Number of units //--- virtual bool feedForward(CNeuronBaseOCL *prevLayer); ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool updateInputWeights(CNeuronBaseOCL *prevLayer); ///< Method for updating weights.@param prevLayer Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. public: /** Constructor */ CNeuronAttentionOCL(void) : iWindow(1), iUnits(0) {}; /** Destructor */~CNeuronAttentionOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int Type(void) const { return defNeuronAttentionOCL; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation virtual CLayerDescription* GetLayerInfo(void); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronAttentionOCL::~CNeuronAttentionOCL(void) { DeleteObj(Querys); DeleteObj(Values); DeleteObj(Scores); DeleteObj(AttentionOut); DeleteObj(FF1); DeleteObj(FF2); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) { Querys = new CNeuronConvOCL(); if(CheckPointer(Querys) == POINTER_INVALID) ReturnFalse; if(!Querys.Init(0, 0, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys.SetActivationFunction(None); } //--- if(CheckPointer(Values) == POINTER_INVALID) { Values = new CNeuronConvOCL(); if(CheckPointer(Values) == POINTER_INVALID) ReturnFalse; if(!Values.Init(0, 2, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values.SetActivationFunction(None); } //--- if(CheckPointer(Scores) == POINTER_INVALID) { Scores = new CBufferFloat(); if(CheckPointer(Scores) == POINTER_INVALID) ReturnFalse; } if(!Scores.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) { AttentionOut = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut) == POINTER_INVALID) ReturnFalse; if(!AttentionOut.Init(0, 3, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); } //--- if(CheckPointer(FF1) == POINTER_INVALID) { FF1 = new CNeuronConvOCL(); if(CheckPointer(FF1) == POINTER_INVALID) ReturnFalse; if(!FF1.Init(0, 4, open_cl, window, window, window * 4, units_count, optimization_type, batch)) ReturnFalse; FF1.SetActivationFunction(LReLU); } //--- if(CheckPointer(FF2) == POINTER_INVALID) { FF2 = new CNeuronConvOCL(); if(CheckPointer(FF2) == POINTER_INVALID) ReturnFalse; if(!FF2.Init(0, 5, open_cl, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; FF2.SetActivationFunction(None); FF2.SetGradientIndex(Gradient.GetIndex()); } //--- iWindow = window; iUnits = units_count; activation = (ENUM_ACTIVATION)FF2.Activation(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normilize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normilize, def_k_norm_dimension, prevLayer.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normilize, global_work_offset, global_work_size) //if(!prevLayer.Output.BufferRead()) // ReturnFalse; } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, (int)iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) //if(!Scores.BufferRead()) // ReturnFalse; } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) //float temp[]; //if(!AttentionOut.getOutputVal(temp)) // ReturnFalse; } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normilize, def_k_norm_buffer, AttentionOut.getOutputIndex()); setArgument(def_k_Normilize, def_k_norm_dimension, AttentionOut.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normilize, global_work_offset, global_work_size) //float temp[]; //if(!AttentionOut.getOutputVal(temp)) // ReturnFalse; } //--- if(!FF1.FeedForward(AttentionOut)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) //if(!Output.BufferRead()) // ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!AttentionOut.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) float temp[]; if(AttentionOut.getGradient(temp) <= 0) ReturnFalse; } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, AttentionOut.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, Querys.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, Values.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, Values.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, Scores.GetIndex()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) float temp[]; if(Querys.getGradient(temp) <= 0) ReturnFalse; } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) float temp[]; if(AttentionOut.getGradient(temp) <= 0) ReturnFalse; } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Querys)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, AttentionOut.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) float temp[]; if(AttentionOut.getGradient(temp) <= 0) ReturnFalse; } //--- if(!prevLayer.CalcHiddenGradients((CObject *)Values)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, AttentionOut.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, prevLayer.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)(iWindow + 1)) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.1f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) float temp[]; if(prevLayer.getGradient(temp) <= 0) ReturnFalse; } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normilize, def_k_norm_buffer, prevLayer.getGradientIndex()); setArgument(def_k_Normilize, def_k_norm_dimension, prevLayer.Neurons()); //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_Normilize, global_work_offset, global_work_size) float temp[]; if(prevLayer.getGradient(temp) <= 0) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Values.UpdateInputWeights(prevLayer)) ReturnFalse; if(!FF1.UpdateInputWeights(AttentionOut)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronAttentionOCL *Source = source; dWeightsUpdate(Querys, Source, tau); dWeightsUpdate(Values, Source, tau); dWeightsUpdate(FF1, Source, tau); dWeightsUpdate(FF2, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CBufferFloat::BufferToCSV(const string file_name) { BufferRead(); int h = FileOpen(file_name, FILE_CSV | FILE_WRITE); if(h == INVALID_HANDLE) return; //--- for(int i = 0; i < m_data_total; i++) FileWrite(h, m_data[i]); FileFlush(h); FileClose(h); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(CheckPointer(Querys) == POINTER_INVALID || !Querys.Save(file_handle)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores) == POINTER_INVALID || !Scores.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut) == POINTER_INVALID || !AttentionOut.Save(file_handle)) ReturnFalse; if(CheckPointer(FF1) == POINTER_INVALID || !FF1.Save(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID || !FF2.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iUnits, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys) == POINTER_INVALID) Querys = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values) == POINTER_INVALID) Values = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores) == POINTER_INVALID) Scores = new CBufferFloat(); if(Scores.GetIndex() >= 0) Scores.BufferFree(); if(!Scores.Load(file_handle)) ReturnFalse; if(!Scores.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut) == POINTER_INVALID) AttentionOut = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(FF1) == POINTER_INVALID) FF1 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF1.Load(file_handle)) ReturnFalse; if(CheckPointer(FF2) == POINTER_INVALID) FF2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !FF2.Load(file_handle)) ReturnFalse; iWindow = FileReadInteger(file_handle); iUnits = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronAttentionOCL::GetLayerInfo() { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; result.window = (int)iWindow; result.count = (int)iUnits; return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::Load(const int file_handle) { if(!CNeuronProofOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(WeightsConv) == POINTER_INVALID) WeightsConv = new CBufferFloat(); if(WeightsConv.GetIndex() >= 0) WeightsConv.BufferFree(); if(!WeightsConv.Load(file_handle)) ReturnFalse; if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) DeltaWeightsConv = new CBufferFloat(); if(DeltaWeightsConv.GetIndex() >= 0) DeltaWeightsConv.BufferFree(); if(!DeltaWeightsConv.Load(file_handle)) ReturnFalse; if(!DeltaWeightsConv.BufferCreate(OpenCL)) ReturnFalse; } else { if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) FirstMomentumConv = new CBufferFloat(); if(FirstMomentumConv.GetIndex() >= 0) FirstMomentumConv.BufferFree(); if(!FirstMomentumConv.Load(file_handle)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) SecondMomentumConv = new CBufferFloat(); if(SecondMomentumConv.GetIndex() >= 0) SecondMomentumConv.BufferFree(); if(!SecondMomentumConv.Load(file_handle)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; } iWindowOut = FileReadInteger(file_handle); iVariables = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::Save(const int file_handle) { if(!CNeuronProofOCL::Save(file_handle)) ReturnFalse; //--- if(CheckPointer(WeightsConv) == POINTER_INVALID || !WeightsConv.Save(file_handle)) ReturnFalse; if(optimization == SGD && (CheckPointer(DeltaWeightsConv) == POINTER_INVALID || !DeltaWeightsConv.Save(file_handle))) ReturnFalse; if(optimization != SGD && (CheckPointer(FirstMomentumConv) == POINTER_INVALID || !FirstMomentumConv.Save(file_handle))) ReturnFalse; if(optimization != SGD && (CheckPointer(SecondMomentumConv) == POINTER_INVALID || !SecondMomentumConv.Save(file_handle))) ReturnFalse; if(FileWriteInteger(file_handle, iWindowOut, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iVariables, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProofOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindow, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iStep, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronProofOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; iWindow = FileReadInteger(file_handle); iStep = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ ///\ingroup neuron_base ///\class CNeuronMHAttentionOCL ///\brief Class of Multi-Head Self-Attention layer GPU calculation ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronMHAttentionOCL : public CNeuronAttentionOCL { protected: CNeuronConvOCL *Querys2; ///< Convolution layer for Querys Head 2 CNeuronConvOCL *Querys3; ///< Convolution layer for Querys Head 3 CNeuronConvOCL *Querys4; ///< Convolution layer for Querys Head 4 CNeuronConvOCL *Keys2; ///< Convolution layer for Keys Head 2 CNeuronConvOCL *Keys3; ///< Convolution layer for Keys Head 3 CNeuronConvOCL *Keys4; ///< Convolution layer for Keys Head 4 CNeuronConvOCL *Values2; ///< Convolution layer for Values Head 2 CNeuronConvOCL *Values3; ///< Convolution layer for Values Head 3 CNeuronConvOCL *Values4; ///< Convolution layer for Values Head 4 CBufferFloat *Scores2; ///< Buffer for Scores matrix Head 2 CBufferFloat *Scores3; ///< Buffer for Scores matrix Head 3 CBufferFloat *Scores4; ///< Buffer for Scores matrix Head 4 CNeuronBaseOCL *AttentionOut2; ///< Layer of Self-Attention Out CNeuronBaseOCL *AttentionOut3; ///< Layer of Self-Attention Out CNeuronBaseOCL *AttentionOut4; ///< Layer of Self-Attention Out CNeuronBaseOCL *AttentionConcatenate;///< Layer of Concatenate Self-Attention Out CNeuronConvOCL *Weights0; ///< Convolution layer for Weights0 //--- virtual bool feedForward(CNeuronBaseOCL *prevLayer); ///< Feed Forward method.@param prevLayer Pointer to previos layer. virtual bool updateInputWeights(CNeuronBaseOCL *prevLayer); ///< Method for updating weights.@param prevLayer Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. /// Method to transfer gradients inside Head Self-Attention virtual bool calcHeadGradient(CNeuronConvOCL *query, CNeuronConvOCL *value, CBufferFloat *score, CNeuronBaseOCL *attention, CNeuronBaseOCL *prevLayer); public: /** Constructor */ CNeuronMHAttentionOCL(void) {}; /** Destructor */~CNeuronMHAttentionOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int Type(void) const { return defNeuronMHAttentionOCL; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual CLayerDescription* GetLayerInfo(void); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMHAttentionOCL::~CNeuronMHAttentionOCL(void) { DeleteObj(Querys2); DeleteObj(Querys3); DeleteObj(Querys4); DeleteObj(Keys2); DeleteObj(Keys3); DeleteObj(Keys4); DeleteObj(Values2); DeleteObj(Values3); DeleteObj(Values4); DeleteObj(Scores2); DeleteObj(Scores3); DeleteObj(Scores4); DeleteObj(Weights0); DeleteObj(AttentionOut2); DeleteObj(AttentionOut3); DeleteObj(AttentionOut4); DeleteObj(AttentionConcatenate); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAttentionOCL::Init(numOutputs, myIndex, open_cl, window, units_count, optimization_type, batch)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) { Querys2 = new CNeuronConvOCL(); if(CheckPointer(Querys2) == POINTER_INVALID) ReturnFalse; if(!Querys2.Init(0, 6, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys2.SetActivationFunction(None); } //--- if(CheckPointer(Querys3) == POINTER_INVALID) { Querys3 = new CNeuronConvOCL(); if(CheckPointer(Querys3) == POINTER_INVALID) ReturnFalse; if(!Querys3.Init(0, 7, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys3.SetActivationFunction(None); } //--- if(CheckPointer(Querys4) == POINTER_INVALID) { Querys4 = new CNeuronConvOCL(); if(CheckPointer(Querys4) == POINTER_INVALID) ReturnFalse; if(!Querys4.Init(0, 8, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Querys4.SetActivationFunction(None); } //--- if(CheckPointer(Values2) == POINTER_INVALID) { Values2 = new CNeuronConvOCL(); if(CheckPointer(Values2) == POINTER_INVALID) ReturnFalse; if(!Values2.Init(0, 9, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values2.SetActivationFunction(None); } //--- if(CheckPointer(Values3) == POINTER_INVALID) { Values3 = new CNeuronConvOCL(); if(CheckPointer(Values3) == POINTER_INVALID) ReturnFalse; if(!Values3.Init(0, 10, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values3.SetActivationFunction(None); } //--- if(CheckPointer(Values4) == POINTER_INVALID) { Values4 = new CNeuronConvOCL(); if(CheckPointer(Values4) == POINTER_INVALID) ReturnFalse; if(!Values4.Init(0, 11, open_cl, window, window, window, units_count, optimization_type, batch)) ReturnFalse; Values4.SetActivationFunction(None); } //--- if(CheckPointer(Scores2) == POINTER_INVALID) { Scores2 = new CBufferFloat(); if(CheckPointer(Scores2) == POINTER_INVALID) ReturnFalse; } if(!Scores2.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) { Scores3 = new CBufferFloat(); if(CheckPointer(Scores3) == POINTER_INVALID) ReturnFalse; } if(!Scores3.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) { Scores4 = new CBufferFloat(); if(CheckPointer(Scores4) == POINTER_INVALID) ReturnFalse; } if(!Scores4.BufferInit(units_count * units_count, 0.0)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) { AttentionOut2 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut2) == POINTER_INVALID) ReturnFalse; if(!AttentionOut2.Init(0, 12, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut2.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) { AttentionOut3 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut3) == POINTER_INVALID) ReturnFalse; if(!AttentionOut3.Init(0, 13, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut3.SetActivationFunction(None); } //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) { AttentionOut4 = new CNeuronBaseOCL(); if(CheckPointer(AttentionOut4) == POINTER_INVALID) ReturnFalse; if(!AttentionOut4.Init(0, 14, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; AttentionOut4.SetActivationFunction(None); } //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) { AttentionConcatenate = new CNeuronBaseOCL(); if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) ReturnFalse; if(!AttentionConcatenate.Init(0, 15, open_cl, 4 * window * units_count, optimization_type, batch)) ReturnFalse; AttentionConcatenate.SetActivationFunction(None); } //--- if(CheckPointer(Weights0) == POINTER_INVALID) { Weights0 = new CNeuronConvOCL(); if(CheckPointer(Weights0) == POINTER_INVALID) ReturnFalse; if(!Weights0.Init(0, 16, open_cl, 4 * window, 4 * window, window, units_count, optimization_type, batch)) ReturnFalse; Weights0.SetActivationFunction(None); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::feedForward(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {1}; setBuffer(def_k_Normilize, def_k_norm_buffer, prevLayer.getOutputIndex()); setArgument(def_k_Normilize, def_k_norm_dimension, prevLayer.Neurons()); kernelExecute(def_k_Normilize, global_work_offset, global_work_size) } //--- if(CheckPointer(Querys) == POINTER_INVALID || !Querys.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values) == POINTER_INVALID || !Values.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.FeedForward(prevLayer)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.FeedForward(prevLayer)) ReturnFalse; //--- Scores Head 1 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores.GetIndex()) setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_values, Values.getOutputIndex()) setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut.getOutputIndex()) kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) float temp[]; if(!AttentionOut.getOutputVal(temp)) ReturnFalse; } //--- Scores Head 2 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys2.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores2.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores2.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values2.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut2.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) } //--- Scores Head 3 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys3.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores3.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores3.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values3.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut3.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) float temp[]; if(!AttentionOut3.getOutputVal(temp)) ReturnFalse; } //--- Scores Head 4 { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_AttentionScore, def_k_as_querys, Querys4.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionScore, def_k_as_score, Scores4.GetIndex()); setArgument(def_k_AttentionScore, def_k_as_dimension, iWindow); setArgument(def_k_AttentionScore, def_k_as_mask, 0); kernelExecute(def_k_AttentionScore, global_work_offset, global_work_size) } //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionOut, def_k_aout_scores, Scores4.GetIndex()); setBuffer(def_k_AttentionOut, def_k_aout_inputs, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_values, Values4.getOutputIndex()); setBuffer(def_k_AttentionOut, def_k_aout_out, AttentionOut4.getOutputIndex()); kernelExecute(def_k_AttentionOut, global_work_offset, global_work_size) float temp[]; if(!AttentionOut4.getOutputVal(temp)) ReturnFalse; } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_ConcatenateMatrix, def_k_conc_input1, AttentionOut.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window1, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input2, AttentionOut2.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window2, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input3, AttentionOut3.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window3, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_input4, AttentionOut4.getOutputIndex()); setArgument(def_k_ConcatenateMatrix, def_k_conc_window4, iWindow); setBuffer(def_k_ConcatenateMatrix, def_k_conc_out, AttentionConcatenate.getOutputIndex()); kernelExecute(def_k_ConcatenateMatrix, global_work_offset, global_work_size) float temp[]; if(!AttentionConcatenate.getOutputVal(temp)) ReturnFalse; } //--- if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.FeedForward(AttentionConcatenate)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getOutputIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = 1; setBuffer(def_k_Normilize, def_k_norm_buffer, Weights0.getOutputIndex()); setArgument(def_k_Normilize, def_k_norm_dimension, Weights0.Neurons()); kernelExecute(def_k_Normilize, global_work_offset, global_work_size) } //--- if(!FF1.FeedForward(Weights0)) ReturnFalse; if(!FF2.FeedForward(FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, FF2.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- if(!FF1.CalcHiddenGradients((CObject *)FF2)) ReturnFalse; if(!Weights0.CalcHiddenGradients((CObject *)FF1)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, Weights0.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, Gradient.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Weights0.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- if(!AttentionConcatenate.CalcHiddenGradients((CObject *)Weights0)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output1, AttentionOut.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window1, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output2, AttentionOut2.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window2, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output3, AttentionOut3.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window3, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output4, AttentionOut4.getGradientIndex()); setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window4, iWindow); setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_inputs, AttentionConcatenate.getGradientIndex()); kernelExecute(def_k_DeconcatenateMatrix, global_work_offset, global_work_size) } //--- if(!calcHeadGradient(Querys, Values, Scores, AttentionOut, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys2, Values2, Scores2, AttentionOut2, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys3, Values3, Scores3, AttentionOut3, prevLayer)) ReturnFalse; if(!calcHeadGradient(Querys4, Values4, Scores4, AttentionOut4, prevLayer)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix1, AttentionOut.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix2, AttentionOut2.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix3, AttentionOut3.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix4, AttentionOut4.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix5, Weights0.getGradientIndex()); setBuffer(def_k_Matrix5Sum, def_k_sum5_matrix_out, prevLayer.getGradientIndex()); setArgument(def_k_Matrix5Sum, def_k_sum5_dimension, (int)iWindow); setArgument(def_k_Matrix5Sum, def_k_sum5_multiplyer, (float)0.2); kernelExecute(def_k_Matrix5Sum, global_work_offset, global_work_size) } //--- { //uint global_work_offset[1]={0}; //uint global_work_size[1]; //global_work_size[0]=1; //setBuffer(def_k_Normilize,def_k_norm_buffer,prevLayer.getGradientIndex()); //setArgument(def_k_Normilize,def_k_norm_dimension,prevLayer.Neurons()); //kernelExecute(def_k_Normilize,1,global_work_offset,global_work_size)) // { // printf("Error of execution kernel Normalize: %d",GetLastError()); // ReturnFalse; // } //float temp[]; //if(prevLayer.getGradient(temp)<=0) // ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::calcHeadGradient(CNeuronConvOCL *query, CNeuronConvOCL *value, CBufferFloat *score, CNeuronBaseOCL *attention, CNeuronBaseOCL *prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iWindow; setBuffer(def_k_AttentionGradients, def_k_ag_gradient, attention.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys, prevLayer.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_keys_g, prevLayer.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys, query.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_querys_g, query.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values, value.getOutputIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_values_g, value.getGradientIndex()); setBuffer(def_k_AttentionGradients, def_k_ag_scores, score.GetIndex()); kernelExecute(def_k_AttentionGradients, global_work_offset, global_work_size) } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.5f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- if(!prevLayer.CalcHiddenGradients((CObject *)query)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- if(!prevLayer.CalcHiddenGradients((CObject *)value)) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iUnits; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, attention.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, prevLayer.getGradientIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, attention.getGradientIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)iWindow + 1) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 0.33f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::updateInputWeights(CNeuronBaseOCL *prevLayer) { if(!Querys.UpdateInputWeights(prevLayer) || !Querys2.UpdateInputWeights(prevLayer) || !Querys3.UpdateInputWeights(prevLayer) || !Querys4.UpdateInputWeights(prevLayer)) ReturnFalse; //--- if(!Values.UpdateInputWeights(prevLayer) || !Values2.UpdateInputWeights(prevLayer) || !Values3.UpdateInputWeights(prevLayer) || !Values4.UpdateInputWeights(prevLayer)) ReturnFalse; if(!Weights0.UpdateInputWeights(AttentionConcatenate)) ReturnFalse; if(!FF1.UpdateInputWeights(Weights0)) ReturnFalse; if(!FF2.UpdateInputWeights(FF1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMHAttentionOCL *Source = source; if(!Querys.WeightsUpdate(Source.Querys, tau) || !Querys2.WeightsUpdate(Source.Querys2, tau) || !Querys3.WeightsUpdate(Source.Querys3, tau) || !Querys4.WeightsUpdate(Source.Querys4, tau)) ReturnFalse; //--- if(!Values.WeightsUpdate(Source.Values, tau) || !Values2.WeightsUpdate(Source.Values2, tau) || !Values3.WeightsUpdate(Source.Values3, tau) || !Values4.WeightsUpdate(Source.Values4, tau)) ReturnFalse; if(!Weights0.WeightsUpdate(Source.Weights0, tau)) ReturnFalse; if(!FF1.WeightsUpdate(Source.FF1, tau)) ReturnFalse; if(!FF2.WeightsUpdate(Source.FF2, tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Save(const int file_handle) { if(!CNeuronAttentionOCL::Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID || !Querys2.Save(file_handle)) ReturnFalse; if(CheckPointer(Values2) == POINTER_INVALID || !Values2.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores2) == POINTER_INVALID || !Scores2.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut2) == POINTER_INVALID || !AttentionOut2.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID || !Querys3.Save(file_handle)) ReturnFalse; if(CheckPointer(Values3) == POINTER_INVALID || !Values3.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores3) == POINTER_INVALID || !Scores3.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut3) == POINTER_INVALID || !AttentionOut3.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID || !Querys4.Save(file_handle)) ReturnFalse; if(CheckPointer(Values4) == POINTER_INVALID || !Values4.Save(file_handle)) ReturnFalse; if(CheckPointer(Scores4) == POINTER_INVALID || !Scores4.Save(file_handle)) ReturnFalse; if(CheckPointer(AttentionOut4) == POINTER_INVALID || !AttentionOut4.Save(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID || !AttentionConcatenate.Save(file_handle)) ReturnFalse; if(CheckPointer(Weights0) == POINTER_INVALID || !Weights0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionOCL::Load(const int file_handle) { if(!CNeuronAttentionOCL::Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys2) == POINTER_INVALID) Querys2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values2) == POINTER_INVALID) Values2 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores2) == POINTER_INVALID) Scores2 = new CBufferFloat(); if(Scores2.GetIndex() >= 0) Scores2.BufferFree(); if(!Scores2.Load(file_handle)) ReturnFalse; if(!Scores2.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut2) == POINTER_INVALID) AttentionOut2 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut2.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys3) == POINTER_INVALID) Querys3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values3) == POINTER_INVALID) Values3 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores3) == POINTER_INVALID) Scores3 = new CBufferFloat(); if(Scores3.GetIndex() >= 0) Scores3.BufferFree(); if(!Scores3.Load(file_handle)) ReturnFalse; if(!Scores3.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut3) == POINTER_INVALID) AttentionOut3 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut3.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Querys4) == POINTER_INVALID) Querys4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Querys4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Values4) == POINTER_INVALID) Values4 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Values4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Scores4) == POINTER_INVALID) Scores4 = new CBufferFloat(); if(Scores4.GetIndex() >= 0) Scores4.BufferFree(); if(!Scores4.Load(file_handle)) ReturnFalse; if(!Scores4.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(AttentionOut4) == POINTER_INVALID) AttentionOut4 = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionOut4.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(AttentionConcatenate) == POINTER_INVALID) AttentionConcatenate = new CNeuronBaseOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronBaseOCL || !AttentionConcatenate.Load(file_handle)) ReturnFalse; //--- if(CheckPointer(Weights0) == POINTER_INVALID) Weights0 = new CNeuronConvOCL(); if(FileReadInteger(file_handle, INT_VALUE) != defNeuronConvOCL || !Weights0.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronAttentionOCL::GetLayerInfo(); if(!result) return result; result.step = 4; //--- return result; } //+------------------------------------------------------------------+ ///\class CCollection ///\brief Class of objects collection. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CCollection : public CArrayObj { public: CCollection(void) {}; ~CCollection(void) {}; //--- CObject* operator[](const int i) const { return(m_data[i]); } //--- virtual bool CreateElement(const int index) { if(index < 0) ReturnFalse; //--- if(index >= m_data_max) { if(!Reserve(index - m_data_total)) ReturnFalse; } m_data[index] = new CBufferFloat(); return(CheckPointer(m_data[index]) != POINTER_INVALID); } //--- virtual bool SetOpenCL(COpenCLMy *open_cl) { if(!open_cl) ReturnFalse; bool result = true; for(int i = 0; (i < m_data_total && result); i++) { CBufferFloat* temp = m_data[i]; if(!temp || !temp.BufferCreate(open_cl)) { result = false; break; } } //--- return result; } //--- }; //+------------------------------------------------------------------+ ///\ingroup neuron_base ///\class CNeuronMLMHAttentionOCL ///\brief Class of Multilayer multi-headed attention neuron. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronMLMHAttentionOCL : public CNeuronBaseOCL { protected: uint iLayers; ///< Number of inner layers uint iHeads; ///< Number of heads uint iWindow; ///< Input window size uint iUnits; ///< Number of units uint iWindowKey; ///< Size of Key/Query window //--- CCollection *QKV_Tensors; ///< The collection of tensors of Queries, Keys and Values CCollection *QKV_Weights; ///< The collection of Matrix of weights to previous layer CCollection *S_Tensors; ///< The collection of Scores tensors CCollection *AO_Tensors; ///< The collection of Attention Out tensors CCollection *FF_Tensors; ///< The collection of tensors of Feed Forward output CCollection *FF_Weights; ///< The collection of Matrix of Feed Forward weights ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. virtual bool ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0, uint variables = 1); ///< \brief Convolution Feed Forward method of calling kernel ::FeedForwardConv(). virtual bool AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false); ///< \brief Multi-heads attention scores method of calling kernel ::MHAttentionScore(). virtual bool AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out); ///< \brief Multi-heads attention out method of calling kernel ::MHAttentionOut(). ///\ingroup neuron_base_opt virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *momentum1, CBufferFloat *momentum2, uint window, uint window_out, uint step = 0, uint heads = 0, uint variables = 1); virtual bool ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *source, CBufferFloat *momentum1, CBufferFloat *momentum2, float tau); ///< Method for updating weights in convolution layer.\details Calling one of kernels ::UpdateWeightsConvMomentum() or ::UpdateWeightsConvAdam() in depends of optimization type (#ENUM_OPTIMIZATION). virtual bool ConvolutionInputGradients(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0, uint variables = 1); ///< Method of passing gradients through a convolutional layer. virtual bool AttentionInsideGradients(CBufferFloat *qkv, CBufferFloat *qkv_g, CBufferFloat *scores, CBufferFloat *gradient); ///< Method of passing gradients through attention layer. public: /** Constructor */ CNeuronMLMHAttentionOCL(void); /** Destructor */~CNeuronMLMHAttentionOCL(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int Type(void) const { return defNeuronMLMHAttentionOCL; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual CLayerDescription* GetLayerInfo(void); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::CNeuronMLMHAttentionOCL(void) : iLayers(0), iHeads(0), iWindow(0), iWindowKey(0), iUnits(0) { QKV_Tensors = new CCollection(); QKV_Weights = new CCollection(); S_Tensors = new CCollection(); AO_Tensors = new CCollection(); FF_Tensors = new CCollection(); FF_Weights = new CCollection(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMLMHAttentionOCL::~CNeuronMLMHAttentionOCL(void) { DeleteObj(QKV_Tensors); DeleteObj(QKV_Weights); DeleteObj(S_Tensors); DeleteObj(AO_Tensors); DeleteObj(FF_Tensors); DeleteObj(FF_Weights); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of our tensore uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? qkv_weights : 3 * iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? w0 : out), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(((d == 0 || optimization == ADAM) ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::feedForward(CNeuronBaseOCL *NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = { outputs.Total() / (window_out * variables), window_out, variables }; setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_w, weights.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_i, inputs.GetIndex()); setBuffer(def_k_FeedForwardConv, def_k_ffc_matrix_o, outputs.GetIndex()); setArgument(def_k_FeedForwardConv, def_k_ffc_inputs, (int)(inputs.Total() / variables)); setArgument(def_k_FeedForwardConv, def_k_ffc_step, (int)step); setArgument(def_k_FeedForwardConv, def_k_ffc_window_in, (int)window); setArgument(def_k_FeedForwardConv, def_k_ffс_window_out, (int)window_out); setArgument(def_k_FeedForwardConv, def_k_ffc_activation, (int)activ); kernelExecute(def_k_FeedForwardConv, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHAttentionScore, def_k_mhas_mask, (int)mask); kernelExecute(def_k_MHAttentionScore, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHAttentionOut, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SumAndNormilize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() - shift_in1, tensor2.Total() - shift_in2), out.Total() - shift_out); if(size <= 0 || size < dimension) { PrintFormat("Tensor 1 total = %d -> shift = %d", tensor1.Total(), shift_in1); PrintFormat("Tensor 2 total = %d -> shift = %d", tensor2.Total(), shift_in2); PrintFormat("Output total = %d -> shift = %d", out.Total(), shift_out); PrintFormat("size=%d -> dimension=%d", size, dimension); ReturnFalse; } global_work_size[0] = size / dimension; const int kernel = def_k_MatrixSum; setBuffer(kernel, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_sum_matrix_out, out.GetIndex()) setArgument(kernel, def_k_sum_dimension, dimension) setArgument(kernel, def_k_sum_shift_in1, shift_in1) setArgument(kernel, def_k_sum_shift_in2, shift_in2) setArgument(kernel, def_k_sum_shift_out, shift_out) setArgument(kernel, def_k_sum_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) if(!normilize) return true; return Normilize(out, dimension); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Normilize(CBufferFloat *tensor, int dimension) { if(!OpenCL || !tensor) ReturnFalse; if(tensor.GetIndex() < 0 || dimension <= 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {tensor.Total() / dimension}; const int kernel = def_k_Normilize; setBuffer(kernel, def_k_norm_buffer, tensor.GetIndex()) setArgument(kernel, def_k_norm_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Different(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- int size = MathMin( MathMin(tensor1.Total() / (shift_in1 + dimension), tensor2.Total() / (shift_in2 + dimension)), out.Total() / (shift_out + dimension)); if(size < 1) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {size, dimension}; const int kernel = def_k_MatrixDif; setBuffer(kernel, def_k_dif_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_dif_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_dif_matrix_out, out.GetIndex()) setArgument(kernel, def_k_dif_shift_in1, shift_in1) setArgument(kernel, def_k_dif_shift_in2, shift_in2) setArgument(kernel, def_k_dif_shift_out, shift_out) setArgument(kernel, def_k_dif_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DifferentGrad(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- int size = MathMin( MathMin(tensor1.Total() / (shift_in1 + dimension), tensor2.Total() / (shift_in2 + dimension)), out.Total() / (shift_out + dimension)); if(size < 1) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {size, dimension}; const int kernel = def_k_MatrixDifGrad; setBuffer(kernel, def_k_dif_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_dif_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_dif_matrix_out, out.GetIndex()) setArgument(kernel, def_k_dif_shift_in1, shift_in1) setArgument(kernel, def_k_dif_shift_in2, shift_in2) setArgument(kernel, def_k_dif_shift_out, shift_out) setArgument(kernel, def_k_dif_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::IdentSum(CBufferFloat *tensor, CBufferFloat *out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- int size = MathMin(tensor.Total() / (shift_in + dimension), out.Total() / (shift_out + dimension)); if(size < 1) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {size, dimension}; const int kernel = def_k_IdentMatrixSum; setBuffer(kernel, def_k_idsum_matrix_in, tensor.GetIndex()) setBuffer(kernel, def_k_idsum_matrix_out, out.GetIndex()) setArgument(kernel, def_k_idsum_shift_in, shift_in) setArgument(kernel, def_k_idsum_shift_out, shift_out) setArgument(kernel, def_k_idsum_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::IdentDifferent(CBufferFloat * tensor, CBufferFloat * out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- int size = MathMin(tensor.Total() / (shift_in + dimension), out.Total() / (shift_out + dimension)); if(size < 1) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {size, dimension}; const int kernel = def_k_IdentMatrixDif; setBuffer(kernel, def_k_iddif_matrix_in, tensor.GetIndex()) setBuffer(kernel, def_k_iddif_matrix_out, out.GetIndex()) setArgument(kernel, def_k_iddif_shift_in, shift_in) setArgument(kernel, def_k_iddif_shift_out, shift_out) setArgument(kernel, def_k_iddif_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::IdentDifferentGrad(CBufferFloat * tensor, CBufferFloat * out, int dimension, int shift_in = 0, int shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- int size = MathMin(tensor.Total() / (shift_in + dimension), out.Total() / (shift_out + dimension)); if(size < 1) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {size, dimension}; const int kernel = def_k_IdentMatrixDifGrad; setBuffer(kernel, def_k_iddif_matrix_in, tensor.GetIndex()) setBuffer(kernel, def_k_iddif_matrix_out, out.GetIndex()) setArgument(kernel, def_k_iddif_shift_in, shift_in) setArgument(kernel, def_k_iddif_shift_out, shift_out) setArgument(kernel, def_k_iddif_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SumVecMatrix(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(vector_in) == POINTER_INVALID || CheckPointer(matrix_in) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(variables < 1) ReturnFalse; if(vector_in.GetIndex() < 0 || vector_in.Total() < int((shift_in1 + dimension)*variables)) ReturnFalse; if(matrix_in.GetIndex() < 0 || matrix_in.Total() < int((shift_in2 + dimension)*variables)) ReturnFalse; if(out.GetIndex() < 0 || out.Total() < int((shift_out + dimension)*variables)) ReturnFalse; //--- int size = MathMin(matrix_in.Total() / int((shift_in2 + dimension) * variables), out.Total() / int((shift_out + dimension) * variables)); if(size < 1) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3] = {size, dimension, variables}; const int kernel = def_k_SumVecMatrix; setBuffer(kernel, def_k_svecmat_vector_in, vector_in.GetIndex()) setBuffer(kernel, def_k_svecmat_matrix_in, matrix_in.GetIndex()) setBuffer(kernel, def_k_svecmat_matrix_out, out.GetIndex()) setArgument(kernel, def_k_svecmat_shift_in1, shift_in1) setArgument(kernel, def_k_svecmat_shift_in2, shift_in2) setArgument(kernel, def_k_svecmat_shift_out, shift_out) setArgument(kernel, def_k_svecmat_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SumVecMatrixGrad(CBufferFloat *vector_in, CBufferFloat *matrix_in, CBufferFloat *out, uint dimension, uint variables, uint shift_in1 = 0, uint shift_in2 = 0, uint shift_out = 0, float multiplyer = 1.0f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(vector_in) == POINTER_INVALID || CheckPointer(matrix_in) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(variables < 1) ReturnFalse; if(vector_in.GetIndex() < 0 || vector_in.Total() < int((shift_in1 + dimension)*variables)) ReturnFalse; if(matrix_in.GetIndex() < 0 || matrix_in.Total() < int((shift_in2 + dimension)*variables)) ReturnFalse; if(out.GetIndex() < 0 || out.Total() < int((shift_out + dimension)*variables)) ReturnFalse; //--- int size = MathMin(matrix_in.Total() / int((shift_in2 + dimension) * variables), out.Total() / int((shift_out + dimension) * variables)); if(size < 1) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3] = {size, dimension, variables}; const int kernel = def_k_SumVecMatrixGrad; setBuffer(kernel, def_k_svecmat_vector_in, vector_in.GetIndex()) setBuffer(kernel, def_k_svecmat_matrix_in, matrix_in.GetIndex()) setBuffer(kernel, def_k_svecmat_matrix_out, out.GetIndex()) setArgument(kernel, def_k_svecmat_shift_in1, shift_in1) setArgument(kernel, def_k_svecmat_shift_in2, shift_in2) setArgument(kernel, def_k_svecmat_shift_out, shift_out) setArgument(kernel, def_k_svecmat_multiplyer, multiplyer) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::MatMul(const CBufferFloat * matr1, const CBufferFloat * matr2, CBufferFloat * result, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {rows1, cols2, variables}; setBuffer(def_k_MatMult, def_k_mm_matr1, matr1.GetIndex()) setBuffer(def_k_MatMult, def_k_mm_matr2, matr2.GetIndex()) setBuffer(def_k_MatMult, def_k_mm_result, result.GetIndex()) setArgument(def_k_MatMult, def_k_mm_dimension, (int)cols1) setArgument(def_k_MatMult, def_k_mm_multvarsecond, (int)mult_var_second) //--- kernelExecute(def_k_MatMult, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::MatMulGrad(const CBufferFloat * matr1, CBufferFloat * matr1_gr, const CBufferFloat * matr2, CBufferFloat * matr2_gr, const CBufferFloat * result_gr, const int rows1, const int cols1, const int cols2, const int variables = 1, const bool mult_var_second = true) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {rows1, cols2, variables}; setBuffer(def_k_MatMultGrad, def_k_mmg_matr1, matr1.GetIndex()) setBuffer(def_k_MatMultGrad, def_k_mmg_matr1_gr, matr1_gr.GetIndex()) setBuffer(def_k_MatMultGrad, def_k_mmg_matr2, matr2.GetIndex()) setBuffer(def_k_MatMultGrad, def_k_mmg_matr2_gr, matr2_gr.GetIndex()) setBuffer(def_k_MatMultGrad, def_k_mmg_result_gr, result_gr.GetIndex()) setArgument(def_k_MatMultGrad, def_k_mmg_dimension, (int)cols1) setArgument(def_k_MatMultGrad, def_k_mmg_multvarsecond, (int)mult_var_second) //--- kernelExecute(def_k_MatMultGrad, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DiagMatMul(const CBufferFloat * diag, const CBufferFloat * matr, CBufferFloat * result, const int rows, const int cols, const int variables = 1, const int activat_func = None) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {rows, cols, variables}; uint local_work_size[3] = {1, cols, 1}; uint kernel = def_k_DiagMatMult; setBuffer(kernel, def_k_diagmm_diag, diag.GetIndex()) setBuffer(kernel, def_k_diagmm_matr, matr.GetIndex()) setBuffer(kernel, def_k_diagmm_result, result.GetIndex()) setArgument(kernel, def_k_diagmm_activation, (int)activat_func) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DiagMatMulGrad(const CBufferFloat * diag, CBufferFloat * diag_gr, const CBufferFloat * matr, CBufferFloat * matr_gr, const CBufferFloat * result_gr, const int rows, const int cols, const int variables = 1) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {rows, cols, variables}; uint local_work_size[3] = {1, cols, 1}; int kernel = def_k_DiagMatMultGrad; setBuffer(kernel, def_k_diagmmgr_diag, diag.GetIndex()) setBuffer(kernel, def_k_diagmmgr_grad_diag, diag_gr.GetIndex()) setBuffer(kernel, def_k_diagmmgr_matr, matr.GetIndex()) setBuffer(kernel, def_k_diagmmgr_grad_matr, matr_gr.GetIndex()) setBuffer(kernel, def_k_diagmmgr_grad_result, result_gr.GetIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SparseMatMul(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {sparse_rows, sparse_cols, full_cols}; uint local_work_size[3] = {1, global_work_size[1], 1}; uint kernel = def_k_SparseMatMult; setBuffer(kernel, def_k_smm_sparse_index, sparse_indexs.GetIndex()) setBuffer(kernel, def_k_smm_sparse_data, sparse_data.GetIndex()) setBuffer(kernel, def_k_smm_full, full.GetIndex()) setBuffer(kernel, def_k_smm_result, result.GetIndex()) setArgument(kernel, def_k_smm_full_rows, full_rows) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SparseMatMulGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {(uint)MathMax(sparse_rows, full_rows), (uint)MathMin(MathMax(sparse_cols, full_cols), (int)OpenCL.GetMaxLocalSize(1)), (uint)MathMax(sparse_cols, full_cols) }; uint local_work_size[3] = {1, global_work_size[1], 1}; uint kernel = def_k_SparseMatMultGrad; setBuffer(kernel, def_k_smm_gr_sparse_index, sparse_indexs.GetIndex()) setBuffer(kernel, def_k_smm_gr_sparse_data, sparse_data.GetIndex()) setBuffer(kernel, def_k_smm_gr_sparse_gr, sparse_grad.GetIndex()) setBuffer(kernel, def_k_smm_gr_full, full.GetIndex()) setBuffer(kernel, def_k_smm_gr_full_gr, full_gr.GetIndex()) setBuffer(kernel, def_k_smm_gr_result_gr, grad.GetIndex()) setArgument(kernel, def_k_smm_gr_sparse_rows, sparse_rows) setArgument(kernel, def_k_smm_gr_sparse_cols, sparse_cols) setArgument(kernel, def_k_smm_gr_full_rows, full_rows) setArgument(kernel, def_k_smm_gr_full_cols, full_cols) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SparseConcatenate(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, const CBufferFloat *full, const CBufferFloat *result, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {sparse_rows, sparse_cols, full_cols}; uint local_work_size[3] = {1, global_work_size[1], 1}; uint kernel = def_k_SparseConcatenate; setBuffer(kernel, def_k_sparconc_sparse_index, sparse_indexs.GetIndex()) setBuffer(kernel, def_k_sparconc_sparse_data, sparse_data.GetIndex()) setBuffer(kernel, def_k_sparconc_full, full.GetIndex()) setBuffer(kernel, def_k_sparconc_result, result.GetIndex()) setArgument(kernel, def_k_sparconc_full_rows, full_rows) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SparseConcatenateGrad(const CBufferFloat *sparse_indexs, const CBufferFloat *sparse_data, CBufferFloat *sparse_grad, const CBufferFloat *full, CBufferFloat *full_gr, const CBufferFloat *grad, const int sparse_rows, const int sparse_cols, const int full_rows, const int full_cols) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {(uint)MathMax(sparse_rows, full_rows), (uint)MathMin(MathMax(sparse_cols, full_cols), (int)OpenCL.GetMaxLocalSize(1)), (uint)MathMax(sparse_cols, full_cols) }; uint local_work_size[3] = {1, global_work_size[1], 1}; uint kernel = def_k_SparseConcatenateGrad; setBuffer(kernel, def_k_sparconc_gr_sparse_index, sparse_indexs.GetIndex()) setBuffer(kernel, def_k_sparconc_gr_sparse_data, sparse_data.GetIndex()) setBuffer(kernel, def_k_sparconc_gr_sparse_gr, sparse_grad.GetIndex()) setBuffer(kernel, def_k_sparconc_gr_full, full.GetIndex()) setBuffer(kernel, def_k_sparconc_gr_full_gr, full_gr.GetIndex()) setBuffer(kernel, def_k_sparconc_gr_result_gr, grad.GetIndex()) setArgument(kernel, def_k_sparconc_gr_sparse_rows, sparse_rows) setArgument(kernel, def_k_sparconc_gr_sparse_cols, sparse_cols) setArgument(kernel, def_k_sparconc_gr_full_rows, full_rows) setArgument(kernel, def_k_sparconc_gr_full_cols, full_cols) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ScalarToVector(CBufferFloat *scalar, CBufferFloat *vector_in, CBufferFloat *vector_out, int dimension) { if(!OpenCL) ReturnFalse; //--- if(!scalar || !vector_in || !vector_out) ReturnFalse; int total = scalar.Total() * dimension; if(vector_in.Total() < total || vector_out.Total() < total) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {(uint)scalar.Total(), (uint)dimension }; uint kernel = def_k_ScalarToVector; setBuffer(kernel, def_k_stv_scalar, scalar.GetIndex()) setBuffer(kernel, def_k_stv_vector_in, vector_in.GetIndex()) setBuffer(kernel, def_k_stv_vector_out, vector_out.GetIndex()) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ScalarToVectorGrad(CBufferFloat *scalar, CBufferFloat *scalar_gr, CBufferFloat *vector_in, CBufferFloat *vector_in_gr, CBufferFloat *vector_out_gr, int dimension) { if(!OpenCL) ReturnFalse; //--- if(!scalar || !vector_in || !vector_out_gr || !scalar_gr || !vector_in_gr) ReturnFalse; int total = scalar.Total() * dimension; if(vector_in.Total() < total || vector_in_gr.Total() < total || vector_out_gr.Total() < total || scalar.Total() > scalar_gr.Total()) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {(uint)scalar.Total(), (uint)MathMin(dimension, (int)OpenCL.GetMaxLocalSize(1)) }; uint local_work_size[] = {1, global_work_size[1]}; //--- uint kernel = def_k_ScalarToVectorGrad; setBuffer(kernel, def_k_stvg_scalar, scalar.GetIndex()) setBuffer(kernel, def_k_stvg_scalar_gr, scalar_gr.GetIndex()) setBuffer(kernel, def_k_stvg_vector_in, vector_in.GetIndex()) setBuffer(kernel, def_k_stvg_vector_in_gr, vector_in_gr.GetIndex()) setBuffer(kernel, def_k_stvg_vector_out_gr, vector_out_gr.GetIndex()) setArgument(kernel, def_k_stvg_dimension, dimension) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, GELU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutionInputGradients(CBufferFloat * weights, CBufferFloat * gradient, CBufferFloat * inputs, CBufferFloat * inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total(); global_work_size[1] = variables; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, weights.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, gradient.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, inputs.GetIndex()); setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, gradient.Total() - shift_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)step); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)window); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)window_out); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)activ); setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)shift_out); kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) //--- return true;//inp_gradient.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::AttentionInsideGradients(CBufferFloat * qkv, CBufferFloat * qkv_g, CBufferFloat * scores, CBufferFloat * gradient) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_score, scores.GetIndex()); setBuffer(def_k_MHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()); kernelExecute(def_k_MHAttentionGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads, 0, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat * weights, CBufferFloat * gradient, CBufferFloat * inputs, CBufferFloat * momentum1, CBufferFloat * momentum2, uint window, uint window_out, uint step = 0, uint heads = 0, uint variables = 1) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = weights.Total(); uint global_work_offset_am[3] = {0, 0, 0}; uint global_work_size_am[3] = {window, window_out, variables}; uint local_work_size_am[3] = {window, (heads > 0 ? window_out / heads : 1), variables}; if(weights.GetIndex() < 0) ReturnFalse; float lt = 0; switch(optimization) { case SGD: if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()); setBuffer(def_k_UpdateWeightsConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_inputs, inputs.Total()); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_learning_rates, lr); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_momentum, alpha); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_in, (int)window); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_window_out, (int)window_out); setArgument(def_k_UpdateWeightsConvMomentum, def_k_uwcm_step, (int)step); ResetLastError(); kernelExecute(def_k_UpdateWeightsConvMomentum, global_work_offset, global_work_size) break; case ADAM: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsConvAdam, global_work_offset, global_work_size) t++; break; case ADAM_MINI: if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsConvAdamMini, def_k_wucam_matrix_v, momentum2.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_inputs, inputs.Total()) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_l, lt) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b1, b1) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_b2, b2) setArgument(def_k_UpdateWeightsConvAdamMini, def_k_wucam_step, (int)step) ResetLastError(); kernelExecuteLoc(def_k_UpdateWeightsConvAdamMini, global_work_offset_am, global_work_size_am, local_work_size_am) t++; break; //--- default: printf("Error of optimization type %s: %s", __FUNCSIG__, EnumToString(optimization)); ReturnFalse; } //weights.BufferRead(); //--- global_work_size[0] = window_out; setBuffer(def_k_NormilizeWeights, def_k_norm_buffer, weights.GetIndex()); setArgument(def_k_NormilizeWeights, def_k_norm_dimension, (int)window + 1); kernelExecute(def_k_NormilizeWeights, global_work_offset, global_work_size) //--- return true;//weights.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayers, INT_VALUE) || !FileWriteInteger(file_handle, iHeads, INT_VALUE) || !FileWriteInteger(file_handle, iWindow, INT_VALUE) || !FileWriteInteger(file_handle, iUnits, INT_VALUE) || !FileWriteInteger(file_handle, iWindowKey, INT_VALUE)) ReturnFalse; //--- Saving objects if(!QKV_Tensors.Save(file_handle) || !QKV_Weights.Save(file_handle) || !S_Tensors.Save(file_handle) || !AO_Tensors.Save(file_handle) || !FF_Tensors.Save(file_handle) || !FF_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 6 - 4); Gradient = FF_Tensors.At(iLayers * 6 - 1); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); QKV_Tensors.SetOpenCL(OpenCL); QKV_Weights.SetOpenCL(OpenCL); S_Tensors.SetOpenCL(OpenCL); AO_Tensors.SetOpenCL(OpenCL); FF_Tensors.SetOpenCL(OpenCL); FF_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronMLMHAttentionOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; //--- result.window = (int)iWindow; result.step = (int)iHeads; result.window_out = (int)iWindowKey; result.count = (int)iUnits; result.layers = (int)iLayers; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ float CNeuronBaseOCL::GenerateWeight(void) { xor128; float result = (float)rnd_w / UINT_MAX; if(result == 0) result = GenerateWeight(); //--- return result; } //+------------------------------------------------------------------+ ///\ingroup neuron_base ///\class CNeuronDropoutOCL ///\brief The Dropout neuron for GPU calculation. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronDropoutOCL : public CNeuronBaseOCL { protected: CNeuronBaseOCL *PrevLayer; float OutProbability; int OutNumber; CBufferFloat *DropOutMultiplier; float dInitValue; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method.@param NeuronOCL Pointer to previos layer. //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) {return true;} ///< Blank method for updating weights.@param NeuronOCL Pointer to previos layer. //--- public: /** Constructor */ CNeuronDropoutOCL(void); /** Destructor */ ~CNeuronDropoutOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, float out_prob, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer #param[in] out_prob Probability of neurons shutdown @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int getOutputIndex(void) const override { return (bTrain ? Output.GetIndex() : PrevLayer.getOutputIndex()); } ///< Get index of output buffer @return Index virtual int getGradientIndex(void) const override { return (bTrain ? Gradient.GetIndex() : PrevLayer.getGradientIndex()); } ///< Get index of gradient buffer @return Index //--- virtual int getOutputVal(float &values[]) override { return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(vector &values) override { return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(CArrayFloat *values) override { return (bTrain ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getGradient(float &values[]) override { return (bTrain ? Gradient.GetData(values) : PrevLayer.getGradient(values)); } ///< Get values of gradient buffer @param[out] values Array of data @return number of items virtual int getGradient(vector &values) override { return (bTrain ? Gradient.GetData(values) : PrevLayer.getGradient(values)); } ///< Get values of gradient buffer @param[out] values Array of data @return number of items virtual CBufferFloat *getOutput(void) const override { return (bTrain ? Output : PrevLayer.getOutput()); } ///< Get pointer of output buffer @return Pointer to object virtual CBufferFloat *getGradient(void) const override { return (bTrain ? Gradient : PrevLayer.getGradient()); } ///< Get pointer of gradient buffer @return Pointer to object //--- virtual bool Save(int const file_handle) override ;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override ;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronDropoutOCL; }///< Identificator of class.@return Type of class virtual CLayerDescription* GetLayerInfo(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDropoutOCL::CNeuronDropoutOCL(void) : OutProbability((float)0.1), OutNumber(0), dInitValue(1.0) { PrevLayer = NULL; DropOutMultiplier = new CBufferFloat(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDropoutOCL::~CNeuronDropoutOCL(void) { DeleteObj(DropOutMultiplier); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDropoutOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, float out_prob, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; OutProbability = out_prob; OutNumber = (int)(numNeurons * out_prob); dInitValue = 1 / (1 - OutProbability); if(CheckPointer(DropOutMultiplier) == POINTER_INVALID) DropOutMultiplier = new CBufferFloat(); if(!DropOutMultiplier.BufferInit(numNeurons + 1, dInitValue)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDropoutOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- activation = (ENUM_ACTIVATION)NeuronOCL.Activation(); PrevLayer = NeuronOCL; if(!bTrain) return true; //--- if(CheckPointer(DropOutMultiplier) == POINTER_INVALID) DropOutMultiplier = new CBufferFloat(); if(!DropOutMultiplier.BufferInit(NeuronOCL.Neurons(), dInitValue)) ReturnFalse; for(int i = 0; i < OutNumber; i++) { uint p = RND(Neurons()); float val = DropOutMultiplier.At(p); if(val == 0 || val == DBL_MAX) { i--; continue; } if(!DropOutMultiplier.Update(RND(Neurons()), 0)) ReturnFalse; } //--- if(!DropOutMultiplier.BufferCreate(OpenCL)) ReturnFalse; return ElementMult(NeuronOCL.getOutput(), DropOutMultiplier, Output); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ElementMult(const CBufferFloat * input1, const CBufferFloat * input2, CBufferFloat * output) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = (output.Total() + 3) / 4; setBuffer(def_k_Dropout, def_k_dout_input, input1.GetIndex()) setBuffer(def_k_Dropout, def_k_dout_map, input2.GetIndex()) setBuffer(def_k_Dropout, def_k_dout_out, output.GetIndex()) setArgument(def_k_Dropout, def_k_dout_dimension, output.Total()) ResetLastError(); kernelExecute(def_k_Dropout, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::GateElementMult(const CBufferFloat * input1, const CBufferFloat * input2, const CBufferFloat * gate, CBufferFloat * output) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {output.Total()}; uint kernel = def_k_GateElementMul; setBuffer(kernel, def_k_gem_inputs1, input1.GetIndex()) setBuffer(kernel, def_k_gem_inputs2, input2.GetIndex()) setBuffer(kernel, def_k_gem_gate, gate.GetIndex()) setBuffer(kernel, def_k_gem_out, output.GetIndex()) ResetLastError(); kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::GateElementMultGrad(const CBufferFloat * input1, CBufferFloat * input1_gr, const CBufferFloat * input2, CBufferFloat * input2_gr, const CBufferFloat * gate, CBufferFloat * gate_gr, const CBufferFloat * output_gr, const int activation1, const int activation2, const int activation_gate) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = output_gr.Total(); setBuffer(def_k_GateElementMulGrad, def_k_gemg_inputs1, input1.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_inputs1_gr, input1_gr.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_inputs2, input2.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_inputs2_gr, input2_gr.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_gate, gate.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_gate_gr, gate_gr.GetIndex()) setBuffer(def_k_GateElementMulGrad, def_k_gemg_out, output_gr.GetIndex()) setArgument(def_k_GateElementMulGrad, def_k_gemg_activ1, int(activation1)) setArgument(def_k_GateElementMulGrad, def_k_gemg_activ2, int(activation2)) setArgument(def_k_GateElementMulGrad, def_k_gemg_activ_gr, int(activation_gate)) ResetLastError(); kernelExecute(def_k_GateElementMulGrad, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDropoutOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- if(!bTrain) return true; //--- if(CheckPointer(DropOutMultiplier) == POINTER_INVALID) ReturnFalse; //--- if(!DropOutMultiplier.BufferCreate(OpenCL)) ReturnFalse; //--- return ElementMult(Gradient, DropOutMultiplier, NeuronOCL.getGradient()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDropoutOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteDouble(file_handle, OutProbability) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDropoutOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- OutProbability = (float)FileReadDouble(file_handle); OutNumber = (int)(Neurons() * OutProbability); dInitValue = 1 / (1 - OutProbability); if(CheckPointer(DropOutMultiplier) == POINTER_INVALID) DropOutMultiplier = new CBufferFloat(); if(!DropOutMultiplier.BufferInit(Neurons() + 1, dInitValue)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronDropoutOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; //--- result.probability = OutProbability; //--- return result; } //+------------------------------------------------------------------+ ///\class CNeuronBatchNormOCL ///\brief The class of Batch Normalization neuron for GPU calculation. ///\details Detailed description on the link. //+------------------------------------------------------------------+ class CNeuronBatchNormOCL : public CNeuronBaseOCL { protected: CNeuronBaseOCL *PrevLayer; ///< Pointer to the object of the previous layer int iBatchSize; ///< Batch size int iBatchCount; ///< Batch count CBufferFloat *BatchOptions; ///< Container of method parameters ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method of calling kernel ::BatchFeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; ///< Method for updating weights.\details Calling one of kernels ::UpdateBatchOptionsMomentum() or ::UpdateBatchOptionsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradientBatch(). @param NeuronOCL Pointer to next layer. public: /** Constructor */ CNeuronBatchNormOCL(void); /** Destructor */~CNeuronBatchNormOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override; ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int getOutputIndex(void) { return (iBatchSize > 1 || !PrevLayer ? Output.GetIndex() : PrevLayer.getOutputIndex()); } ///< Get index of output buffer @return Index virtual int getGradientIndex(void) { return (iBatchSize > 1 || !PrevLayer ? Gradient.GetIndex() : PrevLayer.getGradientIndex()); } ///< Get index of gradient buffer @return Index //--- virtual int getOutputVal(float &values[]) { return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(CArrayFloat *values) { return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getOutputVal(vector &values) { return (iBatchSize > 1 || !PrevLayer ? Output.GetData(values) : PrevLayer.getOutputVal(values)); } ///< Get values of output buffer @param[out] values Array of data @return number of items virtual int getGradient(float &values[]) { return (iBatchSize > 1 || !PrevLayer ? Gradient.GetData(values) : PrevLayer.getGradient(values)); } ///< Get values of gradient buffer @param[out] values Array of data @return number of items virtual CBufferFloat *getOutput(void) { return (iBatchSize > 1 || !PrevLayer ? Output : PrevLayer.getOutput()); } ///< Get pointer of output buffer @return Pointer to object virtual CBufferFloat *getGradient(void) { return (iBatchSize > 1 || !PrevLayer ? Gradient : PrevLayer.getGradient()); } ///< Get pointer of gradient buffer @return Pointer to object virtual CBufferFloat *getBatchOptions(void) { return (iBatchSize > 1 ? BatchOptions : NULL); } ///< Get pointer of Batch Options buffer @return Pointer to object //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronBatchNormOCL; }///< Identificator of class.@return Type of class virtual CLayerDescription* GetLayerInfo(void); virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual CBufferFloat *getWeights(void) override { return BatchOptions; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBatchNormOCL::CNeuronBatchNormOCL(void) : iBatchSize(1) { PrevLayer = NULL; BatchOptions = NULL; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBatchNormOCL::~CNeuronBatchNormOCL(void) { if(CheckPointer(PrevLayer) != POINTER_INVALID) PrevLayer = NULL; DeleteObj(BatchOptions); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return Init(numOutputs, myIndex, OpenCL, numNeurons, iBatch, optimization_type); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batchSize)) ReturnFalse; activation = None; iBatchSize = (int)batchSize; //--- DeleteObj(BatchOptions); int count = (int)numNeurons * (optimization_type == SGD ? 7 : 9); BatchOptions = new CBufferFloat(); if(!BatchOptions.BufferInit(count, 0.0f)) ReturnFalse; if(!BatchOptions.BufferCreate(OpenCL)) ReturnFalse; iBatchCount = 1; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- PrevLayer = NeuronOCL; if(iBatchSize <= 1) { activation = (ENUM_ACTIVATION)NeuronOCL.Activation(); return true; } //--- if(CheckPointer(BatchOptions) == POINTER_INVALID) { int count = Neurons() * (optimization == SGD ? 7 : 9); BatchOptions = new CBufferFloat(); if(!BatchOptions.BufferInit(count, 0)) ReturnFalse; } //if(!BatchOptions.BufferCreate(OpenCL)) // ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); iBatchCount = MathMin(iBatchCount, iBatchSize); setBuffer(def_k_BatchFeedForward, def_k_bff_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_options, BatchOptions.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_output, Output.GetIndex()) setArgument(def_k_BatchFeedForward, def_k_bff_batch, (int)iBatchCount) setArgument(def_k_BatchFeedForward, def_k_bff_optimization, (int)optimization) setArgument(def_k_BatchFeedForward, def_k_bff_activation, (int)activation) ResetLastError(); kernelExecute(def_k_BatchFeedForward, global_work_offset, global_work_size) iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- if(iBatchSize <= 1) return (CheckPointer(PrevLayer) != POINTER_INVALID); //--- if(CheckPointer(BatchOptions) == POINTER_INVALID || BatchOptions.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_options, BatchOptions.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_g, Gradient.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_activation, NeuronOCL.Activation()) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_batch, (int)iBatchCount) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_optimization, (int)optimization) ResetLastError(); kernelExecute(def_k_CalcHiddenGradientBatch, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- if(iBatchSize <= 1) return (CheckPointer(PrevLayer) != POINTER_INVALID); //--- if(CheckPointer(BatchOptions) == POINTER_INVALID || BatchOptions.GetIndex() < 0) ReturnFalse; if(iBatchCount < 100 * iBatchSize) return true; uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); //--- if(optimization == SGD) { setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_options, BatchOptions.GetIndex()) setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_matrix_g, Gradient.GetIndex()) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_learning_rates, lr) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_momentum, alpha) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsMomentum, global_work_offset, global_work_size) } else { setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_options, BatchOptions.GetIndex()) setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_matrix_g, Gradient.GetIndex()) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_l, lr) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b1, b1) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b2, b2) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsAdam, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, iBatchSize, INT_VALUE) <= 0) ReturnFalse; return BatchOptions.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iBatchSize = FileReadInteger(file_handle, INT_VALUE); if(!BatchOptions.Load(file_handle)) ReturnFalse; if(!!OpenCL && !BatchOptions.BufferCreate(OpenCL)) ReturnFalse; iBatchCount = iBatchSize; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronBatchNormOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; //--- result.batch = (int)iBatchSize; //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBatchNormOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!BatchOptions) BatchOptions.BufferCreate(obj); ///< Container of method parameters } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::GetLayerOutput(uint layer, CBufferFloat *&result) { if(!layers || layers.Total() <= (int)layer) ReturnFalse; CLayer *Layer = layers.At(layer); if(!Layer) ReturnFalse; //--- if(!result) { result = new CBufferFloat(); if(!result) ReturnFalse; } //--- CNeuronBaseOCL *temp = Layer.At(0); if(!temp || temp.getOutputVal(result) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::GetLayerOutput(uint layer, vector &result) { if(!layers || layers.Total() <= (int)layer) ReturnFalse; CLayer *Layer = layers.At(layer); if(!Layer) ReturnFalse; //--- //--- CNeuronBaseOCL *temp = Layer.At(0); if(!temp || temp.getOutputVal(result) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::SetOpenCL(COpenCLMy * obj) { if(opencl != obj) DeleteObj(opencl); //--- if(!obj || !layers) return; //--- opencl = obj; for(int i = 0; i < layers.Total(); i++) { CLayer *layer = layers.At(i); if(!layer) continue; layer.SetOpenCL(obj); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLSTMOCL : public CNeuronBaseOCL { protected: CBufferFloat m_cWeightsLSTM; CBufferFloat m_cFirstMomentumLSTM; ///< Buffer of first momentum matrix (#ADAM) or last delta weights matrix (#SGD) CBufferFloat m_cSecondMomentumLSTM; ///< Buffer of second momentum matrix (#ADAM) int m_iMemory; int m_iConcatenated; int m_iConcatenatedGradient; int m_iHiddenState; int m_iWeightsGradient; int m_iInputs; int m_iVariables; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method.@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; ///< Method for updating weights. //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronLSTMOCL(void); ~CNeuronLSTMOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool SetInputs(int count); ///@} //--- virtual bool Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) override const { return defNeuronLSTMOCL; }///< Identificator of class.@return Type of class virtual bool Clear(void); virtual CBufferFloat *getLSTMWeights(void) { return GetPointer(m_cWeightsLSTM);} virtual void SetOpenCL(COpenCLMy *obj); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::CNeuronLSTMOCL(void) : m_iMemory(-1), m_iConcatenated(-1), m_iConcatenatedGradient(-1), m_iHiddenState(-1), m_iWeightsGradient(-1), m_iInputs(-1), m_iVariables(1) {} //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronLSTMOCL::~CNeuronLSTMOCL(void) { if(!OpenCL) return; OpenCL.BufferFree(m_iConcatenated); OpenCL.BufferFree(m_iConcatenatedGradient); OpenCL.BufferFree(m_iHiddenState); OpenCL.BufferFree(m_iMemory); OpenCL.BufferFree(m_iWeightsGradient); m_cFirstMomentumLSTM.BufferFree(); m_cSecondMomentumLSTM.BufferFree(); m_cWeightsLSTM.BufferFree(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return Init(numOutputs, myIndex, open_cl, numNeurons, 1, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons * variables, optimization_type, batch)) ReturnFalse; //--- m_iVariables = int(variables); m_iMemory = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 2, CL_MEM_READ_WRITE); if(m_iMemory < 0) ReturnFalse; m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables), CL_MEM_READ_WRITE); if(m_iHiddenState < 0) ReturnFalse; m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 4, CL_MEM_READ_WRITE); if(m_iConcatenated < 0) ReturnFalse; m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * (numNeurons * m_iVariables) * 4, CL_MEM_READ_WRITE); if(m_iConcatenatedGradient < 0) ReturnFalse; if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() <= 0 || NeuronOCL.getOutputIndex() < 0 || !OpenCL) ReturnFalse; //--- if(m_iInputs != NeuronOCL.Neurons() / m_iVariables) { if(!SetInputs(NeuronOCL.Neurons() / m_iVariables)) ReturnFalse; } //--- if(m_iMemory < 0 || m_iConcatenated < 0) ReturnFalse; //--- setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_concatenated, m_iConcatenated) setArgument(def_k_LSTM_FeedForward, def_k_lstmff_inputs_size, m_iInputs) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_memory, m_iMemory) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_outputs, getOutputIndex()) setBuffer(def_k_LSTM_FeedForward, def_k_lstmff_weights, m_cWeightsLSTM.GetIndex()) uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {Neurons() / m_iVariables, 4, m_iVariables}; uint local_work_size[] = {1, 4, 1}; kernelExecuteLoc(def_k_LSTM_FeedForward, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::SetInputs(int count) { m_iInputs = count; count = (int)((m_iInputs + Neurons() / m_iVariables + 1) * (Neurons() / m_iVariables) * 4); if(!m_cWeightsLSTM.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(Neurons() + 1)); for(int i = 0; i < count; i++) { if(!m_cWeightsLSTM.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!m_cWeightsLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(!m_cFirstMomentumLSTM.BufferInit(count, 0)) ReturnFalse; if(!m_cFirstMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(!m_cSecondMomentumLSTM.BufferInit(count, 0)) ReturnFalse; if(!m_cSecondMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; if(m_iWeightsGradient >= 0) OpenCL.BufferFree(m_iWeightsGradient); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * count, CL_MEM_READ_WRITE); if(m_iWeightsGradient < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() <= 0 || NeuronOCL.getGradientIndex() < 0 || NeuronOCL.getOutputIndex() < 0 || !OpenCL) ReturnFalse; //--- if(m_cWeightsLSTM.GetIndex() < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; if(m_iInputs < 0 || m_iConcatenated < 0 || m_iMemory < 0 || m_iConcatenatedGradient < 0 || m_iHiddenState < 0 || m_iInputs != NeuronOCL.Neurons() / m_iVariables) ReturnFalse; //--- setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_concatenated, m_iConcatenated) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_concatenated_gradient, m_iConcatenatedGradient) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_gradient, getGradientIndex()) setBuffer(def_k_LSTM_ConcatenatedGradient, def_k_lstmcg_memory, m_iMemory) uint global_work_offset[] = {0, 0}; uint global_work_size[] = {Neurons() / m_iVariables, m_iVariables}; kernelExecute(def_k_LSTM_ConcatenatedGradient, global_work_offset, global_work_size) vector temp = vector::Zeros(Neurons() * 4); if(!OpenCL.BufferToVector(m_iConcatenatedGradient, temp, temp.Size())) Sleep(0); //--- uint local_work_size[] = {1, m_iVariables}; setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_concatenated_gradient, m_iConcatenatedGradient) setArgument(def_k_LSTM_HiddenGradient, def_k_lstmhg_hidden_size, Neurons() / m_iVariables) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_hidden_state, m_iHiddenState) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs_gradient, NeuronOCL.getGradientIndex()) setArgument(def_k_LSTM_HiddenGradient, def_k_lstmhg_inputs_size, m_iInputs) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_output, getOutputIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_weeights, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_LSTM_HiddenGradient, def_k_lstmhg_weights_gradient, m_iWeightsGradient) kernelExecuteLoc(def_k_LSTM_HiddenGradient, global_work_offset, global_work_size, local_work_size) //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || m_cWeightsLSTM.GetIndex() < 0 || m_iWeightsGradient < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_weights, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_weights_gradient, m_iWeightsGradient) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_matrix_m, m_cFirstMomentumLSTM.GetIndex()) setBuffer(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_matrix_v, m_cSecondMomentumLSTM.GetIndex()) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_l, lr) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_b1, b1) setArgument(def_k_LSTM_UpdateWeightsAdam, def_k_lstmuw_b2, b2) uint global_work_offset[] = {0, 0}; uint global_work_size[] = {(m_iInputs + Neurons()) / m_iVariables + 1, Neurons() / m_iVariables}; kernelExecute(def_k_LSTM_UpdateWeightsAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronLSTMOCL *Source = source; if(!OpenCL || m_cWeightsLSTM.GetIndex() < 0 || m_cFirstMomentumLSTM.GetIndex() < 0 || m_cSecondMomentumLSTM.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {m_cWeightsLSTM.Total()}; ResetLastError(); setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, m_cWeightsLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, Source.m_cWeightsLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, m_cFirstMomentumLSTM.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, m_cSecondMomentumLSTM.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, m_iInputs, INT_VALUE) < sizeof(m_iInputs)) ReturnFalse; if(FileWriteInteger(file_handle, m_iVariables, INT_VALUE) < sizeof(m_iInputs)) ReturnFalse; if((m_cWeightsLSTM.GetIndex() >= 0 && !m_cWeightsLSTM.BufferRead()) || !m_cWeightsLSTM.Save(file_handle)) ReturnFalse; if((m_cFirstMomentumLSTM.GetIndex() >= 0 && !m_cFirstMomentumLSTM.BufferRead()) || !m_cFirstMomentumLSTM.Save(file_handle)) ReturnFalse; if((m_cSecondMomentumLSTM.GetIndex() >= 0 && !m_cSecondMomentumLSTM.BufferRead()) || !m_cSecondMomentumLSTM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; m_iInputs = FileReadInteger(file_handle); m_iVariables = FileReadInteger(file_handle); //--- m_cWeightsLSTM.BufferFree(); if(!m_cWeightsLSTM.Load(file_handle) || !m_cWeightsLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- m_cFirstMomentumLSTM.BufferFree(); if(!m_cFirstMomentumLSTM.Load(file_handle) || !m_cFirstMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- m_cSecondMomentumLSTM.BufferFree(); if(!m_cSecondMomentumLSTM.Load(file_handle) || !m_cSecondMomentumLSTM.BufferCreate(OpenCL)) ReturnFalse; //--- if(m_iMemory >= 0) OpenCL.BufferFree(m_iMemory); m_iMemory = OpenCL.AddBuffer(sizeof(float) * 2 * Neurons(), CL_MEM_READ_WRITE); if(m_iMemory < 0) ReturnFalse; //--- if(m_iConcatenated >= 0) OpenCL.BufferFree(m_iConcatenated); m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * 4 * Neurons(), CL_MEM_READ_WRITE); if(m_iConcatenated < 0) ReturnFalse; //--- if(m_iConcatenatedGradient >= 0) OpenCL.BufferFree(m_iConcatenatedGradient); m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * 4 * Neurons(), CL_MEM_READ_WRITE); if(m_iConcatenatedGradient < 0) ReturnFalse; //--- if(m_iHiddenState >= 0) OpenCL.BufferFree(m_iHiddenState); m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * Neurons(), CL_MEM_READ_WRITE); if(m_iHiddenState < 0) ReturnFalse; //--- if(m_iWeightsGradient >= 0) OpenCL.BufferFree(m_iWeightsGradient); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * m_cWeightsLSTM.Total(), CL_MEM_READ_WRITE); if(m_iWeightsGradient < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLSTMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- float emp[]; ArrayResize(emp, Neurons() * 2); ArrayInitialize(emp, 0); if(!OpenCL.BufferWrite(m_iHiddenState, emp, 0, 0, Neurons())) ReturnFalse; if(!OpenCL.BufferWrite(m_iMemory, emp, 0, 0, Neurons() * 2)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLSTMOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { OpenCL.BufferFree(m_iMemory); OpenCL.BufferFree(m_iHiddenState); OpenCL.BufferFree(m_iConcatenated); OpenCL.BufferFree(m_iConcatenatedGradient); } CNeuronBaseOCL::SetOpenCL(obj); m_cWeightsLSTM.BufferCreate(OpenCL); m_cFirstMomentumLSTM.BufferCreate(OpenCL); m_cSecondMomentumLSTM.BufferCreate(OpenCL); m_iWeightsGradient = OpenCL.AddBuffer(sizeof(float) * m_cWeightsLSTM.Total(), CL_MEM_READ_WRITE); int numNeurons = Neurons(); m_iMemory = OpenCL.AddBuffer(sizeof(float) * numNeurons * 2, CL_MEM_READ_WRITE); m_iHiddenState = OpenCL.AddBuffer(sizeof(float) * numNeurons, CL_MEM_READ_WRITE); m_iConcatenated = OpenCL.AddBuffer(sizeof(float) * numNeurons * 4, CL_MEM_READ_WRITE); m_iConcatenatedGradient = OpenCL.AddBuffer(sizeof(float) * numNeurons * 4, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #ifndef class_vae #include "..\Unsupervised\AE\VAE.mqh" #endif //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSoftMaxOCL : public CNeuronBaseOCL { protected: uint iHeads; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronSoftMaxOCL(void) : iHeads(1) { activation = None; } ~CNeuronSoftMaxOCL(void) {}; //--- ///\ingroup neuron_base_gr ///@{ virtual bool calcOutputGradients(CBufferFloat *Target, float &error) override; ///< Method of output gradients calculation by calling kernel ::CalcOutputGradient().@param Target Traget value virtual void SetHeads(int heads) { iHeads = heads; } ///@} //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual CLayerDescription* GetLayerInfo(void) override; virtual int Type(void) override const { return defNeuronSoftMaxOCL; }///< Identificator of class.@return Type of class virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; virtual uint Heads(void) const { return iHeads; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftMaxOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint size = Output.Total() / iHeads; uint global_work_size[2] = { size, iHeads }; uint local_work_size[2] = { size, 1 }; setBuffer(def_k_SoftMax_FeedForward, def_k_softmaxff_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_SoftMax_FeedForward, def_k_softmaxff_outputs, getOutputIndex()) kernelExecuteLoc(def_k_SoftMax_FeedForward, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftMaxOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint size = Output.Total() / iHeads; uint global_work_size[2] = { size, iHeads }; uint local_work_size[2] = { size, 1 }; setBuffer(def_k_SoftMax_HiddenGradient, def_k_softmaxhg_input_gr, NeuronOCL.getGradientIndex()); setBuffer(def_k_SoftMax_HiddenGradient, def_k_softmaxhg_output_gr, getGradientIndex()); setBuffer(def_k_SoftMax_HiddenGradient, def_k_softmaxhg_outputs, getOutputIndex()); kernelExecuteLoc(def_k_SoftMax_HiddenGradient, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftMaxOCL::calcOutputGradients(CBufferFloat * Target, float & error) { if(!OpenCL || !Target) ReturnFalse; //--- if(!Output.BufferRead()) ReturnFalse; if(Output.Total() != Target.Total()) ReturnFalse; error = 0; for(int i = 0; i < Output.Total(); i++) { if(Output[i] > 0) error += -MathAbs(Target[i]) * MathLog(Output[i]); Gradient.Update(i, Target[i]); } error /= (float)iHeads; if(!Gradient.BufferWrite()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); setBuffer(def_k_SoftMax_OutputGradient, def_k_softmaxog_targets, getGradientIndex()); setBuffer(def_k_SoftMax_OutputGradient, def_k_softmaxog_output_gr, getGradientIndex()); setBuffer(def_k_SoftMax_OutputGradient, def_k_softmaxog_outputs, getOutputIndex()); kernelExecute(def_k_SoftMax_OutputGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronSoftMaxOCL::GetLayerInfo(void) { CLayerDescription* result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; result.step = (int)iHeads; result.count = (int)(Output.Total() / iHeads); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftMaxOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iHeads) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftMaxOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(iHeads <= 0) iHeads = 1; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFQF : protected CNeuronBaseOCL { protected: //--- Fractal Net CNeuronBaseOCL cFraction; CNeuronSoftMaxOCL cSoftMax; //--- Cosine embeding CNeuronBaseOCL cCosine; CNeuronBaseOCL cCosineEmbeding; //--- Quantile Net CNeuronBaseOCL cQuantile0; CNeuronBaseOCL cQuantile1; CNeuronBaseOCL cQuantile2; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronFQF(); ~CNeuronFQF(); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual bool Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) override const { return defNeuronFQF; }///< Identificator of class.@return Type of class virtual CLayerDescription* GetLayerInfo(void) override; virtual CObject* AsObject(void) { return GetPointer(this); } virtual void SetOpenCL(COpenCLMy *obj); virtual void SetActivationFunction(ENUM_ACTIVATION function) { CNeuronBaseOCL::SetActivationFunction(function); } virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronFQF::CNeuronFQF(void) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronFQF::~CNeuronFQF(void) { } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, actions, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- if(!cFraction.Init(0, myIndex, open_cl, actions * quantiles, optimization, batch)) ReturnFalse; cFraction.SetActivationFunction(None); //--- if(!cSoftMax.Init(0, myIndex, open_cl, actions * quantiles, optimization, batch)) ReturnFalse; cSoftMax.SetHeads(actions); cSoftMax.SetActivationFunction(None); //--- if(!cCosine.Init(numInputs, myIndex, open_cl, actions * quantiles, optimization, batch)) ReturnFalse; cCosine.SetActivationFunction(None); //--- if(!cCosineEmbeding.Init(0, myIndex, open_cl, numInputs, optimization, batch)) ReturnFalse; cCosineEmbeding.SetActivationFunction(LReLU); //--- if(!cQuantile0.Init(4 * actions * quantiles, myIndex, open_cl, numInputs, optimization, batch)) ReturnFalse; cQuantile0.SetActivationFunction(None); //--- if(!cQuantile1.Init(actions * quantiles, myIndex, open_cl, 4 * actions * quantiles, optimization, batch)) ReturnFalse; cQuantile1.SetActivationFunction(LReLU); //--- if(!cQuantile2.Init(0, myIndex, open_cl, actions * quantiles, optimization, batch)) ReturnFalse; cQuantile2.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cFraction.FeedForward(NeuronOCL)) ReturnFalse; if(!cSoftMax.FeedForward(GetPointer(cFraction))) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[1] = Output.Total(); global_work_size[0] = cSoftMax.Neurons() / global_work_size[1]; setBuffer(def_k_FQF_Cosine, def_k_fqf_cosine_softmax, cSoftMax.getOutputIndex()); setBuffer(def_k_FQF_Cosine, def_k_fqf_cosine_outputs, cCosine.getOutputIndex()); kernelExecute(def_k_FQF_Cosine, global_work_offset, global_work_size) } //--- if(!cCosineEmbeding.FeedForward(GetPointer(cCosine))) ReturnFalse; //--- if(!ElementMult(NeuronOCL.getOutput(), cCosineEmbeding.getOutput(), cQuantile0.getOutput())) ReturnFalse; //--- if(!cQuantile1.FeedForward(GetPointer(cQuantile0))) ReturnFalse; //--- if(!cQuantile2.FeedForward(GetPointer(cQuantile1))) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; setBuffer(def_k_FQF_Output, def_k_fqfout_quantiles, cQuantile2.getOutputIndex()); setBuffer(def_k_FQF_Output, def_k_fqfout_delta_taus, cSoftMax.getOutputIndex()); setBuffer(def_k_FQF_Output, def_k_fqfout_output, getOutputIndex()); setArgument(def_k_FQF_Output, def_k_fqfout_total, (int)(cQuantile2.Neurons() / global_work_size[0])); kernelExecute(def_k_FQF_Output, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !Gradient || !Output) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { cSoftMax.Neurons() / Neurons(), Neurons() }; setBuffer(def_k_FQF_OutputGradient, def_k_fqfoutgr_quantiles, cQuantile2.getOutputIndex()); setBuffer(def_k_FQF_OutputGradient, def_k_fqfoutgr_taus, cSoftMax.getOutputIndex()); setBuffer(def_k_FQF_OutputGradient, def_k_fqfoutgr_output_gr, getGradientIndex()); setBuffer(def_k_FQF_OutputGradient, def_k_fqfoutgr_quantiles_gr, cQuantile2.getGradientIndex()); setBuffer(def_k_FQF_OutputGradient, def_k_fqfoutgr_taus_gr, cSoftMax.getGradientIndex()); kernelExecute(def_k_FQF_OutputGradient, global_work_offset, global_work_size) } //--- if(!cQuantile2.CalcHiddenGradients(cQuantile1.AsObject())) ReturnFalse; if(!cQuantile1.CalcHiddenGradients(cQuantile0.AsObject())) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { cCosineEmbeding.Neurons(), 1 }; setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_state_enbeding, NeuronOCL.getOutputIndex()); setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_taus_embedding, cCosineEmbeding.getOutputIndex()); setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_quantiles_gr, cQuantile0.getGradientIndex()); setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_state_gr, NeuronOCL.getGradientIndex()); setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_taus_gr, cCosineEmbeding.getGradientIndex()); kernelExecute(def_k_FQF_QuantileGradient, global_work_offset, global_work_size) } //--- if(!cCosineEmbeding.CalcHiddenGradients(cCosine.AsObject())) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { cSoftMax.Neurons() / Neurons(), Neurons() }; setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_softmax, cSoftMax.getOutputIndex()); setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_output_gr, cCosine.getGradientIndex()); setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_softmax_gr, cSoftMax.getGradientIndex()); kernelExecute(def_k_FQF_CosineGradient, global_work_offset, global_work_size) } //--- //cSoftMax.getGradient().BufferRead(); if(!cFraction.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; //cFraction.getGradient().BufferRead(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cFraction.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cCosineEmbeding.UpdateInputWeights(GetPointer(cCosine))) ReturnFalse; if(!cQuantile1.UpdateInputWeights(GetPointer(cQuantile0))) ReturnFalse; if(!cQuantile2.UpdateInputWeights(GetPointer(cQuantile1))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cFraction.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cCosine.Save(file_handle)) ReturnFalse; if(!cCosineEmbeding.Save(file_handle)) ReturnFalse; if(!cQuantile0.Save(file_handle)) ReturnFalse; if(!cQuantile1.Save(file_handle)) ReturnFalse; if(!cQuantile2.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cFraction.Type() || !cFraction.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cSoftMax.Type() || !cSoftMax.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cCosine.Type() || !cCosine.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cCosineEmbeding.Type() || !cCosineEmbeding.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cQuantile0.Type() || !cQuantile0.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cQuantile1.Type() || !cQuantile1.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cQuantile2.Type() || !cQuantile2.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CLayerDescription* CNeuronFQF::GetLayerInfo(void) { CLayerDescription *result = CNeuronBaseOCL::GetLayerInfo(); if(!result) return result; result.window_out = cSoftMax.Neurons() / Neurons(); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFQF::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cFraction.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cCosine.SetOpenCL(OpenCL); cCosineEmbeding.SetOpenCL(OpenCL); cQuantile0.SetOpenCL(OpenCL); cQuantile1.SetOpenCL(OpenCL); cQuantile2.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::getResults(vector &resultVals) { CBufferFloat* temp; getResults(temp); temp.GetData(resultVals); DeleteObj(temp); return; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNet::getResults(float & resultVals[]) { CBufferFloat* temp; getResults(temp); if(!temp.GetData(resultVals)) { DeleteObj(temp); return; } DeleteObj(temp); return; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMLMHSparseAttention : public CNeuronMLMHAttentionOCL { protected: float m_dSparse; //--- virtual bool AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = true); ///< \brief Multi-heads attention scores method of calling kernel ::MHAttentionScore(). virtual bool AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out); ///< \brief Multi-heads attention out method of calling kernel ::MHAttentionOut(). public: CNeuronMLMHSparseAttention(void) : m_dSparse(0.3f) {}; ~CNeuronMLMHSparseAttention(void) {}; //--- void Sparse(float value) { m_dSparse = value;} float Sparse(void) { return m_dSparse; } virtual int Type(void) const { return defNeuronMLMHSparseAttentionOCL; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionScore(CBufferFloat * qkv, CBufferFloat * scores, bool mask = true) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionScore, def_k_mhas_score, scores.GetIndex()); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_dimension, (int)iWindowKey); setArgument(def_k_MHSparseAttentionScore, def_k_mhas_sparse, m_dSparse); kernelExecute(def_k_MHSparseAttentionScore, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::AttentionOut(CBufferFloat * qkv, CBufferFloat * scores, CBufferFloat * out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_qkv, qkv.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_score, scores.GetIndex()); setBuffer(def_k_MHSparseAttentionOut, def_k_mhao_out, out.GetIndex()); setArgument(def_k_MHSparseAttentionOut, def_k_mhao_dimension, (int)iWindowKey); kernelExecute(def_k_MHSparseAttentionOut, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteFloat(file_handle, m_dSparse) < sizeof(float)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSparseAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; m_dSparse = FileReadFloat(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiModel : public CNeuronBaseOCL { protected: int iModels; int iUpdateModel; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. public: CNeuronMultiModel(void) {}; ~CNeuronMultiModel(void) {}; virtual bool Init(uint numInputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models); //--- ///\ingroup neuron_base_gr ///@{ virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. ///@} //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronMultiModels; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Init(uint numInputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, int models) { if(CheckPointer(open_cl) == POINTER_INVALID || numNeurons <= 0) ReturnFalse; OpenCL = open_cl; optimization = ADAM; iBatch = 1; iModels = models; //--- if(CheckPointer(Output) == POINTER_INVALID) { Output = new CBufferFloat(); if(CheckPointer(Output) == POINTER_INVALID) ReturnFalse; } if(!Output.BufferInit(numNeurons * models, 0.0)) ReturnFalse; if(!Output.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(PrevOutput) == POINTER_INVALID) { PrevOutput = new CBufferFloat(); if(CheckPointer(PrevOutput) == POINTER_INVALID) ReturnFalse; } if(!PrevOutput.BufferInit(numNeurons * models, 1.0)) ReturnFalse; if(!PrevOutput.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Gradient) == POINTER_INVALID) { Gradient = new CBufferFloat(); if(CheckPointer(Gradient) == POINTER_INVALID) ReturnFalse; } if(!Gradient.BufferInit((numNeurons + 1)*models, 0.0)) ReturnFalse; if(!Gradient.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(Weights) == POINTER_INVALID) { Weights = new CBufferFloat(); if(CheckPointer(Weights) == POINTER_INVALID) ReturnFalse; } int count = (int)((numInputs + 1) * numNeurons * models); if(!Weights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(numInputs + 1)); for(int i = 0; i < count; i++) { if(!Weights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!Weights.BufferCreate(OpenCL)) ReturnFalse; //--- DeleteObj(DeltaWeights); //--- if(CheckPointer(FirstMomentum) == POINTER_INVALID) { FirstMomentum = new CBufferFloat(); if(CheckPointer(FirstMomentum) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!FirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentum) == POINTER_INVALID) { SecondMomentum = new CBufferFloat(); if(CheckPointer(SecondMomentum) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentum.BufferInit(count, 0)) ReturnFalse; if(!SecondMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Output.Total() / iModels; global_work_size[1] = iModels; setBuffer(def_k_FFMultiModels, def_k_ff_matrix_w, getWeightsIndex()) setBuffer(def_k_FFMultiModels, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_FFMultiModels, def_k_ff_matrix_o, Output.GetIndex()) setArgument(def_k_FFMultiModels, def_k_ff_inputs, NeuronOCL.Neurons() / iModels) setArgument(def_k_FFMultiModels, def_k_ff_activation, (int)activation) kernelExecute(def_k_FFMultiModels, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::calcHiddenGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iModels; global_work_size[1] = iModels; setBuffer(def_k_HGMultiModels, def_k_chg_matrix_w, getWeightsIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_g, getGradientIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(def_k_HGMultiModels, def_k_chg_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(def_k_HGMultiModels, def_k_chg_outputs, Neurons() / iModels) setArgument(def_k_HGMultiModels, def_k_chg_activation, NeuronOCL.Activation()) iUpdateModel = (int)MathRound(MathRand() / 32767.0 * (iModels - 1)); setArgument(def_k_HGMultiModels, def_k_chg_model, iUpdateModel) //Comment(com+"\n "+(string)__LINE__+"-"__FUNCTION__); kernelExecute(def_k_HGMultiModels, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons() / iModels; global_work_size[1] = NeuronOCL.Neurons() / iModels + 1; uint rest = 0; float lt = lr; setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_w, getWeightsIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_g, getGradientIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_m, getFirstMomentumIndex()) setBuffer(def_k_UWMultiModels, def_k_uwa_matrix_v, getSecondMomentumIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_UWMultiModels, def_k_uwa_inputs, NeuronOCL.Neurons() / iModels) setArgument(def_k_UWMultiModels, def_k_uwa_l, lt) setArgument(def_k_UWMultiModels, def_k_uwa_b1, b1) setArgument(def_k_UWMultiModels, def_k_uwa_b2, b2) setArgument(def_k_UWMultiModels, def_k_uwa_model, iUpdateModel) global_work_size[1] = (global_work_size[1] + 3) / 4; ResetLastError(); kernelExecute(def_k_UWMultiModels, global_work_offset, global_work_size) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iModels) <= 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiModel::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; iModels = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConcatenate : public CNeuronBaseOCL { protected: int i_SecondInputs; CBufferFloat *ConcWeights; ///< Buffer of weights matrix CBufferFloat *ConcDeltaWeights; ///< Buffer of last delta weights matrix (#SGD) CBufferFloat *ConcFirstMomentum; ///< Buffer of first momentum matrix (#ADAM) CBufferFloat *ConcSecondMomentum; ///< Buffer of second momentum matrix (#ADAM) ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronConcatenate(void); /** Destructor */~CNeuronConcatenate(void); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint inputs1, uint inputs2, ENUM_OPTIMIZATION optimization_type, uint batch); ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. //--- methods for working with files virtual bool Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) override const { return defNeuronConcatenate; }///< Identificator of class.@return Type of class virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual CBufferFloat *getConcatWeights(void) { return ConcWeights; } ///< Get pointer of gradient buffer @return Pointer to object }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronConcatenate::CNeuronConcatenate(void) : i_SecondInputs(0) { ConcWeights = new CBufferFloat(); ConcDeltaWeights = new CBufferFloat(); ConcFirstMomentum = new CBufferFloat(); ConcSecondMomentum = new CBufferFloat; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronConcatenate::~CNeuronConcatenate() { DeleteObj(ConcWeights); DeleteObj(ConcDeltaWeights); DeleteObj(ConcFirstMomentum); DeleteObj(ConcSecondMomentum); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, uint numInputs1, uint numInputs2, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; //--- i_SecondInputs = (int)numInputs2; if(!ConcWeights) { ConcWeights = new CBufferFloat(); if(!ConcWeights) ReturnFalse; } int count = (int)((numInputs1 + numInputs2 + 1) * numNeurons); if(!ConcWeights.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(numNeurons + 1)); for(int i = 0; i < count; i++) { if(!ConcWeights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!ConcWeights.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(!ConcDeltaWeights) { ConcDeltaWeights = new CBufferFloat(); if(!ConcDeltaWeights) ReturnFalse; } if(!ConcDeltaWeights.BufferInit(count, 0)) ReturnFalse; if(!ConcDeltaWeights.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(ConcFirstMomentum); DeleteObj(ConcSecondMomentum); } else { DeleteObj(ConcDeltaWeights); //--- if(!ConcFirstMomentum) { ConcFirstMomentum = new CBufferFloat(); if(CheckPointer(ConcFirstMomentum) == POINTER_INVALID) ReturnFalse; } if(!ConcFirstMomentum.BufferInit(count, 0)) ReturnFalse; if(!ConcFirstMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- if(!ConcSecondMomentum) { ConcSecondMomentum = new CBufferFloat(); if(!ConcSecondMomentum) ReturnFalse; } if(!ConcSecondMomentum.BufferInit(count, 0)) ReturnFalse; if(!ConcSecondMomentum.BufferCreate(OpenCL)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatFeedForward, def_k_cff_matrix_o, Output.GetIndex()) setArgument(def_k_ConcatFeedForward, def_k_cff_inputs1, (int)NeuronOCL.Neurons()) setArgument(def_k_ConcatFeedForward, def_k_cff_inputs2, (int)i_SecondInputs) setArgument(def_k_ConcatFeedForward, def_k_cff_activation, (int)activation) //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Output.Total(); kernelExecute(def_k_ConcatFeedForward, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(SecondInput.Total() < i_SecondInputs || SecondGradient.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; if(SecondGradient.GetIndex() < 0 && !SecondGradient.BufferCreate(OpenCL)) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = NeuronOCL.Neurons() + i_SecondInputs; setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_g, Gradient.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig1, NeuronOCL.getGradientIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_ig2, SecondGradient.GetIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatCalcHiddenGradient, def_k_cchg_matrix_o2, SecondInput.GetIndex()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_inputs2, i_SecondInputs) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_outputs, Neurons()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation1, NeuronOCL.Activation()) setArgument(def_k_ConcatCalcHiddenGradient, def_k_cchg_activation2, (int)SecondActivation) kernelExecute(def_k_ConcatCalcHiddenGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(SecondInput.Total() < i_SecondInputs) ReturnFalse; if(SecondInput.GetIndex() < 0 && !SecondInput.BufferCreate(OpenCL)) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = Neurons(); global_work_size[1] = NeuronOCL.Neurons() + i_SecondInputs + 1; float lt = lr; ResetLastError(); switch(NeuronOCL.Optimization()) { case SGD: setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_g, getGradientIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_matrix_dw, ConcDeltaWeights.GetIndex()) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_inputs1, i_SecondInputs) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_learning_rates, lr) setArgument(def_k_ConcatUpdWeightsMomentum, def_k_cuwm_momentum, alpha) ResetLastError(); kernelExecute(def_k_ConcatUpdWeightsMomentum, global_work_offset, global_work_size) break; case ADAM: case ADAM_MINI: setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_w, ConcWeights.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_g, getGradientIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i1, NeuronOCL.getOutputIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_i2, SecondInput.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_m, ConcFirstMomentum.GetIndex()) setBuffer(def_k_ConcatUpdWeightsAdam, def_k_cuwa_matrix_v, ConcSecondMomentum.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, (float)t)) / (1 - pow(b1, (float)t))); setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs1, NeuronOCL.Neurons()) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_inputs2, i_SecondInputs) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_l, lt) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b1, b1) setArgument(def_k_ConcatUpdWeightsAdam, def_k_cuwa_b2, b2) kernelExecute(def_k_ConcatUpdWeightsAdam, global_work_offset, global_work_size) t++; break; default: ReturnFalse; break; } //--- return true;//NeuronOCL.Weights.BufferRead(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, i_SecondInputs) < INT_VALUE) ReturnFalse; if(!ConcWeights.Save(file_handle)) ReturnFalse; if(optimization == SGD) { if(!ConcDeltaWeights.Save(file_handle)) ReturnFalse; } else { if(!ConcFirstMomentum.Save(file_handle)) ReturnFalse; if(!ConcSecondMomentum.Save(file_handle)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- i_SecondInputs = FileReadInteger(file_handle); if(!ConcWeights.Load(file_handle)) ReturnFalse; ConcWeights.BufferCreate(OpenCL); if(optimization == SGD) { if(!ConcDeltaWeights.Load(file_handle)) ReturnFalse; ConcDeltaWeights.BufferCreate(OpenCL); } else { if(!ConcFirstMomentum.Load(file_handle)) ReturnFalse; if(!ConcSecondMomentum.Load(file_handle)) ReturnFalse; ConcFirstMomentum.BufferCreate(OpenCL); ConcSecondMomentum.BufferCreate(OpenCL); } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConcatenate::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!ConcWeights) return; ConcWeights.BufferCreate(OpenCL); if(optimization == SGD) { if(!ConcDeltaWeights) return; ConcDeltaWeights.BufferCreate(OpenCL); } else { if(!ConcFirstMomentum || !ConcSecondMomentum) return; ConcFirstMomentum.BufferCreate(OpenCL); ConcSecondMomentum.BufferCreate(OpenCL); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::WeightsUpdate(CNet * net, float tau) { if(!layers || !net || !net.layers || layers.Total() != net.layers.Total()) ReturnFalse; if(tau < 0) ReturnFalse; if(tau > 1) return true; //--- for(int l = 0; l < layers.Total(); l++) { CLayer *layer = layers.At(l); if(!layer || !layer.WeightsUpdate(net.layers.At(l), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::WeightsUpdate(CLayer * source, float tau) { if(!source || Type() != source.Type() || m_data_total != source.Total()) ReturnFalse; //--- for(int i = 0; i < m_data_total; i++) { CNeuronBaseOCL *temp = m_data[i]; if(temp == source.At(i)) continue; //--- if(!temp || !temp.WeightsUpdate(source.At(i), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CLayer::ClearStates(void) { for(int i = 0; i < m_data_total; i++) { CNeuronBaseOCL *temp = m_data[i]; if(!temp || !temp.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(tau != 1.0f && optimization == ADAM) return WeightsUpdateAdam(source, tau); //--- if(!OpenCL || !source) ReturnFalse; if(Type() != source.Type()) ReturnFalse; if(!Weights || Weights.Total() == 0) return true; if(!source.Weights || Weights.Total() != source.Weights.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Weights.Total()}; ResetLastError(); setBuffer(def_k_SoftUpdate, def_k_su_target, Weights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, source.getWeightsIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::WeightsUpdateAdam(CNeuronBaseOCL * source, float tau) { if(GetPointer(this) == source) return true; //--- if(!OpenCL || !source) ReturnFalse; if(Type() != source.Type()) ReturnFalse; if(!Weights || Weights.Total() == 0) return true; if(!source.Weights || Weights.Total() != source.Weights.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Weights.Total()}; ResetLastError(); setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, getWeightsIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, source.getWeightsIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, getFirstMomentumIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, getSecondMomentumIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::SwapBuffers(CBufferFloat *&buffer1, CBufferFloat *&buffer2, bool check_size = true) { if(!buffer1 || !buffer2) ReturnFalse; if(check_size) { if(buffer1.Total() != buffer2.Total()) ReturnFalse; } //--- CBufferFloat *temp = buffer1; buffer1 = buffer2; buffer2 = temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatenate::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronConcatenate *temp = source; if(ConcWeights.Total() != temp.ConcWeights.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {ConcWeights.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, ConcFirstMomentum.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, ConcSecondMomentum.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, ConcWeights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, temp.ConcWeights.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronConvOCL *temp = source; if(WeightsConv.Total() != temp.WeightsConv.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {WeightsConv.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, WeightsConv.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.WeightsConv.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, SecondMomentumConv.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, WeightsConv.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, temp.WeightsConv.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronBatchNormOCL *temp = source; if(BatchOptions.Total() != temp.BatchOptions.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {BatchOptions.Total()}; ResetLastError(); setBuffer(def_k_SoftUpdate, def_k_su_target, BatchOptions.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, temp.BatchOptions.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CNet * inputNet, int inputLayer = -1, CNet *secondNet = NULL, int secondLayer = -1) { if(!inputNet || !opencl) ReturnFalse; if(inputLayer < 0) { inputLayer = inputNet.layers.Total() + inputLayer; if(inputLayer < 0) ReturnFalse; } //--- CBufferFloat *second = NULL; bool del_second = false; if(!!secondNet) { if(secondLayer < 0) secondLayer = secondNet.layers.Total() - 1; if(secondNet.GetOpenCL() != opencl) { secondNet.GetLayerOutput(secondLayer, second); if(!!second) { if(!second.BufferCreate(opencl)) { DeleteObj(second); ReturnFalse; } del_second = true; } } else { if(secondNet.layers.Total() <= secondLayer) ReturnFalse; CLayer *layer = secondNet.layers.At(secondLayer); CNeuronBaseOCL *neuron = layer.At(0); second = neuron.getOutput(); } } //--- if(inputNet.opencl != opencl) { CBufferFloat *inputs; if(!inputNet.GetLayerOutput(inputLayer, inputs)) { if(del_second) DeleteObj(second); DeleteObj(inputs); ReturnFalse; } bool result = feedForward(inputs, 1, false, second); if(del_second) DeleteObj(second); DeleteObj(inputs); return result; } //--- CLayer *layer = inputNet.layers.At(inputLayer); if(!layer) { if(del_second) DeleteObj(second); ReturnFalse; } CNeuronBaseOCL *neuron = layer.At(0); layer = layers.At(0); if(!layer) { if(del_second) DeleteObj(second); ReturnFalse; } if(layer.At(0) != neuron) { CNeuronBaseOCL *temp = layer.At(0); if(temp.getConnections() <= 0) { if(!layer.Update(0, neuron)) { if(del_second) DeleteObj(second); ReturnFalse; } else layer.FreeMode(false); } else { if(neuron.getOutputIndex() != temp.getOutputIndex()) temp.getOutput().BufferSet(neuron.getOutputIndex()); if(neuron.getGradientIndex() != temp.getGradientIndex()) temp.getGradient().BufferSet(neuron.getGradientIndex()); } } //--- for(int l = 1; l < layers.Total() && !IsStopped(); l++) { layer = layers.At(l); neuron = layer.At(0); layer = layers.At(l - 1); if(!neuron.FeedForward(layer.At(0), second)) { if(del_second) DeleteObj(second); ReturnFalse; } #ifdef _DEBUG neuron.getOutput().BufferRead(); Sleep(0); #endif } //--- if(del_second) DeleteObj(second); return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CNet * inputNet, int inputLayer = -1, CBufferFloat *SecondInput = NULL) { if(!inputNet || !opencl) ReturnFalse; if(inputLayer < 0) { inputLayer = inputNet.layers.Total() + inputLayer; if(inputLayer < 0) ReturnFalse; } //--- if(!!SecondInput) { if(SecondInput.GetIndex() < 0) if(!SecondInput.BufferCreate(opencl)) ReturnFalse; } //--- if(inputNet.opencl != opencl) { CBufferFloat *inputs; if(!inputNet.GetLayerOutput(inputLayer, inputs)) ReturnFalse; bool result = feedForward(inputs, 1, false, SecondInput); return result; } //--- CLayer *layer = inputNet.layers.At(inputLayer); if(!layer) ReturnFalse; CNeuronBaseOCL *neuron = layer.At(0); layer = layers.At(0); if(!layer) ReturnFalse; if(layer.At(0) != neuron) { CNeuronBaseOCL *temp = layer.At(0); if(neuron.getOutputIndex() != temp.getOutputIndex()) temp.getOutput().BufferSet(neuron.getOutputIndex()); if(neuron.getGradientIndex() != temp.getGradientIndex()) temp.getGradient().BufferSet(neuron.getGradientIndex()); temp.SetActivationFunction((ENUM_ACTIVATION)neuron.Activation()); } //--- for(int l = 1; l < layers.Total() && !IsStopped(); l++) { layer = layers.At(l); neuron = layer.At(0); layer = layers.At(l - 1); if(!neuron.FeedForward(layer.At(0), SecondInput)) ReturnFalse; //neuron.getOutput().BufferRead(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::backProp(CArrayFloat * targetVals, CNet *secondNet = NULL, int secondLayer = -1) { if(!secondNet || !secondNet.layers) return backPropOCL(targetVals); //--- if(secondLayer < 0) secondLayer = secondNet.layers.Total() - 1; //--- if(opencl != secondNet.GetOpenCL()) { CBufferFloat *second; if(!secondNet.GetLayerOutput(secondLayer, second)) ReturnFalse; if(!second) return backPropOCL(targetVals); CLayer *layer = secondNet.layers.At(secondLayer); CNeuronBaseOCL *neuron = layer.At(0); CBufferFloat gradient; neuron.getGradient().BufferRead(); gradient.AssignArray(neuron.getGradient()); if(!second.BufferCreate(opencl) || !gradient.BufferCreate(opencl)) { DeleteObj(second); ReturnFalse; } bool result = backPropOCL(targetVals, second, GetPointer(gradient), false, (ENUM_ACTIVATION)neuron.Activation()); DeleteObj(second); if(result) { second = neuron.getGradient(); gradient.BufferRead(); second.AssignArray(GetPointer(gradient)); result = second.BufferWrite(); } return result; } //--- CLayer *layer = secondNet.layers.At(secondLayer); CNeuronBaseOCL *neuron = layer.At(0); if(!backPropOCL(targetVals, neuron.getOutput(), neuron.getGradient(), false, (ENUM_ACTIVATION)neuron.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::backPropGradient(CBufferFloat *SecondInput = NULL, CBufferFloat *SecondGradient = NULL, int LastLayer = -1, bool load_weights = true) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(opencl) == POINTER_INVALID) ReturnFalse; if(LastLayer < 0) { LastLayer = layers.Total() + LastLayer; if(LastLayer < 0) ReturnFalse; } CLayer *currentLayer = layers.At(LastLayer); CNeuronBaseOCL *neuron = NULL; if(CheckPointer(currentLayer) == POINTER_INVALID) ReturnFalse; //--- Calc Hidden Gradients for(int layerNum = LastLayer - 1; layerNum >= 0 && !IsStopped(); layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); if(CheckPointer(currentLayer) == POINTER_INVALID) ReturnFalse; neuron = currentLayer.At(0); if(!neuron || !neuron.CalcHiddenGradients(nextLayer.At(0), SecondInput, SecondGradient)) ReturnFalse; #ifdef _DEBUG if(!neuron.getGradient().BufferRead()) ReturnFalse; #endif } //--- CLayer *prevLayer = layers.At(LastLayer); for(int layerNum = LastLayer; layerNum > 0 && !IsStopped(); layerNum--) { currentLayer = prevLayer; prevLayer = layers.At(layerNum - 1); neuron = currentLayer.At(0); if(!neuron.UpdateInputWeights(prevLayer.At(0), SecondInput)) ReturnFalse; } //--- if(!load_weights) return true; //--- bool result = false; for(int layerNum = 0; (layerNum < layers.Total() && !result && !IsStopped()); layerNum++) { currentLayer = layers.At(layerNum); CNeuronBaseOCL *temp = currentLayer.At(0); CNeuronConvOCL *conv = NULL; CNeuronConcatenate *conc = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; if(!temp.TrainMode()) { if(layerNum == layers.Total() - 1) result = true; continue; } switch(temp.Type()) { case defNeuronConvOCL: conv = temp; if(!!conv.GetWeightsConv() && conv.GetWeightsConv().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronConcatenate: if(!!temp.getWeights() && temp.getWeights().BufferRead()) { result = true; break; } conc = temp; if(!!conc.getConcatWeights() && conc.getConcatWeights().BufferRead()) result = true; break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; if(!!batch.getBatchOptions() && batch.getBatchOptions().BufferRead()) result = true; else if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronLSTMOCL: lstm = temp; if(!!lstm.getLSTMWeights() && lstm.getLSTMWeights().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; default: if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::backPropGradient(CNet * secondNet, int secondLayer = -1, int LastLayer = -1, bool load_weights = true) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(opencl) == POINTER_INVALID || !secondNet) ReturnFalse; if(LastLayer < 0) { LastLayer = layers.Total() + LastLayer; if(LastLayer < 0) ReturnFalse; } if(secondLayer < 0) secondLayer = secondNet.layers.Total() - 1; CLayer *currentLayer = layers.At(LastLayer); CNeuronBaseOCL *neuron = NULL; if(!currentLayer || !secondNet.layers.At(secondLayer)) ReturnFalse; CNeuronBaseOCL *secondNeuron = ((CArrayObj *)secondNet.layers.At(secondLayer)).At(0); if(!secondNeuron) ReturnFalse; //--- Calc Hidden Gradients for(int layerNum = LastLayer - 1; layerNum >= 0 && !IsStopped(); layerNum--) { CLayer *nextLayer = currentLayer; currentLayer = layers.At(layerNum); if(CheckPointer(currentLayer) == POINTER_INVALID) ReturnFalse; neuron = currentLayer.At(0); if(!neuron || !neuron.CalcHiddenGradients(nextLayer.At(0), secondNeuron.getOutput(), secondNeuron.getGradient())) ReturnFalse; #ifdef _DEBUG if(!neuron.getGradient().BufferRead()) ReturnFalse; #endif } //--- CLayer *prevLayer = layers.At(LastLayer); for(int layerNum = LastLayer; layerNum > 0 && !IsStopped(); layerNum--) { currentLayer = prevLayer; prevLayer = layers.At(layerNum - 1); neuron = currentLayer.At(0); if(!neuron.UpdateInputWeights(prevLayer.At(0), secondNeuron.getOutput())) ReturnFalse; } //--- if(!load_weights) return true; //--- bool result = false; for(int layerNum = 0; (layerNum < layers.Total() && !result && !IsStopped()); layerNum++) { currentLayer = layers.At(layerNum); CNeuronBaseOCL *temp = currentLayer.At(0); CNeuronConvOCL *conv = NULL; CNeuronConcatenate *conc = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; if(!temp.TrainMode()) { if(layerNum == layers.Total() - 1) result = true; continue; } switch(temp.Type()) { case defNeuronConvOCL: conv = temp; if(!!conv.GetWeightsConv() && conv.GetWeightsConv().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronConcatenate: if(!!temp.getWeights() && temp.getWeights().BufferRead()) { result = true; break; } conc = temp; if(!!conc.getConcatWeights() && conc.getConcatWeights().BufferRead()) result = true; break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; if(!!batch.getBatchOptions() && batch.getBatchOptions().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; case defNeuronLSTMOCL: lstm = temp; if(!!lstm.getLSTMWeights() && lstm.getLSTMWeights().BufferRead()) result = true; if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; default: if(!!temp.getWeights() && temp.getWeights().BufferRead()) result = true; break; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSoftActorCritic : public CNeuronFQF { protected: CNeuronConcatenate cAlphas; CBufferFloat cLogProbs; CBufferFloat cRandomize; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronSoftActorCritic(void) {}; ~CNeuronSoftActorCritic(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. virtual bool calcAlphaGradients(CNeuronBaseOCL *NeuronOCL); virtual bool GetAlphaLogProbs(vector &log_probs) { return (cLogProbs.GetData(log_probs) > 0); } virtual bool CalcLogProbs(CBufferFloat *buffer); virtual bool ReCalcLogProbs(void); //--- virtual bool Save(int const file_handle) override;///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle) override;///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) override const { return defNeuronSoftActorCritic; }///< Identificator of class.@return Type of class virtual void SetOpenCL(COpenCLMy *obj); //--- virtual void getCheckData(vector &output, vector &grad, vector &quantiles, vector &probability, vector &alphas); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint actions, uint quantiles, uint numInputs, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFQF::Init(numOutputs, myIndex, open_cl, actions, quantiles, numInputs, optimization_type, batch)) ReturnFalse; //--- if(!cAlphas.Init(0, 0, OpenCL, actions, numInputs, cQuantile2.Neurons(), optimization_type, batch)) ReturnFalse; cAlphas.SetActivationFunction(SIGMOID); //--- if(!cLogProbs.BufferInit(actions, 0) || !cLogProbs.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cRandomize.BufferInit(actions, 0) || !cRandomize.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronFQF::feedForward(NeuronOCL)) ReturnFalse; if(!cAlphas.FeedForward(GetPointer(cQuantile0), cQuantile2.getOutput())) ReturnFalse; //--- int actions = cRandomize.Total(); for(int i = 0; i < actions; i++) { float probability = (float)MathRand() / 32767.0f; cRandomize.Update(i, probability); } if(!cRandomize.BufferWrite()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_alphas, cAlphas.getOutputIndex()) setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_log_probs, cLogProbs.GetIndex()) setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_outputs, getOutputIndex()) setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_probs, cSoftMax.getOutputIndex()) setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_quantiles, cQuantile2.getOutputIndex()) setBuffer(def_k_SAC_AlphaLogProbs, def_k_sac_alp_random, cRandomize.GetIndex()) setArgument(def_k_SAC_AlphaLogProbs, def_k_sac_alp_count_quants, (int)(cSoftMax.Neurons() / global_work_size[0])) setArgument(def_k_SAC_AlphaLogProbs, def_k_sac_alp_activation, (int)activation) kernelExecute(def_k_SAC_AlphaLogProbs, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::calcAlphaGradients(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getGradient() || NeuronOCL.getGradientIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; setBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_outputs, cAlphas.getOutputIndex()) setBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_alphas_grad, cAlphas.getGradientIndex()) setBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_gradient, NeuronOCL.getGradientIndex()) setBuffer(def_k_SAC_AlphaGradients, def_k_sac_alg_log_probs, cLogProbs.GetIndex()) setArgument(def_k_SAC_AlphaGradients, def_k_sac_alg_activation, (int)cAlphas.Activation()) kernelExecute(def_k_SAC_AlphaGradients, global_work_offset, global_work_size) //--- return cQuantile0.CalcHiddenGradients((CObject*)GetPointer(cAlphas), cQuantile2.getOutput(), cQuantile2.getGradient()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronFQF::updateInputWeights(NeuronOCL)) ReturnFalse; //--- return cAlphas.UpdateInputWeights(cQuantile0.AsObject(), cQuantile2.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronFQF::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSoftActorCritic *Source = source; //--- return cAlphas.WeightsUpdate(GetPointer(Source.cAlphas), tau); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSoftActorCritic::SetOpenCL(COpenCLMy * obj) { CNeuronFQF::SetOpenCL(obj); cAlphas.SetOpenCL(OpenCL); if(cLogProbs.Total() != Neurons()) cLogProbs.BufferInit(Neurons(), 0); cLogProbs.BufferCreate(OpenCL); if(cRandomize.Total() != Neurons()) cRandomize.BufferInit(Neurons(), 0); cRandomize.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::AlphasGradient(CNet * PolicyNet) { if(!PolicyNet || !PolicyNet.layers) ReturnFalse; int total = PolicyNet.layers.Total(); if(total <= 0) ReturnFalse; CLayer *layer = PolicyNet.layers.At(total - 1); if(!layer || !layer.At(0)) ReturnFalse; if(layer.At(0).Type() != defNeuronSoftActorCritic) return true; //--- CNeuronSoftActorCritic *neuron = layer.At(0); if(!layers) ReturnFalse; total = layers.Total(); if(total <= 0 || !layers.At(total - 1)) ReturnFalse; layer = layers.At(total - 1); //--- return neuron.calcAlphaGradients((CNeuronBaseOCL*) layer.At(0)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::Save(const int file_handle) { if(!CNeuronFQF::Save(file_handle)) ReturnFalse; return cAlphas.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::Load(const int file_handle) { if(!CNeuronFQF::Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cAlphas.Type() || !cAlphas.Load(file_handle)) ReturnFalse; //--- if(cLogProbs.Total() != Neurons()) { cLogProbs.BufferFree(); if(!cLogProbs.BufferInit(Neurons(), 0) || (!!OpenCL && !cLogProbs.BufferCreate(OpenCL))) ReturnFalse; } //--- if(cRandomize.Total() != Neurons()) { cRandomize.BufferFree(); if(!cRandomize.BufferInit(Neurons(), 0) || (!!OpenCL && !cRandomize.BufferCreate(OpenCL))) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::GetLogProbs(vectorf & log_probs) { //--- if(!layers) ReturnFalse; int total = layers.Total(); if(total <= 0 || !layers.At(total - 1)) ReturnFalse; CLayer *layer = layers.At(total - 1); if(!layer.At(0) || layer.At(0).Type() != defNeuronSoftActorCritic) ReturnFalse; //--- CNeuronSoftActorCritic *neuron = layer.At(0); //--- return neuron.GetAlphaLogProbs(log_probs); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::CalcLogProbs(CBufferFloat * buffer) { if(!layers) ReturnFalse; int total = layers.Total(); if(total <= 0 || !layers.At(total - 1)) ReturnFalse; CLayer *layer = layers.At(total - 1); if(!layer.At(0) || layer.At(0).Type() != defNeuronSoftActorCritic) ReturnFalse; //--- CNeuronSoftActorCritic *neuron = layer.At(0); //--- return neuron.CalcLogProbs(buffer); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::CalcLogProbs(CBufferFloat * buffer) { if(!buffer || buffer.Total() < Neurons()) ReturnFalse; if(buffer.GetIndex() < 0 && !buffer.BufferCreate(OpenCL)) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_alphas, cAlphas.getOutputIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_log_probs, buffer.GetIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_outputs, buffer.GetIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_probs, cSoftMax.getOutputIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_quantiles, cQuantile2.getOutputIndex()) setArgument(def_k_SAC_CalcLogProbs, def_k_sacclp_count_quants, (int)(cSoftMax.Neurons() / global_work_size[0])) setArgument(def_k_SAC_CalcLogProbs, def_k_sacclp_activation, (int)Activation()) kernelExecute(def_k_SAC_CalcLogProbs, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::ReCalcLogProbs(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_alphas, cAlphas.getOutputIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_log_probs, cLogProbs.GetIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_outputs, Output.GetIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_probs, cSoftMax.getOutputIndex()) setBuffer(def_k_SAC_CalcLogProbs, def_k_sacclp_quantiles, cQuantile2.getOutputIndex()) setArgument(def_k_SAC_CalcLogProbs, def_k_sacclp_count_quants, (int)(cSoftMax.Neurons() / global_work_size[0])) setArgument(def_k_SAC_CalcLogProbs, def_k_sacclp_activation, (int)Activation()) kernelExecute(def_k_SAC_CalcLogProbs, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSoftActorCritic::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !Gradient || !Output) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_quantiles, cQuantile2.getOutputIndex()) setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_taus, cSoftMax.getOutputIndex()) setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_output_gr, getGradientIndex()) setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_quantiles_gr, cQuantile2.getGradientIndex()) setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_taus_gr, cSoftMax.getGradientIndex()) setBuffer(def_k_SAC_OutputGradient, def_k_sacoutgr_outputs, getOutputIndex()) setArgument(def_k_SAC_OutputGradient, def_k_sacoutgr_activation, (int)Activation()) setArgument(def_k_SAC_OutputGradient, def_k_sacoutgr_count_quants, (int)(cSoftMax.Neurons() / Neurons())) kernelExecute(def_k_SAC_OutputGradient, global_work_offset, global_work_size) } //--- if(!cQuantile1.CalcHiddenGradients((CObject *)GetPointer(cQuantile2))) ReturnFalse; if(!cQuantile0.CalcHiddenGradients((CObject *)GetPointer(cQuantile1))) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { cCosineEmbeding.Neurons(), 1 }; setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_state_enbeding, NeuronOCL.getOutputIndex()) setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_taus_embedding, cCosineEmbeding.getOutputIndex()) setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_quantiles_gr, cQuantile0.getGradientIndex()) setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_state_gr, NeuronOCL.getGradientIndex()) setBuffer(def_k_FQF_QuantileGradient, def_k_fqfqgr_taus_gr, cCosineEmbeding.getGradientIndex()) kernelExecute(def_k_FQF_QuantileGradient, global_work_offset, global_work_size) } //--- if(!cCosine.CalcHiddenGradients((CObject *)GetPointer(cCosineEmbeding))) ReturnFalse; //--- { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { cSoftMax.Neurons() / Neurons(), Neurons() }; setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_softmax, cSoftMax.getOutputIndex()) setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_output_gr, cCosine.getGradientIndex()) setBuffer(def_k_FQF_CosineGradient, def_k_fqfcosgr_softmax_gr, cSoftMax.getGradientIndex()) kernelExecute(def_k_FQF_CosineGradient, global_work_offset, global_work_size) } if(!cSoftMax.CalcHiddenGradients((CObject *)GetPointer(cFraction))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::SetResult(float & result[]) { if(!layers || layers.Total() <= 0) ReturnFalse; //--- CNeuronBaseOCL *output = ((CLayer*)layers.At(layers.Total() - 1)).At(0); if(!output) ReturnFalse; if(!output.getOutput().AssignArray(result) || !output.getOutput().BufferWrite()) ReturnFalse; if(output.Type() == defNeuronSoftActorCritic && !output.ReCalcLogProbs()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSoftActorCritic::getCheckData(vector &output, vector &grad, vector &quantiles, vector &probability, vector &alphas) { Output.GetData(output); Gradient.GetData(grad); cQuantile2.getGradient().GetData(quantiles); cSoftMax.getGradient().GetData(probability); cAlphas.getGradient().GetData(alphas); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronEmbeddingOCL : public CNeuronBaseOCL { protected: int a_Windows[]; int i_WindowOut; int i_StackSize; int i_WindowsBuffer; int i_STDBuffer; //--- CBufferFloat WeightsEmbedding; CBufferFloat FirstMomentumEmbed; CBufferFloat SecondMomentumEmbed; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_gr ///@{ virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. ///@} ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. public: CNeuronEmbeddingOCL(void); ~CNeuronEmbeddingOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint window_out, uint &windows[]); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object. #param[in] numNeurons Number of neurons in layer @param optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronEmbeddingOCL; }///< Identificator of class.@return Type of class virtual CLayerDescription* GetLayerInfo(void) { return CNeuronBaseOCL::GetLayerInfo(); } virtual void SetOpenCL(COpenCLMy *obj); virtual bool Clear(void); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronEmbeddingOCL::CNeuronEmbeddingOCL(void) { ArrayFree(a_Windows); if(!!OpenCL) { if(i_WindowsBuffer >= 0) OpenCL.BufferFree(i_WindowsBuffer); if(i_STDBuffer >= 0) OpenCL.BufferFree(i_STDBuffer); } //--- i_WindowsBuffer = INVALID_HANDLE; i_STDBuffer = INVALID_HANDLE; i_WindowOut = 0; i_StackSize = 1; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronEmbeddingOCL::~CNeuronEmbeddingOCL(void) { ArrayFree(a_Windows); if(!!OpenCL) { if(i_WindowsBuffer >= 0) OpenCL.BufferFree(i_WindowsBuffer); if(i_STDBuffer >= 0) OpenCL.BufferFree(i_STDBuffer); } //--- i_WindowsBuffer = INVALID_HANDLE; i_STDBuffer = INVALID_HANDLE; i_WindowOut = 0; i_StackSize = 1; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEmbeddingOCL::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; CNeuronBaseOCL::SetOpenCL(obj); if(!OpenCL) return; //--- i_WindowsBuffer = OpenCL.AddBuffer(sizeof(int) * a_Windows.Size(), CL_MEM_READ_WRITE); if(i_WindowsBuffer >= 0) OpenCL.BufferWrite(i_WindowsBuffer, a_Windows, 0, 0, a_Windows.Size()); i_STDBuffer = OpenCL.AddBuffer(sizeof(float) * a_Windows.Size(), CL_MEM_READ_WRITE); //--- WeightsEmbedding.BufferCreate(OpenCL); FirstMomentumEmbed.BufferCreate(OpenCL); SecondMomentumEmbed.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint stack_size, uint window_out, uint & windows[]) { if(CheckPointer(open_cl) == POINTER_INVALID || window_out <= 0 || windows.Size() <= 0 || stack_size <= 0) ReturnFalse; if(OpenCL != open_cl) DeleteObj(OpenCL); uint numNeurons = window_out * windows.Size() * stack_size; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, ADAM, 1)) ReturnFalse; //--- uint weights = 0; ArrayCopy(a_Windows, windows); i_WindowOut = (int)window_out; i_StackSize = (int)stack_size; for(uint i = 0; i < windows.Size(); i++) weights += (windows[i] + 1) * window_out; if(!WeightsEmbedding.Reserve(weights)) ReturnFalse; float k = 1.0f / sqrt((float)weights / (float)window_out); for(uint i = 0; i < weights; i++) if(!WeightsEmbedding.Add(k * (2 * GenerateWeight() - 1.0f)*WeightsMultiplier)) ReturnFalse; if(!WeightsEmbedding.BufferCreate(OpenCL)) ReturnFalse; //--- if(!FirstMomentumEmbed.BufferInit(weights, 0)) ReturnFalse; if(!FirstMomentumEmbed.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SecondMomentumEmbed.BufferInit(weights, 0)) ReturnFalse; if(!SecondMomentumEmbed.BufferCreate(OpenCL)) ReturnFalse; //--- i_WindowsBuffer = OpenCL.AddBuffer(sizeof(int) * a_Windows.Size(), CL_MEM_READ_WRITE); if(i_WindowsBuffer < 0 || !OpenCL.BufferWrite(i_WindowsBuffer, a_Windows, 0, 0, a_Windows.Size())) ReturnFalse; i_STDBuffer = OpenCL.AddBuffer(sizeof(float) * a_Windows.Size(), CL_MEM_READ_WRITE); if(i_STDBuffer < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- bool res = true; int check_w[]; ArrayResize(check_w, a_Windows.Size()); if(!OpenCL.BufferRead(i_WindowsBuffer, check_w, 0, 0, a_Windows.Size())) ReturnFalse; for(uint i = 0; (i < a_Windows.Size() && res); i++) res = check_w[i] == a_Windows[i]; if(!res) { if(!OpenCL.BufferWrite(i_WindowsBuffer, a_Windows, 0, 0, a_Windows.Size())) ReturnFalse; } //--- setBuffer(def_k_Embedding, def_k_emb_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_Embedding, def_k_emb_outputs, getOutputIndex()) setBuffer(def_k_Embedding, def_k_emb_std, i_STDBuffer) setBuffer(def_k_Embedding, def_k_emb_weights, WeightsEmbedding.GetIndex()) setBuffer(def_k_Embedding, def_k_emb_windows, i_WindowsBuffer) setArgument(def_k_Embedding, def_k_emb_stack_size, i_StackSize) //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {i_WindowOut, a_Windows.Size()}; uint local_work_size[2] = {i_WindowOut, 1}; kernelExecuteLoc(def_k_Embedding, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::Clear(void) { if(!Output.BufferInit(Output.Total(), 0)) ReturnFalse; if(!OpenCL) return true; //--- return Output.BufferWrite(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- setBuffer(def_k_EmbeddingHiddenGradient, def_k_ehg_inputs_gradient, NeuronOCL.getGradientIndex()) setBuffer(def_k_EmbeddingHiddenGradient, def_k_ehg_outputs_gradient, getGradientIndex()) setBuffer(def_k_EmbeddingHiddenGradient, def_k_ehg_std, i_STDBuffer) setBuffer(def_k_EmbeddingHiddenGradient, def_k_ehg_weights, WeightsEmbedding.GetIndex()) setBuffer(def_k_EmbeddingHiddenGradient, def_k_ehg_windows, i_WindowsBuffer) setArgument(def_k_EmbeddingHiddenGradient, def_k_ehg_window_out, i_WindowOut) //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {NeuronOCL.Neurons()}; kernelExecute(def_k_EmbeddingHiddenGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_gradient, getGradientIndex()) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_std, i_STDBuffer) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_weights, WeightsEmbedding.GetIndex()) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_matrix_m, FirstMomentumEmbed.GetIndex()) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_matrix_v, SecondMomentumEmbed.GetIndex()) setBuffer(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_windows, i_WindowsBuffer) setArgument(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_window_out, i_WindowOut) setArgument(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_learning_rate, lr) setArgument(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_b1, b1) setArgument(def_k_EmbeddingUpdateWeightsAdam, def_k_euw_b2, b2) //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {WeightsEmbedding.Total()}; kernelExecute(def_k_EmbeddingUpdateWeightsAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronEmbeddingOCL *temp = source; if(WeightsEmbedding.Total() != temp.WeightsEmbedding.Total()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {WeightsEmbedding.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, WeightsEmbedding.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, temp.WeightsEmbedding.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, FirstMomentumEmbed.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, SecondMomentumEmbed.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, WeightsEmbedding.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, temp.WeightsEmbedding.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!WeightsEmbedding.Save(file_handle)) ReturnFalse; if(!FirstMomentumEmbed.Save(file_handle)) ReturnFalse; if(!SecondMomentumEmbed.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)a_Windows.Size()) != INT_VALUE) ReturnFalse; if(FileWriteArray(file_handle, a_Windows, 0, (int)a_Windows.Size()) != a_Windows.Size()) ReturnFalse; //--- if(FileWriteInteger(file_handle, i_WindowOut) != INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, i_StackSize) != INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEmbeddingOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!WeightsEmbedding.Load(file_handle)) ReturnFalse; if(!FirstMomentumEmbed.Load(file_handle)) ReturnFalse; if(!SecondMomentumEmbed.Load(file_handle)) ReturnFalse; //--- int size = FileReadInteger(file_handle); ArrayResize(a_Windows, size); if(FileReadArray(file_handle, a_Windows, 0, size) != size) ReturnFalse; //--- i_WindowOut = FileReadInteger(file_handle); i_StackSize = FileReadInteger(file_handle); //--- if(!OpenCL) return true; //--- if(i_WindowsBuffer != INVALID_HANDLE) OpenCL.BufferFree(i_WindowsBuffer); i_WindowsBuffer = OpenCL.AddBuffer(sizeof(int) * a_Windows.Size(), CL_MEM_READ_WRITE); if(i_WindowsBuffer >= 0) if(!OpenCL.BufferWrite(i_WindowsBuffer, a_Windows, 0, 0, size)) ReturnFalse; if(i_STDBuffer != INVALID_HANDLE) OpenCL.BufferFree(i_STDBuffer); i_STDBuffer = OpenCL.AddBuffer(sizeof(float) * a_Windows.Size(), CL_MEM_READ_WRITE); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::Clear(void) { if(!opencl) return true; if(!layers || layers.Total() <= 0) return true; for(int i = 0; i < layers.Total(); i++) { CLayer *layer = layers.At(i); if(!layer) ReturnFalse; for(int n = 0; n < layer.Total(); n++) { CNeuronBaseOCL *neuron = layer.At(n); if(!neuron || !neuron.Clear()) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFQF::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFQF *temp = source; if(!cFraction.WeightsUpdate(GetPointer(temp.cFraction), tau)) ReturnFalse; if(!cSoftMax.WeightsUpdate(GetPointer(temp.cSoftMax), tau)) ReturnFalse; if(!cCosine.WeightsUpdate(GetPointer(temp.cCosine), tau)) ReturnFalse; if(!cCosineEmbeding.WeightsUpdate(GetPointer(temp.cCosineEmbeding), tau)) ReturnFalse; if(!cQuantile0.WeightsUpdate(GetPointer(temp.cQuantile0), tau)) ReturnFalse; if(!cQuantile1.WeightsUpdate(GetPointer(temp.cQuantile1), tau)) ReturnFalse; if(!cQuantile2.WeightsUpdate(GetPointer(temp.cQuantile2), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionOCL::ConvolutuionUpdateWeights(CBufferFloat * weights, CBufferFloat * source, CBufferFloat * momentum1, CBufferFloat * momentum2, float tau) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {weights.Total()}; ResetLastError(); if(tau != 1.0f && optimization == ADAM) { setBuffer(def_k_SoftUpdateAdam, def_k_sua_target, weights.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_source, source.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_m, momentum1.GetIndex()) setBuffer(def_k_SoftUpdateAdam, def_k_sua_matrix_v, momentum2.GetIndex()) setArgument(def_k_SoftUpdateAdam, def_k_sua_tau, (float)tau) setArgument(def_k_SoftUpdateAdam, def_k_sua_b1, (float)b1) setArgument(def_k_SoftUpdateAdam, def_k_sua_b2, (float)b2) kernelExecute(def_k_SoftUpdateAdam, global_work_offset, global_work_size) } else { setBuffer(def_k_SoftUpdate, def_k_su_target, weights.GetIndex()) setBuffer(def_k_SoftUpdate, def_k_su_source, source.GetIndex()) setArgument(def_k_SoftUpdate, def_k_su_tau, (float)tau) kernelExecute(def_k_SoftUpdate, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::feedForward(CArrayFloat * inputVals, int window = 1, bool tem = true, CNet *secondNet = NULL, int secondLayer = -1) { if(CheckPointer(layers) == POINTER_INVALID || CheckPointer(inputVals) == POINTER_INVALID || layers.Total() <= 1) ReturnFalse; //--- CBufferFloat *SecondInput = NULL; bool del_second = false; if(!!secondNet) { if(secondLayer < 0) secondLayer = secondNet.layers.Total() - 1; if(secondNet.GetOpenCL() != opencl) { secondNet.GetLayerOutput(secondLayer, SecondInput); if(!!SecondInput) { if(!SecondInput.BufferCreate(opencl)) { DeleteObj(SecondInput); ReturnFalse; } del_second = true; } } else { if(secondNet.layers.Total() <= secondLayer) ReturnFalse; CLayer *layer = secondNet.layers.At(secondLayer); CNeuronBaseOCL *neuron = layer.At(0); SecondInput = neuron.getOutput(); } } //--- CLayer *previous = NULL; CLayer *current = layers.At(0); bool result = true; int total = MathMin(current.Total(), inputVals.Total()); CNeuronBase *neuron = NULL; if(CheckPointer(opencl) == POINTER_INVALID) { for(int i = 0; i < total; i++) { neuron = current.At(i); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } int pos = i; int d = 0; if(window > 1) { d = i % window; pos = (i - d) / window; } neuron.setOutputVal(inputVals.At(i) + (float)(tem ? (d % 2 == 0 ? sin(pos / pow(10000, (2 * d + 1) / (window + 1))) : cos(pos / pow(10000, (2 * d + 1) / (window + 1)))) : 0)); if(tem) inputVals.Update(i, neuron.getOutputVal()); } } else { CNeuronBaseOCL *neuron_ocl = current.At(0); CBufferFloat *inputs = neuron_ocl.getOutput(); if(tem) { int total_data = inputVals.Total(); for(int d = 0; d < total_data; d++) { int pos = d; int dim = 0; if(window > 1) { dim = d % window; pos = (d - dim) / window; } float value = pos / pow(10000, (2 * dim + 1) / (float)(window + 1)); value = (float)(inputVals.At(d) + (dim % 2 == 0 ? sin(value) : cos(value))); if(!inputVals.Update(d, value)) { result = false; break; } } } if(!result || !inputs.Fill(inputVals)) { if(del_second) DeleteObj(SecondInput); ReturnFalse; } } //--- CObject *temp = NULL; #ifdef _DEBUG vector res; #endif for(int l = 1; (l < layers.Total() && result && !IsStopped()); l++) { previous = current; current = layers.At(l); if(CheckPointer(current) == POINTER_INVALID) { result = false; break; } //--- if(CheckPointer(opencl) != POINTER_INVALID) { CNeuronBaseOCL *current_ocl = current.At(0); if(!current_ocl.FeedForward(previous.At(0), SecondInput)) { result = false; break; } #ifdef _DEBUG current_ocl.getOutputVal(res); #endif continue; } //--- total = current.Total(); if(current.At(0).Type() == defNeuron) total--; //--- for(int n = 0; n < total && !IsStopped(); n++) { neuron = current.At(n); if(CheckPointer(neuron) == POINTER_INVALID) { result = false; break; } if(previous.At(0).Type() == defNeuron) { temp = previous; if(!neuron.feedForward(temp)) { result = false; break; } continue; } if(neuron.Type() == defNeuron) { if(n == 0) { CLayer *temp_l = new CLayer(total); if(CheckPointer(temp_l) == POINTER_INVALID) { result = false; break; } CNeuronProof *proof = NULL; for(int p = 0; p < previous.Total(); p++) { proof = previous.At(p); if(CheckPointer(proof) == POINTER_INVALID) { result = false; break; } temp_l.AddArray(proof.getOutputLayer()); } temp = temp_l; } if(!neuron.feedForward(temp)) { result = false; break; } if(n == total - 1) { CLayer *temp_l = temp; temp_l.FreeMode(false); temp_l.Shutdown(); DeleteObj(temp_l); } continue; } temp = previous.At(n); if(CheckPointer(temp) == POINTER_INVALID) { result = false; break; } if(!neuron.feedForward(temp)) { result = false; break; } } } //--- if(del_second) DeleteObj(SecondInput); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPositionEncoder : public CNeuronBaseOCL { protected: CBufferFloat PositionEncoder; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) { return true; } ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronPositionEncoder(void) {}; ~CNeuronPositionEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronPEOCL; }///< Identificator of class.@return Type of class virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPositionEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window, optimization_type, batch)) ReturnFalse; //--- matrix pe = matrix::Zeros(count, window); vector position = vector::Ones(count); position = position.CumSum() - 1; for(uint i = 0; i < window; i += 2) { vector temp = position / MathPow(10000.0f, 2.0f * uint(i / 2) / window); pe.Col(MathSin(temp), i); if((i + 1) < window) pe.Col(MathCos(temp), i + 1); } if(!PositionEncoder.AssignArray(pe)) ReturnFalse; //--- return PositionEncoder.BufferCreate(open_cl); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPositionEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!Gradient || Gradient != NeuronOCL.getGradient()) { if(!!Gradient) DeleteObj(Gradient); Gradient = NeuronOCL.getGradient(); } //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); setBuffer(def_k_MatrixSum, def_k_sum_matrix1, NeuronOCL.getOutputIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, PositionEncoder.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, Output.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, (int)1) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, 1.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0.0f) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0.0f) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPositionEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- return PositionEncoder.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPositionEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- PositionEncoder.BufferFree(); if(!PositionEncoder.Load(file_handle)) ReturnFalse; if(!PositionEncoder.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPositionEncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!OpenCL) PositionEncoder.BufferCreate(OpenCL); else PositionEncoder.BufferFree(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTransposeOCL : public CNeuronBaseOCL { protected: uint iWindow; uint iCount; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronTransposeOCL(void) {}; ~CNeuronTransposeOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronTransposeOCL; }///< Identificator of class.@return Type of class //--- virtual uint GetWindow(void) const { return iWindow; } virtual uint GetCount(void) const { return iCount; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iCount, iWindow}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, Output.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iWindow, iCount}; setBuffer(def_k_Transpose, def_k_tr_matrix_out, NeuronOCL.getGradientIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_in, Gradient.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iCount = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMH2AttentionOCL : public CNeuronBaseOCL { protected: uint iHeads; ///< Number of heads uint iWindow; ///< Input window size uint iUnits; ///< Number of units uint iWindowKey; ///< Size of Key/Query window //--- CNeuronConvOCL Q_Embedding; CNeuronConvOCL KV_Embedding; CNeuronTransposeOCL Transpose; int ScoreIndex; CNeuronBaseOCL MHAttentionOut; CNeuronConvOCL W0; CNeuronBaseOCL AttentionOut; CNeuronConvOCL FF[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool attentionOut(void); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool AttentionInsideGradients(void); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: /** Constructor */ CNeuronMH2AttentionOCL(void); /** Destructor */~CNeuronMH2AttentionOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMH2AttentionOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronMH2AttentionOCL::CNeuronMH2AttentionOCL(void) : iHeads(0), iWindow(0), iUnits(0), iWindowKey(0) { activation = None; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Transpose.Init(0, 0, OpenCL, iUnits, iWindow, optimization_type, batch)) ReturnFalse; Transpose.SetActivationFunction(None); //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iUnits, iUnits, 2 * iWindowKey * iHeads, iWindow, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) FF[i].SetActivationFunction(None); //--- Gradient.BufferFree(); DeleteObj(Gradient); Gradient = FF[1].getGradient(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Transpose.FeedForward(NeuronOCL) || !KV_Embedding.FeedForward(Transpose.AsObject())) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormilize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormilize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindow/*K units*/, iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(GetPointer(Transpose))) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!FF[1].CalcHiddenGradients((CObject *)GetPointer(FF[0]))) ReturnFalse; if(!FF[0].CalcHiddenGradients((CObject *)GetPointer(AttentionOut))) ReturnFalse; if(!SumAndNormilize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!W0.CalcHiddenGradients((CObject *)GetPointer(MHAttentionOut))) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!KV_Embedding.CalcHiddenGradients((CObject *)GetPointer(Transpose))) ReturnFalse; if(!Q_Embedding.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), W0.getGradient(), AttentionOut.getGradient(), iWindow, false)) ReturnFalse; if(!Transpose.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), AttentionOut.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!Q_Embedding.Save(file_handle)) ReturnFalse; if(!KV_Embedding.Save(file_handle)) ReturnFalse; if(!Transpose.Save(file_handle)) ReturnFalse; if(!W0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, Q_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, KV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, Transpose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, W0.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, FF[i].AsObject())) ReturnFalse; if(!SetGradient(FF[1].getGradient(), true)) ReturnFalse; //--- iUnits = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization, iBatch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMH2AttentionOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); Q_Embedding.SetOpenCL(OpenCL); KV_Embedding.SetOpenCL(OpenCL); Transpose.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); W0.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL); FF[0].SetOpenCL(OpenCL); FF[1].SetOpenCL(OpenCL); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iWindow * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMH2AttentionOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMH2AttentionOCL *src = source; if(!Q_Embedding.WeightsUpdate(GetPointer(src.Q_Embedding), tau)) ReturnFalse; if(!KV_Embedding.WeightsUpdate(GetPointer(src.KV_Embedding), tau)) ReturnFalse; if(!W0.WeightsUpdate(GetPointer(src.W0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!FF[i].WeightsUpdate(GetPointer(src.FF[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCGConvOCL : public CNeuronBaseOCL { protected: CNeuronBaseOCL cInputF; CNeuronBaseOCL cInputS; CNeuronBaseOCL cF; CNeuronBaseOCL cS; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronCGConvOCL(void) {}; ~CNeuronCGConvOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); //--- virtual int Type(void) const { return defNeuronCGConvOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cInputF.Init(numNeurons, 0, OpenCL, window, optimization, batch)) ReturnFalse; if(!cInputS.Init(numNeurons, 1, OpenCL, window, optimization, batch)) ReturnFalse; cInputF.SetActivationFunction(None); cInputS.SetActivationFunction(None); //--- if(!cF.Init(0, 2, OpenCL, numNeurons, optimization, batch)) ReturnFalse; cF.SetActivationFunction(SIGMOID); if(!cS.Init(0, 3, OpenCL, numNeurons, optimization, batch)) ReturnFalse; cS.SetActivationFunction(LReLU); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput() || NeuronOCL.getOutputIndex() < 0) ReturnFalse; //--- if(cInputF.getOutputIndex() != NeuronOCL.getOutputIndex()) { if(!cInputF.getOutput().BufferSet(NeuronOCL.getOutputIndex())) ReturnFalse; cInputF.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); } if(cInputS.getOutputIndex() != NeuronOCL.getOutputIndex()) { if(!cInputS.getOutput().BufferSet(NeuronOCL.getOutputIndex())) ReturnFalse; cInputS.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); } //--- if(!cF.FeedForward(GetPointer(cInputF))) ReturnFalse; if(!cS.FeedForward(GetPointer(cInputS))) ReturnFalse; //--- if(!ElementMult(cF.getOutput(), cS.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer || !prevLayer.getGradient() || prevLayer.getGradientIndex() < 0) ReturnFalse; //--- if(!ElementMultGrad(cF.getOutput(), cF.getGradient(), cS.getOutput(), cS.getGradient(), Gradient, cF.Activation(), cS.Activation())) ReturnFalse; //--- if(!cInputF.CalcHiddenGradients((CObject *)GetPointer(cF))) ReturnFalse; if(!cInputS.CalcHiddenGradients((CObject *)GetPointer(cS))) ReturnFalse; if(!SumAndNormilize(cF.getOutput(), cS.getOutput(), prevLayer.getOutput(), 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::ElementMultGrad(const CBufferFloat * input1, CBufferFloat * input1_gr, const CBufferFloat * input2, CBufferFloat * input2_gr, const CBufferFloat * output, const int activation1, const int activation2) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); ResetLastError(); setBuffer(def_k_CGConv_HiddenGradient, def_k_cgc_matrix_f, input1.GetIndex()) setBuffer(def_k_CGConv_HiddenGradient, def_k_cgc_matrix_fg, input1_gr.GetIndex()) setBuffer(def_k_CGConv_HiddenGradient, def_k_cgc_matrix_s, input2.GetIndex()) setBuffer(def_k_CGConv_HiddenGradient, def_k_cgc_matrix_sg, input2_gr.GetIndex()) setBuffer(def_k_CGConv_HiddenGradient, def_k_cgc_matrix_g, output.GetIndex()) setArgument(def_k_CGConv_HiddenGradient, def_k_cgc_activationf, activation1) setArgument(def_k_CGConv_HiddenGradient, def_k_cgc_activations, activation2) kernelExecute(def_k_CGConv_HiddenGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cF.UpdateInputWeights(cInputF.AsObject())) ReturnFalse; if(!cS.UpdateInputWeights(cInputS.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cInputF.Save(file_handle)) ReturnFalse; if(!cInputS.Save(file_handle)) ReturnFalse; if(!cF.Save(file_handle)) ReturnFalse; if(!cS.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cInputF.Type() || !cInputF.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cInputS.Type() || !cInputS.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cF.Type() || !cF.Load(file_handle)) ReturnFalse; if(FileReadInteger(file_handle) != cS.Type() || !cS.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGConvOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cInputF.WeightsUpdate(source, tau)) ReturnFalse; if(!cInputS.WeightsUpdate(source, tau)) ReturnFalse; if(!cF.WeightsUpdate(source, tau)) ReturnFalse; if(!cS.WeightsUpdate(source, tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCGConvOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInputF.SetOpenCL(OpenCL); cInputS.SetOpenCL(OpenCL); cF.SetOpenCL(OpenCL); cS.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMFTOCL : public CNeuronMLMHAttentionOCL { protected: //--- CCollection cTranspose; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool Transpose(CBufferFloat *in, CBufferFloat *out, int rows, int cols); virtual bool MHCA(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *score, CBufferFloat *out); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool MHCAInsideGradients(CBufferFloat *q, CBufferFloat *qg, CBufferFloat *kv, CBufferFloat *kvg, CBufferFloat *score, CBufferFloat *aog); public: CNeuronMFTOCL(void) {}; ~CNeuronMFTOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint features, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); //--- virtual int Type(void) const { return defNeuronMFTOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint features, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * features, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(features, 1); //--- MHSA uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tensor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of output tensor uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor //--- MHCA uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeads * iUnits; //Size of KV tensor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iUnits + 1) * iWindowKey * iHeads; //Size of weights' matrix of KV tenzor uint scores_ca = iUnits * iWindow * iHeads; //Size of Score tensor //--- FF uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- MHSA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cTranspose.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores_ca, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads cross attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- MHSA //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; for(uint w = 0; w < q_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float kv = (float)(1 / sqrt(iUnits + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* kv)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- MHSA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- MHCA temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Transpose(CBufferFloat * in, CBufferFloat * out, int rows, int cols) { if(!in || !out) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {rows, cols}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, in.GetIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, out.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCA(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * score, CBufferFloat * out) { if(!q || !kv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindow, iHeads}; uint local_work_size[3] = {1, iWindow, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, score.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- MHSA //--- Calculate Queries, Keys, Values CBufferFloat *inputs = NeuronOCL.getOutput(); CBufferFloat *qkv = QKV_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 4); if(IsStopped() || !AttentionScore(qkv, temp, false)) ReturnFalse; //--- Multi-heads attention calculation CBufferFloat *out = AO_Tensors.At(i * 4); if(IsStopped() || !AttentionOut(qkv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 8); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow)) ReturnFalse; //--- MHCA inputs = temp; CBufferFloat *q = QKV_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2); if(IsStopped() || !Transpose(inputs, tr, iUnits, iWindow)) ReturnFalse; CBufferFloat *kv = QKV_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), tr, kv, iUnits, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Multi-heads cross attention calculation temp = S_Tensors.At(i * 4 + 1); out = AO_Tensors.At(i * 4 + 1); if(IsStopped() || !MHCA(q, kv, temp, out)) ReturnFalse; //--- Cross Attention out calculation temp = FF_Tensors.At(i * 8 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 8 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 8 + 3); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, Output, iWindow, true, 0, 0, i * inputs.Total())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::MHCAInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, CBufferFloat * score, CBufferFloat * aog) { if(!q || !qg || !kv || !kvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, score.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, aog.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iWindow) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *inp = prevLayer.getOutput(); CBufferFloat *grad = prevLayer.getGradient(); //--- for(int i = 0; (i < (int)iLayers && !IsStopped()); i++) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 3), Gradient, FF_Tensors.At(i * 8 + 2), FF_Tensors.At(i * 8 + 6), 4 * iWindow, iWindow, None, i * inp.Total())) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 8 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(i * 8 + 6), FF_Tensors.At(i * 8 + 1), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradient if(IsStopped() || !SumAndNormilize(Gradient, temp, temp, iWindow, false, i * inp.Total(), 0, 0)) ReturnFalse; out_grad = temp; //--- MHCA //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12) + 1), out_grad, AO_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3), iWindowKey * iHeads, iWindow, None)) ReturnFalse; if(IsStopped() || !MHCAInsideGradients(QKV_Tensors.At(i * 6 + 1), QKV_Tensors.At(i * 6 + 4), QKV_Tensors.At(i * 6 + 2), QKV_Tensors.At(i * 6 + 5), S_Tensors.At(i * 4 + 1), AO_Tensors.At(i * 4 + 3))) ReturnFalse; CBufferFloat *tr = cTranspose.At(i * 2 + 1); if(IsStopped() || !Transpose(QKV_Tensors.At(i * 6 + 5), tr, iWindow, iUnits)) ReturnFalse; //--- Sum temp = FF_Tensors.At(i * 8 + 4); if(IsStopped() || !SumAndNormilize(QKV_Tensors.At(i * 6 + 4), tr, temp, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; //--- MHSA //--- Split gradient to multi-heads out_grad = temp; if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 8 : 12)), out_grad, AO_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 2), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 6), QKV_Tensors.At(i * 6 + 3), S_Tensors.At(i * 4), AO_Tensors.At(i * 4 + 1))) ReturnFalse; //--- if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(i * 6 + 3), inp, tr, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum gradients if(i > 0) { if(IsStopped() || !SumAndNormilize(grad, tr, grad, iWindow, false)) ReturnFalse; if(IsStopped() || !SumAndNormilize(out_grad, grad, grad, iWindow, false)) ReturnFalse; } else if(IsStopped() || !SumAndNormilize(out_grad, tr, grad, iWindow, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), QKV_Tensors.At(l * 6 + 3), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), iWindow, 3 * iWindowKey * iHeads, 0, 3 * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), QKV_Tensors.At(l * 6 + 4), inputs, (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), QKV_Tensors.At(l * 6 + 5), cTranspose.At(l * 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), iUnits, 2 * iWindowKey * iHeads, 0, 2 * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), FF_Tensors.At(l * 8 + 4), AO_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), FF_Tensors.At(l * 8 + 5), AO_Tensors.At(l * 4 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), FF_Tensors.At(l * 8 + 6), FF_Tensors.At(l * 8 + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), FF_Tensors.At(l * 8 + 7), FF_Tensors.At(l * 8 + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayers = FileReadInteger(file_handle, INT_VALUE); iHeads = FileReadInteger(file_handle, INT_VALUE); iWindow = FileReadInteger(file_handle, INT_VALUE); iUnits = FileReadInteger(file_handle, INT_VALUE); iWindowKey = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!QKV_Tensors.Load(file_handle) || !QKV_Weights.Load(file_handle) || !S_Tensors.Load(file_handle) || !AO_Tensors.Load(file_handle) || !FF_Tensors.Load(file_handle) || !FF_Weights.Load(file_handle) || !cTranspose.Load(file_handle)) ReturnFalse; if(!QKV_Tensors.SetOpenCL(OpenCL) || !QKV_Weights.SetOpenCL(OpenCL) || !S_Tensors.SetOpenCL(OpenCL) || !AO_Tensors.SetOpenCL(OpenCL) || !FF_Tensors.SetOpenCL(OpenCL) || !FF_Weights.SetOpenCL(OpenCL) || !cTranspose.SetOpenCL(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMFTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMFTOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMFTOCL *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9)), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9)), (optimization == SGD ? QKV_Weights.At(l * 6 + 3) : QKV_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 6)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? QKV_Weights.At(l * 6 + 4) : QKV_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 7)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), temp.QKV_Weights.At(l * (temp.optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? QKV_Weights.At(l * 6 + 5) : QKV_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : QKV_Weights.At(l * 9 + 8)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12)), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12)), (optimization == SGD ? FF_Weights.At(l * 8 + 4) : FF_Weights.At(l * 12 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 8)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 1), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 1), (optimization == SGD ? FF_Weights.At(l * 8 + 5) : FF_Weights.At(l * 12 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 9)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 2), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 2), (optimization == SGD ? FF_Weights.At(l * 8 + 6) : FF_Weights.At(l * 12 + 6)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 10)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 8 : 12) + 3), temp.FF_Weights.At(l * (temp.optimization == SGD ? 8 : 12) + 3), (optimization == SGD ? FF_Weights.At(l * 8 + 7) : FF_Weights.At(l * 12 + 7)), (optimization == SGD ? NULL : FF_Weights.At(l * 12 + 11)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronXCiTOCL : public CNeuronMLMHAttentionOCL { protected: //--- uint iLPIWindow; uint iLPIStep; uint iBatchCount; //--- CCollection cLPI; CCollection cLPI_Weights; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool XCiT(CBufferFloat *qkv, CBufferFloat *score, CBufferFloat *out); virtual bool BatchNorm(CBufferFloat *inputs, CBufferFloat *options, CBufferFloat *out); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool XCiTInsideGradients(CBufferFloat *qkv, CBufferFloat *qkvg, CBufferFloat *score, CBufferFloat *aog); virtual bool BatchNormInsideGradient(CBufferFloat *inputs, CBufferFloat *inputs_g, CBufferFloat *options, CBufferFloat *out, CBufferFloat *out_g, ENUM_ACTIVATION activation); virtual bool BatchNormUpdateWeights(CBufferFloat *options, CBufferFloat *out_g); public: CNeuronXCiTOCL(void) {}; ~CNeuronXCiTOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronXCiTOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iUnits = fmax(units_count, 1); iHeads = fmax(fmin(heads, iWindow), 1); iWindowKey = fmax((window + iHeads - 1) / iHeads, 1); iLayers = fmax(layers, 1); iLPIWindow = fmax(lpi_window, 1); iLPIStep = 1; //--- XCA uint num = 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tensor uint scores = iWindowKey * iWindowKey * iHeads; //Size of Score tensor uint out = iWindow * iUnits; //Size of output tensor //--- LPI uint lpi1_num = iWindow * iHeads * iUnits; //Size of LPI1 tensor uint lpi1_weights = (iLPIWindow + 1) * iHeads; //Size of weights' matrix of LPI1 tenzor uint lpi2_weights = (iHeads + 1) * 2; //Size of weights' matrix of LPI2 tenzor //--- FF uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- LPI //--- Initilize LPI tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI1 ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI Normalize ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI.Add(temp)) // LPI2 ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi1_weights)) ReturnFalse; for(uint w = 0; w < lpi1_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Normalization int count = (int)lpi1_num * (optimization_type == SGD ? 7 : 9); temp = new CBufferFloat(); if(!temp.BufferInit(count, 0.0f)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize LPI2 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(lpi2_weights)) ReturnFalse; for(uint w = 0; w < lpi2_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- LPI temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi1_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(lpi2_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cLPI_Weights.Add(temp)) ReturnFalse; //--- FF Weights momentus temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } iBatchCount = 1; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiT(CBufferFloat * qkv, CBufferFloat * score, CBufferFloat * out) { if(!OpenCL || !qkv || !score || !out) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; uint local_work_size[3] = {iWindowKey, iUnits, 1}; setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_score, score.GetIndex()) setBuffer(def_k_XCiTFeedForward, def_k_XCiTff_out, out.GetIndex()) ResetLastError(); kernelExecuteLoc(def_k_XCiTFeedForward, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(qkv, temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNorm(CBufferFloat * inputs, CBufferFloat * options, CBufferFloat * out) { if(!OpenCL || !inputs || !options || !out) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); iBatchCount = MathMin(iBatchCount, iBatch); setBuffer(def_k_BatchFeedForward, def_k_bff_inputs, inputs.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_options, options.GetIndex()) setBuffer(def_k_BatchFeedForward, def_k_bff_output, out.GetIndex()) setArgument(def_k_BatchFeedForward, def_k_bff_batch, (int)iBatchCount) setArgument(def_k_BatchFeedForward, def_k_bff_optimization, (int)optimization) setArgument(def_k_BatchFeedForward, def_k_bff_activation, (int)None) ResetLastError(); kernelExecute(def_k_BatchFeedForward, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormInsideGradient(CBufferFloat* inputs, CBufferFloat* inputs_g, CBufferFloat* options, CBufferFloat* out, CBufferFloat* out_g, ENUM_ACTIVATION activ) { if(!OpenCL || !inputs || !inputs_g || !options || !out || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = inputs.Total(); setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_i, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_options, options.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_g, out_g.GetIndex()) setBuffer(def_k_CalcHiddenGradientBatch, def_k_bchg_matrix_ig, inputs_g.GetIndex()) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_activation, (int)activ) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_batch, (int)iBatchCount) setArgument(def_k_CalcHiddenGradientBatch, def_k_bchg_optimization, (int)optimization) ResetLastError(); kernelExecute(def_k_CalcHiddenGradientBatch, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::BatchNormUpdateWeights(CBufferFloat * options, CBufferFloat * out_g) { if(!OpenCL || !options || !out_g) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = out_g.Total(); //--- if(optimization == SGD) { setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsMomentum, def_k_buom_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_learning_rates, lr) setArgument(def_k_UpdateBatchOptionsMomentum, def_k_buom_momentum, alpha) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsMomentum, global_work_offset, global_work_size) } else { setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_options, options.GetIndex()) setBuffer(def_k_UpdateBatchOptionsAdam, def_k_buoa_matrix_g, out_g.GetIndex()) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_l, lr) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b1, b1) setArgument(def_k_UpdateBatchOptionsAdam, def_k_buoa_b2, b2) ResetLastError(); kernelExecute(def_k_UpdateBatchOptionsAdam, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::XCiTInsideGradients(CBufferFloat * qkv, CBufferFloat * qkvg, CBufferFloat * score, CBufferFloat * aog) { if(!OpenCL || !qkv || !qkvg || !score || !aog) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowKey, iUnits, iHeads}; setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv, qkv.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_qkv_g, qkvg.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_scores, score.GetIndex()) setBuffer(def_k_XCiTInsideGradients, def_k_XCiTig_gradient, aog.GetIndex()) ResetLastError(); kernelExecute(def_k_XCiTInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 3 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow)) ReturnFalse; if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 3 * iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLPIWindow, INT_VALUE) || !FileWriteInteger(file_handle, iLPIStep, INT_VALUE) || !FileWriteInteger(file_handle, iBatchCount, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cLPI.Save(file_handle) || !cLPI_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronXCiTOCL::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLPIWindow = FileReadInteger(file_handle, INT_VALUE); iLPIStep = FileReadInteger(file_handle, INT_VALUE); iBatchCount = FileReadInteger(file_handle, INT_VALUE); //--- Loading objects if(!cLPI.Load(file_handle) || !cLPI_Weights.Load(file_handle)) ReturnFalse; if(!cLPI.SetOpenCL(OpenCL) || !cLPI_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Output = FF_Tensors.At(iLayers * 4 - 3); Gradient = FF_Tensors.At(iLayers * 4 - 1); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); cLPI.SetOpenCL(OpenCL); cLPI_Weights.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDOTOCL : public CNeuronBaseOCL { protected: uint iWindowSize; uint iPrevWindowSize; uint iDimension; uint iUnits; uint iHeads; //--- CNeuronConvOCL cProjInput; CNeuronConvOCL cQKV; int iScoreBuffer; CNeuronBaseOCL cRelativePositionsBias; CNeuronBaseOCL MHAttentionOut; CNeuronConvOCL cProj; CNeuronBaseOCL AttentionOut; CNeuronConvOCL cFF1; CNeuronConvOCL cFF2; CNeuronBaseOCL SAttenOut; CNeuronXCiTOCL cCAtten; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool DOT(void); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool updateRelativePositionsBias(void); virtual bool DOTInsideGradients(void); public: CNeuronDOTOCL(void) {}; ~CNeuronDOTOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint dimension, uint heads, uint units_count, uint prev_window, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); //--- virtual int Type(void) const { return defNeuronDOTOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint dimension, uint heads, uint units_count, uint prev_window, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- if(prev_window != window) { if(!cProjInput.Init(0, 0, OpenCL, prev_window, prev_window, window, units_count, optimization_type, batch)) ReturnFalse; } //--- iWindowSize = window; iPrevWindowSize = prev_window; iDimension = dimension; iHeads = heads; iUnits = units_count; //--- if(!cQKV.Init(0, 1, OpenCL, window, window, dimension * heads, units_count, optimization_type, batch)) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- if(!cRelativePositionsBias.Init(1, 2, OpenCL, iUnits * iHeads * 3, optimization_type, batch)) ReturnFalse; if(!MHAttentionOut.Init(0, 3, OpenCL, iUnits * iHeads * iDimension, optimization_type, batch)) ReturnFalse; if(!cProj.Init(0, 4, OpenCL, iHeads * iDimension, iHeads * iDimension, window, iUnits, optimization_type, batch)) ReturnFalse; if(!AttentionOut.Init(0, 5, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cFF1.Init(0, 6, OpenCL, window, window, 4 * window, units_count, optimization_type, batch)) ReturnFalse; if(!cFF2.Init(0, 7, OpenCL, window * 4, window * 4, window, units_count, optimization_type, batch)) ReturnFalse; if(!SAttenOut.Init(0, 8, OpenCL, iUnits * window, optimization_type, batch)) ReturnFalse; if(!cCAtten.Init(0, 9, OpenCL, window, MathMax(window / 2, 3), 8, iUnits, 1, optimization_type, batch)) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; if(iPrevWindowSize != iWindowSize) { if(!cProjInput.FeedForward(inputs) || !cQKV.FeedForward(GetPointer(cProjInput))) ReturnFalse; inputs = GetPointer(cProjInput); } else if(!cQKV.FeedForward(inputs)) ReturnFalse; if(!DOT()) ReturnFalse; if(!cProj.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; if(!SumAndNormilize(inputs.getOutput(), cProj.getOutput(), AttentionOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cFF1.FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.FeedForward(GetPointer(cFF1))) ReturnFalse; if(!SumAndNormilize(AttentionOut.getOutput(), cFF2.getOutput(), SAttenOut.getOutput(), iWindowSize, true)) ReturnFalse; if(!cCAtten.FeedForward(GetPointer(SAttenOut))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOT(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iUnits, iHeads}; uint local_work_size[3] = {iDimension, 1, 1}; setBuffer(def_k_DOTFeedForward, def_k_dot_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_score, iScoreBuffer) setBuffer(def_k_DOTFeedForward, def_k_dot_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTFeedForward, def_k_dot_out, MHAttentionOut.getOutputIndex()) ResetLastError(); kernelExecuteLoc(def_k_DOTFeedForward, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::DOTInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iDimension, iHeads}; setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv, cQKV.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_scores, iScoreBuffer) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_rpb_g, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_DOTInsideGradients, def_k_dotg_gradient, MHAttentionOut.getGradientIndex()) ResetLastError(); kernelExecute(def_k_DOTInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cCAtten.CalcHiddenGradients((CObject *)GetPointer(SAttenOut))) ReturnFalse; if(!cFF2.CalcHiddenGradients((CObject *)GetPointer(cFF1))) ReturnFalse; if(!cFF1.CalcHiddenGradients((CObject *)GetPointer(AttentionOut))) ReturnFalse; if(!SumAndNormilize(AttentionOut.getGradient(), SAttenOut.getGradient(), cProj.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProj.CalcHiddenGradients((CObject *)GetPointer(MHAttentionOut))) ReturnFalse; if(!DOTInsideGradients()) ReturnFalse; //--- if(iPrevWindowSize != iWindowSize) { if(!cQKV.CalcHiddenGradients((CObject *)GetPointer(cProjInput))) ReturnFalse; if(!SumAndNormilize(cProjInput.getGradient(), cProj.getGradient(), cProjInput.getGradient(), iWindowSize, false)) ReturnFalse; if(!cProjInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; } else { if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), cProj.getGradient(), prevLayer.getGradient(), iWindowSize, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDOTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjInput.SetOpenCL(OpenCL); cQKV.SetOpenCL(OpenCL); iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); cRelativePositionsBias.SetOpenCL(OpenCL); MHAttentionOut.SetOpenCL(OpenCL); cProj.SetOpenCL(OpenCL); AttentionOut.SetOpenCL(OpenCL);; cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); SAttenOut.SetOpenCL(OpenCL); cCAtten.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrevWindowSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- ResetLastError(); if(iWindowSize != iPrevWindowSize) if(!cProjInput.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cQKV.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cRelativePositionsBias.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!MHAttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cProj.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!AttentionOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF1.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cFF2.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!SAttenOut.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } if(!cCAtten.Save(file_handle)) { PrintFormat("%s -> %d: %d", __FUNCTION__, __LINE__, GetLastError()); ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindowSize = (uint)FileReadInteger(file_handle); iPrevWindowSize = (uint)FileReadInteger(file_handle); iDimension = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); //--- if(iWindowSize != iPrevWindowSize) if(!LoadInsideLayer(file_handle, GetPointer(cProjInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cRelativePositionsBias))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cProj))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(AttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF1))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF2))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(SAttenOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cCAtten))) ReturnFalse; //--- iScoreBuffer = OpenCL.AddBuffer(sizeof(float) * iUnits * iHeads * 3, CL_MEM_READ_WRITE); if(iScoreBuffer < 0) ReturnFalse; //--- DeleteObj(Output); Output = cCAtten.getOutput(); DeleteObj(Gradient); Gradient = cCAtten.getGradient(); SAttenOut.SetGradientIndex(cFF2.getGradientIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(iWindowSize != iPrevWindowSize) { if(!cProjInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(GetPointer(cProjInput))) ReturnFalse; } else { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; } //--- if(!cProj.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!cFF1.UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!cFF2.UpdateInputWeights(GetPointer(cFF1))) ReturnFalse; if(!cCAtten.UpdateInputWeights(GetPointer(SAttenOut))) ReturnFalse; if(!updateRelativePositionsBias()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronDOTOCL *Source = source; if(iWindowSize != iPrevWindowSize) { if(!cProjInput.WeightsUpdate(GetPointer(Source.cProjInput), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } else { if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; } //--- if(!cProj.WeightsUpdate(GetPointer(Source.cProj), tau)) ReturnFalse; if(!cFF1.WeightsUpdate(GetPointer(Source.cFF1), tau)) ReturnFalse; if(!cFF2.WeightsUpdate(GetPointer(Source.cFF2), tau)) ReturnFalse; if(!cCAtten.WeightsUpdate(GetPointer(Source.cCAtten), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDOTOCL::updateRelativePositionsBias(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {cRelativePositionsBias.Neurons()}; setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_rpb, cRelativePositionsBias.getOutputIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_gradient, cRelativePositionsBias.getGradientIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_m, cRelativePositionsBias.getFirstMomentumIndex()) setBuffer(def_k_RPBUpdateAdam, def_k_rpbw_matrix_v, cRelativePositionsBias.getSecondMomentumIndex()) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b1, b1) setArgument(def_k_RPBUpdateAdam, def_k_rpbw_b2, b2) ResetLastError(); kernelExecute(def_k_RPBUpdateAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::LoadInsideLayer(int file_handle, CNeuronBaseOCL * neuron) { if(!neuron) ReturnFalse; int type = FileReadInteger(file_handle); if(type != neuron.Type()) ReturnFalse; if(!neuron.Load(file_handle)) ReturnFalse; neuron.SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFAQOCL : public CNeuronBaseOCL { protected: //--- CNeuronConvOCL cF; CNeuronBaseOCL cWv; CNeuronBatchNormOCL cNormV; CNeuronBaseOCL cQd; CNeuronXCiTOCL cDQd; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronFAQOCL(void) {}; ~CNeuronFAQOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint heads, uint units_count, uint input_units, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); //--- virtual int Type(void) const { return defNeuronFAQOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint heads, uint units_count, uint input_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- activation = None; //--- if(!cF.Init(0, 0, OpenCL, 3 * window, window, 8, fmax((int)input_units - 2, 1), optimization_type, batch)) ReturnFalse; cF.SetActivationFunction(None); if(!cNormV.Init(8, 1, OpenCL, fmax((int)input_units - 2, 1) * 8, batch, optimization_type)) ReturnFalse; cNormV.SetActivationFunction(None); if(!cWv.Init(units_count * window_out, 2, OpenCL, 8, optimization_type, batch)) ReturnFalse; cWv.SetActivationFunction(SIGMOID); if(!cQd.Init(0, 4, OpenCL, units_count * window_out, optimization_type, batch)) ReturnFalse; cQd.SetActivationFunction(None); if(!cDQd.Init(0, 5, OpenCL, window_out, 3, heads, units_count, 3, optimization_type, batch)) ReturnFalse; cDQd.SetActivationFunction(None); //--- if(Output != cDQd.getOutput()) { Output.BufferFree(); DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { Gradient.BufferFree(); DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- if(!cF.FeedForward(NeuronOCL)) ReturnFalse; if(!cNormV.FeedForward(GetPointer(cF))) ReturnFalse; if(!cWv.FeedForward(GetPointer(cNormV))) ReturnFalse; if(!cQd.FeedForward(GetPointer(cWv))) ReturnFalse; if(!cDQd.FeedForward(GetPointer(cQd))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!cDQd.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cWv.CalcHiddenGradients((CObject *)GetPointer(cQd))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cWv))) ReturnFalse; if(!cNormV.CalcHiddenGradients((CObject *)GetPointer(cF))) ReturnFalse; if(!cF.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cDQd.UpdateInputWeights(GetPointer(cQd))) ReturnFalse; if(!cQd.UpdateInputWeights(GetPointer(cWv))) ReturnFalse; if(!cWv.UpdateInputWeights(GetPointer(cNormV))) ReturnFalse; if(!cNormV.UpdateInputWeights(GetPointer(cF))) ReturnFalse; if(!cF.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cF.Save(file_handle)) ReturnFalse; if(!cNormV.Save(file_handle)) ReturnFalse; if(!cWv.Save(file_handle)) ReturnFalse; if(!cQd.Save(file_handle)) ReturnFalse; if(!cDQd.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cF))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNormV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cWv))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQd))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cDQd))) ReturnFalse; //--- if(Output != cDQd.getOutput()) { DeleteObj(Output); Output = cDQd.getOutput(); } if(Gradient != cDQd.getGradient()) { DeleteObj(Gradient); Gradient = cDQd.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFAQOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cF.WeightsUpdate(source, tau)) ReturnFalse; if(!cNormV.WeightsUpdate(source, tau)) ReturnFalse; if(!cWv.WeightsUpdate(source, tau)) ReturnFalse; if(!cQd.WeightsUpdate(source, tau)) ReturnFalse; if(!cDQd.WeightsUpdate(source, tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFAQOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cF.SetOpenCL(OpenCL); cNormV.SetOpenCL(OpenCL); cWv.SetOpenCL(OpenCL); cQd.SetOpenCL(OpenCL); cDQd.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCrossAttention : public CNeuronMH2AttentionOCL { protected: uint iWindow_K; uint iUnits_K; CNeuronBaseOCL *cContext; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context); virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context); virtual bool attentionOut(void); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CNeuronBaseOCL *Context); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context); virtual bool AttentionInsideGradients(void); public: CNeuronCrossAttention(void) {}; ~CNeuronCrossAttention(void) { DeleteObj(cContext); } //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronCrossAttenOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iWindow_K = fmax(window_k, 1); iUnits_K = fmax(units_k, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!Q_Embedding.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, optimization_type, batch)) ReturnFalse; Q_Embedding.SetActivationFunction(None); if(!KV_Embedding.Init(0, 0, OpenCL, iWindow_K, iWindow_K, 2 * iWindowKey * iHeads, iUnits_K, optimization_type, batch)) ReturnFalse; KV_Embedding.SetActivationFunction(None); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!MHAttentionOut.Init(0, 0, OpenCL, iWindowKey * iUnits * iHeads, optimization_type, batch)) ReturnFalse; MHAttentionOut.SetActivationFunction(None); if(!W0.Init(0, 0, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, optimization_type, batch)) ReturnFalse; W0.SetActivationFunction(None); if(!AttentionOut.Init(0, 0, OpenCL, iWindow * iUnits, optimization_type, batch)) ReturnFalse; AttentionOut.SetActivationFunction(None); if(!FF[0].Init(0, 0, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[0].SetActivationFunction(LReLU); if(!FF[1].Init(0, 0, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, optimization_type, batch)) ReturnFalse; FF[1].SetActivationFunction(None); //--- Gradient.BufferFree(); DeleteObj(Gradient); Gradient = FF[1].getGradient(); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits_K/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits_K, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, ScoreIndex) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, MHAttentionOut.getOutputIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!Context) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, Context.Total(), optimization, iBatch)) ReturnFalse; } //--- if(Context.GetIndex() >= 0) { CBufferFloat *inside = cContext.getOutput(); if(inside.GetIndex() != Context.GetIndex()) inside.BufferSet(Context.GetIndex()); } else { CBufferFloat *inside = cContext.getOutput(); inside.AssignArray(Context); if(!inside.BufferWrite()) ReturnFalse; } //--- return feedForward(NeuronOCL, cContext); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CNeuronBaseOCL *Context) { //--- if(!Q_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!KV_Embedding.FeedForward(Context)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormilize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormilize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, Q_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, Q_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, KV_Embedding.getOutputIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, KV_Embedding.getGradientIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, ScoreIndex) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, MHAttentionOut.getGradientIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits_K) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL * prevLayer, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!SecondInput) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, SecondInput.Total(), optimization, iBatch)) ReturnFalse; } cContext.SetActivationFunction(SecondActivation); //--- if(SecondGradient.GetIndex() >= 0) { CBufferFloat *inside = cContext.getGradient(); if(inside.GetIndex() != SecondGradient.GetIndex()) inside.BufferSet(SecondGradient.GetIndex()); if(!calcInputGradients(prevLayer, cContext)) ReturnFalse; } else { if(!calcInputGradients(prevLayer, cContext)) ReturnFalse; if(cContext.getGradient().GetData(SecondGradient) <= 0) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::calcInputGradients(CNeuronBaseOCL * prevLayer, CNeuronBaseOCL * Context) { if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Context.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!Context) ReturnFalse; if(!cContext) { cContext = new CNeuronBaseOCL(); if(!cContext) ReturnFalse; if(!cContext.Init(0, 0, OpenCL, Context.Total(), optimization, iBatch)) ReturnFalse; } //--- if(Context.GetIndex() >= 0) { CBufferFloat *inside = cContext.getOutput(); if(inside.GetIndex() != Context.GetIndex()) inside.BufferSet(Context.GetIndex()); } else { CBufferFloat *inside = cContext.getOutput(); inside.AssignArray(Context); if(!inside.BufferWrite()) ReturnFalse; DeleteObj(Context); Context = inside; } //--- return updateInputWeights(NeuronOCL, cContext); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CNeuronBaseOCL * Context) { if(!Q_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(Context)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Save(const int file_handle) { if(!CNeuronMH2AttentionOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow_K) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits_K) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossAttention::Load(const int file_handle) { if(!CNeuronMH2AttentionOCL::Load(file_handle)) ReturnFalse; iWindow_K = (uint)FileReadInteger(file_handle); iUnits_K = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits_K * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGTE : public CNeuronBaseOCL { protected: uint iHeads; ///< Number of heads uint iWindow; ///< Input window size uint iUnits; ///< Number of units uint iWindowKey; ///< Size of Key/Query window //--- CNeuronConvOCL cQKV; CNeuronSoftMaxOCL cSoftMax; int ScoreIndex; CNeuronBaseOCL cMHAttentionOut; CNeuronConvOCL cW0; CNeuronBaseOCL cAttentionOut; CNeuronCGConvOCL cGraphConv[2]; CNeuronConvOCL cFF[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool AttentionOut(void); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool AttentionInsideGradients(void); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronGTE(void) {}; ~CNeuronGTE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronGTE; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); ///< Set Training Mode Flag virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); activation = None; //--- if(!cQKV.Init(0, 0, OpenCL, iWindow, iWindow, iWindowKey * 3 * iHeads, iUnits, optimization, iBatch)) ReturnFalse; if(!cSoftMax.Init(0, 1, OpenCL, iWindowKey * 3 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(3 * iHeads * iUnits); //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; //--- if(!cMHAttentionOut.Init(0, 2, OpenCL, iWindowKey * 2 * iHeads * iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cW0.Init(0, 3, OpenCL, 2 * iWindowKey * iHeads, 2 * iWindowKey * iHeads, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- if(!cAttentionOut.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Init(0, 5 + i, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; if(!cFF[i].Init(0, 7 + i, OpenCL, (i == 0 ? iWindow : 4 * iWindow), (i == 0 ? iWindow : 4 * iWindow), (i == 1 ? iWindow : 4 * iWindow), iUnits, optimization, iBatch)) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cSoftMax.FeedForward(GetPointer(cQKV))) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cMHAttentionOut))) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), iWindow, true)) ReturnFalse; if(!cGraphConv[0].FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].FeedForward(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].FeedForward(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].FeedForward(GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormilize(cAttentionOut.getOutput(), cFF[1].getOutput(), Output, iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!cFF[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cGraphConv[1]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cGraphConv[1].calcInputGradients(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormilize(cAttentionOut.getGradient(), Gradient, cW0.getGradient(), iWindow, false)) ReturnFalse; if(!cW0.CalcHiddenGradients((CObject *)GetPointer(cMHAttentionOut))) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!cSoftMax.CalcHiddenGradients((CObject *)GetPointer(cQKV))) ReturnFalse; if(!cQKV.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormilize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cMHAttentionOut))) ReturnFalse; if(!cGraphConv[0].UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; if(!cGraphConv[1].UpdateInputWeights(GetPointer(cGraphConv[0]))) ReturnFalse; if(!cFF[0].UpdateInputWeights(GetPointer(cGraphConv[1]))) ReturnFalse; if(!cFF[1].UpdateInputWeights(GetPointer(cFF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronGTE *Source = source; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].WeightsUpdate(GetPointer(Source.cGraphConv[i]), tau)) ReturnFalse; if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_GTEFeedForward, def_k_gteff_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEFeedForward, def_k_gteff_score, ScoreIndex) setBuffer(def_k_GTEFeedForward, def_k_gteff_out, cAttentionOut.getOutputIndex()) setArgument(def_k_GTEFeedForward, def_k_gteff_dimension, (int)iWindowKey) kernelExecuteLoc(def_k_GTEFeedForward, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv, cQKV.getOutputIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_qkv_g, cQKV.getGradientIndex()) setBuffer(def_k_GTEInsideGradients, def_k_gteig_scores, ScoreIndex) setBuffer(def_k_GTEInsideGradients, def_k_gteig_gradient, cAttentionOut.getGradientIndex()) kernelExecute(def_k_GTEFeedForward, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGTE::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- if(!!OpenCL) if(ScoreIndex >= 0) OpenCL.BufferFree(ScoreIndex); //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) { cGraphConv[i].SetOpenCL(OpenCL); cFF[i].SetOpenCL(OpenCL); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cGraphConv[i].Save(file_handle)) ReturnFalse; if(!cFF[i].Save(file_handle)) ReturnFalse; } //--- if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGTE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cSoftMax))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cMHAttentionOut))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cAttentionOut))) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, GetPointer(cGraphConv[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; } //--- iHeads = (uint)FileReadInteger(file_handle); iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- if(!!OpenCL) { ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); if(ScoreIndex == INVALID_HANDLE) ReturnFalse; } //--- if(cFF[1].getGradient() != Gradient) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGTE::TrainMode(bool flag) { //--- CNeuronBaseOCL::TrainMode(flag); cQKV.TrainMode(flag); cSoftMax.TrainMode(flag); cMHAttentionOut.TrainMode(flag); cW0.TrainMode(flag); cAttentionOut.TrainMode(flag); for(int i = 0; i < 2; i++) { cGraphConv[i].TrainMode(flag); cFF[i].TrainMode(flag); } //--- ScoreIndex = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * 2 * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CResidualConv : public CNeuronBaseOCL { protected: int iWindowOut; //--- CNeuronConvSAMOCL cConvs[3]; CNeuronBatchNormOCL cNorm[3]; CNeuronBaseOCL cTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CResidualConv(void) {}; ~CResidualConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defResidualConv; } //--- methods for working with files virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; ///< Set Training Mode Flag virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!cConvs[0].Init(0, 0, OpenCL, window, window, window_out, count, 1, optimization, iBatch)) ReturnFalse; if(!cNorm[0].Init(0, 1, OpenCL, window_out * count, iBatch, optimization)) ReturnFalse; cNorm[0].SetActivationFunction(GELU); if(!cConvs[1].Init(0, 2, OpenCL, window_out, window_out, window_out, count, 1, optimization, iBatch)) ReturnFalse; if(!cNorm[1].Init(0, 3, OpenCL, window_out * count, iBatch, optimization)) ReturnFalse; cNorm[1].SetActivationFunction(None); //--- if(!cConvs[2].Init(0, 4, OpenCL, window, window, window_out, count, 1, optimization, iBatch)) ReturnFalse; if(!cNorm[2].Init(0, 5, OpenCL, window_out * count, iBatch, optimization)) ReturnFalse; cNorm[2].SetActivationFunction(None); //--- if(!cTemp.Init(0, 6, OpenCL, window * count, optimization, batch)) ReturnFalse; //--- cNorm[1].SetGradient(getGradient()); cNorm[2].SetGradient(getGradient()); SetActivationFunction(None); iWindowOut = (int)window_out; //--- for(int i = 0; i < 3; i++) cConvs[i].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- if(!cConvs[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm[0].FeedForward(GetPointer(cConvs[0]))) ReturnFalse; if(!cConvs[1].FeedForward(GetPointer(cNorm[0]))) ReturnFalse; if(!cNorm[1].FeedForward(GetPointer(cConvs[1]))) ReturnFalse; //--- if(cTemp.getOutput() != NeuronOCL.getOutput()) cTemp.SetOutput(NeuronOCL.getOutput(), true); //--- if(!cConvs[2].FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm[2].FeedForward(GetPointer(cConvs[2]))) ReturnFalse; //--- if(!SumAndNormilize(cNorm[1].getOutput(), cNorm[2].getOutput(), Output, iWindowOut, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!cConvs[2].CalcHiddenGradients(cNorm[2].AsObject())) ReturnFalse; if(!cTemp.CalcHiddenGradients(cConvs[2].AsObject())) ReturnFalse; //--- if(!cConvs[1].CalcHiddenGradients(cNorm[1].AsObject())) ReturnFalse; if(!cNorm[0].CalcHiddenGradients(cConvs[1].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cNorm[0].AsObject())) ReturnFalse; //--- if(!SumAndNormilize(prevLayer.getGradient(), cTemp.getGradient(), prevLayer.getGradient(), iWindowOut, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { for(int i = 2; i >= 0; i--) { if(!cNorm[i].UpdateInputWeights(cConvs[i].AsObject())) ReturnFalse; } if(!cConvs[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConvs[1].UpdateInputWeights(cNorm[0].AsObject())) ReturnFalse; if(!cConvs[2].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CResidualConv *temp = source; for(uint i = 0; i < cNorm.Size(); i--) if(!cNorm[i].WeightsUpdate(temp.cNorm[i].AsObject(), tau)) ReturnFalse; for(uint i = 0; i < cConvs.Size(); i--) if(!cConvs[i].WeightsUpdate(temp.cConvs[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CResidualConv::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 3; i++) { cConvs[i].SetOpenCL(OpenCL); cNorm[i].SetOpenCL(OpenCL); } cTemp.SetOpenCL(OpenCL); //--- if(cNorm[2].getGradient() != getGradient()) { SetGradient(cNorm[2].getGradient()); cNorm[1].SetGradient(cNorm[2].getGradient()); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CResidualConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); for(int i = 0; i < 3; i++) { cConvs[i].TrainMode(bTrain); cNorm[i].TrainMode(bTrain); } cTemp.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) { if(!cConvs[i].Save(file_handle)) ReturnFalse; if(!cNorm[i].Save(file_handle)) ReturnFalse; } if(!cTemp.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iWindowOut) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CResidualConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) { if(!LoadInsideLayer(file_handle, GetPointer(cConvs[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNorm[i]))) ReturnFalse; } if(!LoadInsideLayer(file_handle, GetPointer(cTemp))) ReturnFalse; iWindowOut = FileReadInteger(file_handle); //--- if(cNorm[2].getGradient() != getGradient()) { SetGradient(cNorm[2].getGradient()); cNorm[1].SetGradient(cNorm[2].getGradient()); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CCCMREncoder : public CNeuronBaseOCL { protected: CResidualConv cResidual[6]; CNeuronConvOCL cInput; CNeuronBatchNormOCL cNorm; CNeuronConvOCL cOutput; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CCCMREncoder(void) {}; ~CCCMREncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defCCMREncoder; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); ///< Set Training Mode Flag virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!cInput.Init(0, 0, OpenCL, window, window, 32, count, optimization, iBatch)) ReturnFalse; if(!cNorm.Init(0, 1, OpenCL, 32 * count, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(LReLU); //--- if(!cResidual[0].Init(0, 2, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[1].Init(0, 3, OpenCL, 32, 32, count, optimization, iBatch)) ReturnFalse; if(!cResidual[2].Init(0, 4, OpenCL, 32, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[3].Init(0, 5, OpenCL, 64, 64, count, optimization, iBatch)) ReturnFalse; if(!cResidual[4].Init(0, 6, OpenCL, 64, 128, count, optimization, iBatch)) ReturnFalse; if(!cResidual[5].Init(0, 7, OpenCL, 128, 128, count, optimization, iBatch)) ReturnFalse; //--- if(!cOutput.Init(0, 8, OpenCL, 128, 128, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cInput.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].FeedForward(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].FeedForward(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.FeedForward(GetPointer(cResidual[5]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!cOutput.CalcHiddenGradients((CObject *)GetPointer(cResidual[5]))) ReturnFalse; for(int i = 5; i > 0; i--) if(!cResidual[i].CalcHiddenGradients((CObject *)GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cResidual[0].CalcHiddenGradients((CObject *)GetPointer(cNorm))) ReturnFalse; if(!cNorm.CalcHiddenGradients((CObject *)GetPointer(cInput))) ReturnFalse; if(!cInput.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cInput.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(GetPointer(cInput))) ReturnFalse; if(!cResidual[0].UpdateInputWeights(GetPointer(cNorm))) ReturnFalse; for(int i = 1; i < 6; i++) if(!cResidual[i].UpdateInputWeights(GetPointer(cResidual[i - 1]))) ReturnFalse; if(!cOutput.UpdateInputWeights(GetPointer(cResidual[5]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CCCMREncoder *Source = source; if(!cInput.WeightsUpdate(GetPointer(Source.cInput), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(GetPointer(Source.cNorm), tau)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].WeightsUpdate(GetPointer(Source.cResidual[i]), tau)) ReturnFalse; if(!cOutput.WeightsUpdate(GetPointer(Source.cOutput), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; for(int i = 0; i < 6; i++) if(!cResidual[i].Save(file_handle)) ReturnFalse; if(!cOutput.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCCMREncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cInput))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cNorm))) ReturnFalse; for(int i = 0; i < 6; i++) if(!LoadInsideLayer(file_handle, GetPointer(cResidual[i]))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cOutput))) ReturnFalse; //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCCMREncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInput.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cOutput.SetOpenCL(OpenCL); for(int i = 0; i < 6; i++) cResidual[i].SetOpenCL(OpenCL); //--- if(Output != cOutput.getOutput()) { DeleteObj(Output); Output = cOutput.getOutput(); } //--- if(Gradient != cOutput.getGradient()) { DeleteObj(Gradient); Gradient = cOutput.getGradient(); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCCMREncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cInput.TrainMode(flag); cNorm.TrainMode(flag); cOutput.TrainMode(flag); for(int i = 0; i < 6; i++) cResidual[i].TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCrossXCiTOCL : public CNeuronXCiTOCL { protected: CCollection cConcat; CCollection cValue; CCollection cV_Weights; CBufferFloat TempBuffer; uint iWindow2; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Motion); //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Motion); public: CNeuronCrossXCiTOCL(void) {}; ~CNeuronCrossXCiTOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window1, uint window2, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CNeuronBaseOCL *Motion); virtual int Type(void) const { return defNeuronCrossXCiTOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window1, uint window2, uint lpi_window, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronXCiTOCL::Init(numOutputs, myIndex, open_cl, window1, lpi_window, heads, units_count, layers, optimization_type, batch)) ReturnFalse; //--- Cross XCA iWindow2 = fmax(window2, 1); uint num = iWindowKey * iHeads * iUnits; //Size of V tensor uint v_weights = (iWindow2 + 1) * iWindowKey * iHeads; //Size of weights' matrix of V tensor //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize V tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cValue.Add(temp)) ReturnFalse; //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } //--- XCiT //--- Initilize V weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(v_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < v_weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { //--- XCiT temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(v_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cV_Weights.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Motion) { if(!NeuronOCL || !Motion) ReturnFalse; //--- for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(4 * i - 2)); CBufferFloat *qkv = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, qkv, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; CBufferFloat *v = cValue.At(i * 2); if(IsStopped() || !ConvolutionForward(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), Motion, v, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !Concat(qkv, v, cConcat.At(2 * i), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- Score calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !XCiT(cConcat.At(2 * i), temp, out)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; //--- LPI inputs = out; temp = cLPI.At(i * 6); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), inputs, temp, iLPIWindow, iHeads, LReLU, iLPIStep)) ReturnFalse; out = cLPI.At(i * 6 + 1); if(IsStopped() || !BatchNorm(temp, cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), out)) ReturnFalse; temp = out; out = cLPI.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, out, 2 * iHeads, 2, None, iHeads)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = out; temp = FF_Tensors.At(i * 4); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 4 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Concat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * output, int window1, int window2, int count) { return Concat(input1, input2, input1, input2, output, window1, window2, 0, 0, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Concat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * input3, CBufferFloat * output, int window1, int window2, int window3, int count) { return Concat(input1, input2, input3, input1, output, window1, window2, window3, 0, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Concat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * input3, CBufferFloat * input4, CBufferFloat * output, int window1, int window2, int window3, int window4, int count) { return Concat(input1.GetIndex(), input2.GetIndex(), input3.GetIndex(), input4.GetIndex(), output.GetIndex(), window1, window2, window3, window4, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Concat(int input1, int input2, int input3, int input4, int output, int window1, int window2, int window3, int window4, int count) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; setBuffer(def_k_ConcatenateMatrix, def_k_conc_input1, input1) setArgument(def_k_ConcatenateMatrix, def_k_conc_window1, window1) setBuffer(def_k_ConcatenateMatrix, def_k_conc_input2, input2) setArgument(def_k_ConcatenateMatrix, def_k_conc_window2, window2) setBuffer(def_k_ConcatenateMatrix, def_k_conc_input3, input3) setArgument(def_k_ConcatenateMatrix, def_k_conc_window3, window3) setBuffer(def_k_ConcatenateMatrix, def_k_conc_input4, input4) setArgument(def_k_ConcatenateMatrix, def_k_conc_window4, window4) setBuffer(def_k_ConcatenateMatrix, def_k_conc_out, output) kernelExecute(def_k_ConcatenateMatrix, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DeConcat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * output, int window1, int window2, int count) { return DeConcat(input1, input2, input1, input2, output, window1, window2, 0, 0, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DeConcat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * input3, CBufferFloat * output, int window1, int window2, int window3, int count) { return DeConcat(input1, input2, input3, input1, output, window1, window2, window3, 0, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DeConcat(CBufferFloat * input1, CBufferFloat * input2, CBufferFloat * input3, CBufferFloat * input4, CBufferFloat * output, int window1, int window2, int window3, int window4, int count) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = count; setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output1, input1.GetIndex()) setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window1, window1) setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output2, input2.GetIndex()) setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window2, window2) setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output3, input3.GetIndex()) setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window3, window3) setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_output4, input4.GetIndex()) setArgument(def_k_DeconcatenateMatrix, def_k_dconc_window4, window4) setBuffer(def_k_DeconcatenateMatrix, def_k_dconc_inputs, output.GetIndex()) kernelExecute(def_k_DeconcatenateMatrix, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::calcInputGradients(CNeuronBaseOCL * prevLayer, CNeuronBaseOCL * Motion) { if(!prevLayer || !Motion) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6) + 1), out_grad, FF_Tensors.At(i * 4), FF_Tensors.At(i * 4 + 2), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = cLPI.At(i * 6 + 5); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 4 : 6)), FF_Tensors.At(i * 4 + 1), cLPI.At(i * 6 + 2), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient through LPI if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 2), temp, cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), 2 * iHeads, 2, None, 0, iHeads)) ReturnFalse; if(IsStopped() || !BatchNormInsideGradient(cLPI.At(i * 6), cLPI.At(i * 6 + 3), cLPI_Weights.At(i * (optimization == SGD ? 5 : 7) + 1), cLPI.At(i * 6 + 1), cLPI.At(i * 6 + 4), LReLU)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(cLPI_Weights.At(i * (optimization == SGD ? 5 : 7)), cLPI.At(i * 6 + 3), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iLPIWindow, iHeads, None, 0, iLPIStep)) ReturnFalse; temp = AO_Tensors.At(i * 2 + 1); //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; out_grad = temp; //--- Passing gradient to query, key and value if(IsStopped() || !XCiTInsideGradients(cConcat.At(i * 2), cConcat.At(i * 2 + 1), S_Tensors.At(i * 2), temp)) ReturnFalse; if(IsStopped() || !DeConcat(QKV_Tensors.At(i * 2 + 1), cValue.At(i * 2 + 1), cConcat.At(i * 2 + 1), 2 * iWindowKey * iHeads, iWindowKey * iHeads, iUnits)) ReturnFalse; //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 4 - 1); inp = FF_Tensors.At(i * 4 - 3); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, 2 * iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) out_grad = temp; if(i == iLayers - 1) { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), Motion.getGradient(), iWindow, iWindowKey * iHeads, None)) ReturnFalse; } else { if(IsStopped() || !ConvolutionInputGradients(cV_Weights.At(i * (optimization == SGD ? 2 : 3)), cValue.At(i * 2 + 1), Motion.getOutput(), GetPointer(TempBuffer), iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(TempBuffer), Motion.getGradient(), Motion.getGradient(), iWindow2, false)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Motion) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, 2 * iWindowKey * iHeads)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), cValue.At(l * 2 + 1), inputs, (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), cLPI.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), iLPIWindow, iHeads, iLPIStep)) ReturnFalse; if(IsStopped() || !BatchNormUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 1), cLPI.At(l * 6 + 4))) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), cLPI.At(l * 6 + 5), cLPI.At(l * 6 + 1), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), 2 * iHeads, 2, iHeads)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), FF_Tensors.At(l * 4 + 2), cLPI.At(l * 6 + 2), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), FF_Tensors.At(l * 4 + 3), FF_Tensors.At(l * 4), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), 4 * iWindow, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 4 + 1); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(CheckPointer(source) == POINTER_INVALID || source.Type() != Type()) ReturnFalse; CNeuronCrossXCiTOCL *temp = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.cV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? cV_Weights.At(l * 2 + 1) : cV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : cV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7)), (optimization == SGD ? cLPI_Weights.At(l * 5 + 3) : cLPI_Weights.At(l * 7 + 3)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 5)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), temp.cLPI_Weights.At(l * (optimization == SGD ? 5 : 7) + 2), (optimization == SGD ? cLPI_Weights.At(l * 5 + 4) : cLPI_Weights.At(l * 7 + 4)), (optimization == SGD ? NULL : cLPI_Weights.At(l * 7 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6)), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6)), (optimization == SGD ? FF_Weights.At(l * 4 + 2) : FF_Weights.At(l * 6 + 2)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 4)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), temp.FF_Weights.At(l * (optimization == SGD ? 4 : 6) + 1), (optimization == SGD ? FF_Weights.At(l * 4 + 3) : FF_Weights.At(l * 6 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 6 + 5)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCrossXCiTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronXCiTOCL::SetOpenCL(obj); cConcat.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cV_Weights.SetOpenCL(OpenCL); TempBuffer.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Save(const int file_handle) { if(!CNeuronXCiTOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, (int)iWindow2, INT_VALUE)) ReturnFalse; //--- Saving objects if(!cValue.Save(file_handle) || !cV_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossXCiTOCL::Load(const int file_handle) { if(!CNeuronXCiTOCL::Load(file_handle)) ReturnFalse; //--- Loading constant iWindow2 = (uint)FileReadInteger(file_handle); //--- Loading objects if(!cValue.Load(file_handle) || !cV_Weights.Load(file_handle)) ReturnFalse; //--- uint num = iWindowKey * iHeads * iUnits; //Size of V tensor //--- CBufferFloat *temp = NULL; for(uint i = 0; i < iLayers; i++) { for(int d = 0; d < 2; d++) { //--- XCiT //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(3 * num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cConcat.Add(temp)) ReturnFalse; } } //--- TempBuffer.BufferInit(iWindow2 * iUnits, 0); if(!TempBuffer.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCCMROCL : public CNeuronBaseOCL { protected: CCCMREncoder FeatureExtractor; CNeuronBaseOCL PrevFeatures; CNeuronBaseOCL Motion; CNeuronBaseOCL Temp; CCCMREncoder LocalContext; CNeuronXCiTOCL GlobalContext; CNeuronCrossXCiTOCL MotionContext; CNeuronLSTMOCL RecurentUnit; CNeuronConvOCL UpScale; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronCCMROCL(void) {}; ~CNeuronCCMROCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronCCMROCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); ///< Set Training Mode Flag virtual bool Clear(void); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- if(!FeatureExtractor.Init(0, 0, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!PrevFeatures.Init(0, 1, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(!Motion.Init(0, 2, OpenCL, 16 * count, optimization, iBatch)) ReturnFalse; if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); //--- if(!Temp.Init(0, 3, OpenCL, window * count, optimization, iBatch)) ReturnFalse; if(!LocalContext.Init(0, 4, OpenCL, window, 16, count, optimization, iBatch)) ReturnFalse; if(!GlobalContext.Init(0, 5, OpenCL, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!MotionContext.Init(0, 6, OpenCL, 16, 16, 3, 4, count, 4, optimization, iBatch)) ReturnFalse; if(!RecurentUnit.Init(0, 7, OpenCL, 16 * count, optimization, iBatch) || !RecurentUnit.SetInputs(16 * count)) ReturnFalse; if(!UpScale.Init(0, 8, OpenCL, 16, 16, window_out, count, optimization, iBatch)) ReturnFalse; //--- if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Delta Features if(!SumAndNormilize(FeatureExtractor.getOutput(), FeatureExtractor.getOutput(), PrevFeatures.getOutput(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!FeatureExtractor.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(FeatureExtractor.getOutput(), PrevFeatures.getOutput(), Motion.getOutput(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- Context if(Temp.getOutputIndex() != NeuronOCL.getOutputIndex()) { CBufferFloat* temp = Temp.getOutput(); temp.BufferSet(NeuronOCL.getOutputIndex()); Temp.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); } if(!LocalContext.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalContext.FeedForward(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.FeedForward(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; //--- Flow if(!RecurentUnit.FeedForward(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.FeedForward(GetPointer(RecurentUnit))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!UpScale.CalcHiddenGradients((CObject *)GetPointer(RecurentUnit))) ReturnFalse; if(!RecurentUnit.CalcHiddenGradients((CObject *)GetPointer(MotionContext))) ReturnFalse; //--- if(!MotionContext.calcInputGradients(GetPointer(GlobalContext), GetPointer(Motion))) ReturnFalse; if(!GlobalContext.CalcHiddenGradients((CObject *)GetPointer(LocalContext))) ReturnFalse; if(!LocalContext.CalcHiddenGradients((CObject *)GetPointer(Temp))) ReturnFalse; //--- if(!FeatureExtractor.CalcHiddenGradients((CObject *)prevLayer)) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), Temp.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!FeatureExtractor.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!LocalContext.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!GlobalContext.UpdateInputWeights(GetPointer(LocalContext))) ReturnFalse; if(!MotionContext.UpdateInputWeights(GetPointer(GlobalContext), Motion.getOutput())) ReturnFalse; if(!RecurentUnit.UpdateInputWeights(GetPointer(MotionContext))) ReturnFalse; if(!UpScale.UpdateInputWeights(GetPointer(RecurentUnit))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronCCMROCL *Source = source; if(!FeatureExtractor.WeightsUpdate(GetPointer(Source.FeatureExtractor), tau)) ReturnFalse; if(!LocalContext.WeightsUpdate(GetPointer(Source.LocalContext), tau)) ReturnFalse; if(!GlobalContext.WeightsUpdate(GetPointer(Source.GlobalContext), tau)) ReturnFalse; if(!MotionContext.WeightsUpdate(GetPointer(Source.MotionContext), tau)) ReturnFalse; if(!RecurentUnit.WeightsUpdate(GetPointer(Source.RecurentUnit), tau)) ReturnFalse; if(!UpScale.WeightsUpdate(GetPointer(Source.UpScale), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Clear(void) { if(!RecurentUnit.Clear()) ReturnFalse; //--- CBufferFloat *temp = FeatureExtractor.getOutput(); temp.BufferInit(temp.Total(), 0); if(!temp.BufferWrite()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!FeatureExtractor.Save(file_handle)) ReturnFalse; if(!PrevFeatures.Save(file_handle)) ReturnFalse; if(!Motion.Save(file_handle)) ReturnFalse; if(!Temp.Save(file_handle)) ReturnFalse; if(!LocalContext.Save(file_handle)) ReturnFalse; if(!GlobalContext.Save(file_handle)) ReturnFalse; if(!MotionContext.Save(file_handle)) ReturnFalse; if(!RecurentUnit.Save(file_handle)) ReturnFalse; if(!UpScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCCMROCL::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(FeatureExtractor))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(PrevFeatures))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Motion))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(Temp))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(LocalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(GlobalContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(MotionContext))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(RecurentUnit))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(UpScale))) ReturnFalse; //--- if(Motion.getGradientIndex() != FeatureExtractor.getGradientIndex()) Motion.SetGradientIndex(FeatureExtractor.getGradientIndex()); if(UpScale.getGradientIndex() != getGradientIndex()) SetGradientIndex(UpScale.getGradientIndex()); if(UpScale.getOutputIndex() != getOutputIndex()) Output.BufferSet(UpScale.getOutputIndex()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCCMROCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); FeatureExtractor.SetOpenCL(OpenCL); PrevFeatures.SetOpenCL(OpenCL); Motion.SetOpenCL(OpenCL); Temp.SetOpenCL(OpenCL); LocalContext.SetOpenCL(OpenCL); GlobalContext.SetOpenCL(OpenCL); MotionContext.SetOpenCL(OpenCL); RecurentUnit.SetOpenCL(OpenCL); UpScale.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCCMROCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); FeatureExtractor.TrainMode(bTrain); PrevFeatures.TrainMode(bTrain); Motion.TrainMode(bTrain); Temp.TrainMode(bTrain); LocalContext.TrainMode(bTrain); GlobalContext.TrainMode(bTrain); MotionContext.TrainMode(bTrain); RecurentUnit.TrainMode(bTrain); UpScale.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronNODEOCL : public CNeuronBaseOCL { protected: uint iDimension; uint iVariables; uint iLenth; int iBuffersK[]; int iInputsK[]; int iMeadl[]; CBufferFloat cAlpha; CBufferFloat cTemp; CCollection cBeta; CBufferFloat cSolution; CCollection cWeights; //--- virtual bool CalculateKBuffer(int k); virtual bool CalculateInputK(CBufferFloat* inputs, int k); virtual bool CalculateOutput(CBufferFloat* inputs); virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool CalculateOutputGradient(CBufferFloat* inputs); virtual bool CalculateInputKGradient(CBufferFloat* inputs, int k); virtual bool CalculateKBufferGradient(int k); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronNODEOCL(void) {}; ~CNeuronNODEOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint variables, uint lenth, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronNODEOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint variables, uint lenth, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * variables * lenth, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iVariables = variables; iLenth = lenth; //--- uint mult = 2; uint weights = (iDimension + 2) * iDimension * iVariables; //--- if(ArrayResize(iBuffersK, 18) < 18) ReturnFalse; if(ArrayResize(iInputsK, 18) < 18) ReturnFalse; if(ArrayResize(iMeadl, 12) < 12) ReturnFalse; CBufferFloat *temp = NULL; //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iInputsK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Meadl Output and Gradient buffers iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- Initilize Weights for(int i = 0; i < 2; i++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(weights)) ReturnFalse; float k = (float)(1 / sqrt(iDimension + 2)); for(uint w = 0; w < weights; w++) { if(!temp.Add((GenerateWeight() - 0.5f)* k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; //--- for(uint d = 0; d < 2; d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!cWeights.Add(temp)) ReturnFalse; } } //--- Constants //--- Alpha { float temp_ar[] = {0, 0.2f, 0.3f, 0.8f, 8.0f / 9, 1, 1}; if(!cAlpha.AssignArray(temp_ar)) ReturnFalse; if(!cAlpha.BufferCreate(OpenCL)) ReturnFalse; } //--- Beta K1 { float temp_ar[] = {0, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K2 { float temp_ar[] = {0.2f, 0, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K3 { float temp_ar[] = {3.0f / 40, 9.0f / 40, 0, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K4 { float temp_ar[] = {44.0f / 44, -56.0f / 15, 32.0f / 9, 0, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K5 { float temp_ar[] = {19372.0f / 6561, -25360 / 2187.0f, 64448 / 6561.0f, -212.0f / 729, 0, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Beta K6 { float temp_ar[] = {9017 / 3168.0f, -355 / 33.0f, 46732 / 5247.0f, 49.0f / 176, -5103.0f / 18656, 0}; temp = new CBufferFloat(); if(!temp || !temp.AssignArray(temp_ar)) { DeleteObj(temp); ReturnFalse; } if(!temp.BufferCreate(OpenCL)) { DeleteObj(temp); ReturnFalse; } if(!cBeta.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- Yt+1 { float temp_ar[] = {35.0f / 384, 0, 500.0f / 1113, 125.0f / 192, -2187.0f / 6784, 11.0f / 84}; if(!cSolution.AssignArray(temp_ar)) ReturnFalse; if(!cSolution.BufferCreate(OpenCL)) ReturnFalse; } //--- if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputK(CBufferFloat * inputs, int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- if(k == 0) { if(iInputsK[k] != inputs.GetIndex()) { OpenCL.BufferFree(iInputsK[k]); iInputsK[k] = inputs.GetIndex(); } return true; } //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k]) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBuffer(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iInputsK[k]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iMeadl[k * 2]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, float(cAlpha.At(k))) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(LReLU)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) //--- setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_FeedForwardNODEF, def_k_ffdoprif_matrix_o, iBuffersK[k]) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_dimension, int(iDimension)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_step, cAlpha.At(k)) setArgument(def_k_FeedForwardNODEF, def_k_ffdoprif_activation, int(None)) kernelExecute(def_k_FeedForwardNODEF, global_work_offset, global_work_size) //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutput(CBufferFloat * inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[0]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[1]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[2]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[3]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[4]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[5]) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_FeedForwardNODEInpK, def_k_ffdopriInp_matrix_o, Output.GetIndex()) kernelExecute(def_k_FeedForwardNODEInpK, global_work_offset, global_work_size) //--- if(!SumAndNormilize(Output, inputs, Output, iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { for(int k = 0; k < 6; k++) { if(!CalculateInputK(NeuronOCL.getOutput(), k)) ReturnFalse; if(!CalculateKBuffer(k)) ReturnFalse; } //--- return CalculateOutput(NeuronOCL.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateOutputGradient(CBufferFloat * inputs) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[6]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[7]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[8]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[9]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[10]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[11]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, cSolution.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, Gradient.GetIndex()) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateKBufferGradient(int k) { if(k < 0) ReturnFalse; if(iInputsK.Size() / 3 <= uint(k)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension, iVariables, iLenth}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iMeadl[k * 2]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iMeadl[k * 2 + 1]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iBuffersK[k + 6]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(LReLU)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) //--- setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_i, iInputsK[k]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_ig, iInputsK[k + 12]) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_HiddenGradientNODEF, def_k_hddoprif_matrix_g, iMeadl[k * 2 + 1]) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_dimension_out, int(iDimension)) setArgument(def_k_HiddenGradientNODEF, def_k_hddoprif_activation, int(None)) kernelExecute(def_k_HiddenGradientNODEF, global_work_offset, global_work_size) //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::CalculateInputKGradient(CBufferFloat * inputs, int k) { //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_i, inputs.GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k1, iBuffersK[12]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k2, iBuffersK[13]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k3, iBuffersK[14]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k4, iBuffersK[15]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k5, iBuffersK[16]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_k6, iBuffersK[17]) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_beta, ((CBufferFloat *)cBeta.At(k)).GetIndex()) setBuffer(def_k_HiddenGradientNODEInpK, def_k_ffdopriInp_matrix_o, iInputsK[k + 6]) kernelExecute(def_k_HiddenGradientNODEInpK, global_work_offset, global_work_size) //--- for(int i = k - 1; i >= 0; i--) { float mult = 1.0f / (i == (k - 1) ? 6 - k : 1); uint global_work_offset[1] = {0}; uint global_work_size[1] = {iLenth * iVariables}; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, iBuffersK[k + 6]) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, iBuffersK[k + 12]) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, iBuffersK[k + 6]) setArgument(def_k_MatrixSum, def_k_sum_dimension, iDimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 0) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 0) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, mult) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!CalculateOutputGradient(prevLayer.getGradient())) ReturnFalse; for(int k = 5; k >= 0; k--) { if(!CalculateKBufferGradient(k)) ReturnFalse; if(!CalculateInputKGradient(GetPointer(cTemp), k)) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getOutput(), iDimension, false, 0, 0, 0, 1.0f / (k == 0 ? 6 : 1))) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iDimension + 2, iDimension, iVariables}; ResetLastError(); setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iInputsK[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iMeadl[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iInputsK[1]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iMeadl[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iInputsK[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iMeadl[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iInputsK[3]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iMeadl[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iInputsK[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iMeadl[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iInputsK[5]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iMeadl[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(0)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(1)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(2)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) //--- setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik1, iMeadl[0]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk1, iBuffersK[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik2, iMeadl[2]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk2, iBuffersK[7]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik3, iMeadl[4]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk3, iBuffersK[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik4, iMeadl[6]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk4, iBuffersK[9]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik5, iMeadl[8]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk5, iBuffersK[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_ik6, iMeadl[10]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_gk6, iBuffersK[11]) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_w, ((CBufferFloat*)cWeights.At(3)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_m, ((CBufferFloat*)cWeights.At(4)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_matrix_v, ((CBufferFloat*)cWeights.At(5)).GetIndex()) setBuffer(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_alpha, cAlpha.GetIndex()) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_lenth, int(iLenth)) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_l, lr) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b1, b1) setArgument(def_k_NODEF_UpdateWeightsAdam, def_k_uwdoprif_b2, b2) kernelExecute(def_k_NODEF_UpdateWeightsAdam, global_work_offset, global_work_size) //-- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWeights.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iVariables), INT_VALUE) < INT_VALUE || FileWriteInteger(file_handle, int(iLenth), INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNODEOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cWeights.Load(file_handle)) ReturnFalse; cWeights.SetOpenCL(OpenCL); //--- iDimension = (int)FileReadInteger(file_handle); iVariables = (int)FileReadInteger(file_handle); iLenth = (int)FileReadInteger(file_handle); //--- CBufferFloat *temp = NULL; for(uint i = 0; i < 18; i++) { OpenCL.BufferFree(iBuffersK[i]); OpenCL.BufferFree(iInputsK[i]); //--- iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iBuffersK[i] < 0) ReturnFalse; if(i > 11) continue; //--- Initilize Output and Gradient buffers OpenCL.BufferFree(iMeadl[i]); iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(iMeadl[i] < 0) ReturnFalse; } //--- cTemp.BufferFree(); if(!cTemp.BufferInit(Output.Total(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronNODEOCL::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) return; //--- CNeuronBaseOCL::SetOpenCL(obj); cWeights.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); cAlpha.BufferCreate(OpenCL); cBeta.SetOpenCL(OpenCL); cSolution.BufferCreate(OpenCL); //--- for(uint i = 0; i < 18; i++) { iBuffersK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); iInputsK[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); if(i > 11) continue; iMeadl[i] = OpenCL.AddBuffer(sizeof(float) * Output.Total(), CL_MEM_READ_WRITE); } //--- } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConformer : public CNeuronBaseOCL { protected: //--- int iWindow; int iDimension; int iHeads; int iVariables; int iCount; //--- CNeuronConvOCL cQKV; CNeuronBaseOCL cdQKV; int iScore; CNeuronBaseOCL cAttentionOut; CNeuronConvOCL cW0; CNeuronNODEOCL cNODE[3]; CNeuronConvOCL cFF[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool attentionOut(void); //--- virtual bool AttentionInsideGradients(void); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronConformer(void) : iScore(-1) {}; ~CNeuronConformer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint variables, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronConformerOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint variables, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * variables * units_count, optimization_type, batch)) ReturnFalse; if(!cQKV.Init(0, 0, OpenCL, window, window, 3 * window_key * heads, variables * units_count, optimization, iBatch)) ReturnFalse; //--- iWindow = int(fmax(window, 1)); iDimension = int(fmax(window_key, 1)); iHeads = int(fmax(heads, 1)); iVariables = int(fmax(variables, 1)); iCount = int(fmax(units_count, 1)); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cW0.Init(0, 3, OpenCL, iDimension * iHeads, iDimension * iHeads, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- for(int i = 0; i < 3; i++) if(!cNODE[i].Init(0, 4 + i, OpenCL, iWindow, iVariables, iCount, optimization, iBatch)) ReturnFalse; //--- if(!cFF[0].Init(0, 7, OpenCL, iWindow, iWindow, 4 * iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; if(!cFF[1].Init(0, 8, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::attentionOut(void) { if(!OpenCL) ReturnFalse; //--- Time Derivative { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_TimeDerivative, def_k_tdqkv, cQKV.getOutputIndex()) setBuffer(def_k_TimeDerivative, def_k_tddqkv, cdQKV.getOutputIndex()) setArgument(def_k_TimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_TimeDerivative, global_work_offset, global_work_size) } //--- MH Attention Out { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iCount, iVariables}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_FeedForwardContAtt, def_k_caqkv, cQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_FeedForwardContAtt, def_k_cascore, iScore) setBuffer(def_k_FeedForwardContAtt, def_k_caout, cAttentionOut.getOutputIndex()) setArgument(def_k_FeedForwardContAtt, def_k_cadimension, int(iDimension)) setArgument(def_k_FeedForwardContAtt, def_k_caheads, int(iHeads)) kernelExecuteLoc(def_k_FeedForwardContAtt, global_work_offset, global_work_size, local_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Generate Query, Key, Value if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //--- MH Continuas Attention if(!attentionOut()) ReturnFalse; if(!cW0.FeedForward(GetPointer(cAttentionOut))) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cW0.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } if(!SumAndNormilize(prev.getOutput(), cW0.getOutput(), prev.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } if(!SumAndNormilize(prev.getOutput(), cNODE[2].getOutput(), getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- MH Attention Out Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv, cQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaqkv_g, cQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv, cdQKV.getOutputIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcadqkv_g, cdQKV.getGradientIndex()) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcascore, iScore) setBuffer(def_k_HiddenGradientContAtt, def_k_hgcaout_g, cAttentionOut.getGradientIndex()) setArgument(def_k_HiddenGradientContAtt, def_k_hgcadimension, int(iDimension)) kernelExecute(def_k_HiddenGradientContAtt, global_work_offset, global_work_size) } //--- Time Derivative Gradient { uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iVariables, iHeads}; ResetLastError(); setBuffer(def_k_HGTimeDerivative, def_k_tdqkv, cQKV.getGradientIndex()) setBuffer(def_k_HGTimeDerivative, def_k_tddqkv, cdQKV.getGradientIndex()) setArgument(def_k_HGTimeDerivative, def_k_tddimension, int(iDimension)) kernelExecute(def_k_HGTimeDerivative, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::calcInputGradients(CNeuronBaseOCL * prevLayer) { //--- Feed Forward Gradient if(!cFF[0].CalcHiddenGradients((CObject *)GetPointer(cFF[1]))) ReturnFalse; if(!cNODE[1].CalcHiddenGradients((CObject *)GetPointer(cFF[0]))) ReturnFalse; if(!SumAndNormilize(Gradient, cNODE[2].getGradient(), cNODE[2].getGradient(), iDimension, false)) ReturnFalse; //--- Neural ODE Gradient CNeuronBaseOCL *prev = cNODE[1].AsObject(); for(int i = 2; i > 0; i--) { if(!prev.CalcHiddenGradients(cNODE[i].AsObject())) ReturnFalse; prev = cNODE[i - 1].AsObject(); } if(!cW0.CalcHiddenGradients(prev.AsObject())) ReturnFalse; if(!SumAndNormilize(cW0.getGradient(), cNODE[2].getGradient(), cW0.getGradient(), iDimension, false)) ReturnFalse; //--- MH Attention Gradient if(!cAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; //--- Query, Key, Value Graddients if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!SumAndNormilize(cW0.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iDimension, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronConformer *Source = source; //--- MH Attention if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cW0.WeightsUpdate(GetPointer(Source.cW0), tau)) ReturnFalse; //--- Feed Forward for(int i = 0; i < 2; i++) if(!cFF[i].WeightsUpdate(GetPointer(Source.cFF[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- MH Attention if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(GetPointer(cAttentionOut))) ReturnFalse; //--- Neural ODE CNeuronBaseOCL *prev = GetPointer(cW0); for(int i = 0; i < 3; i++) { if(!cNODE[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cNODE[i]); } //--- Feed Forward for(int i = 0; i < 2; i++) { if(!cFF[i].UpdateInputWeights(prev)) ReturnFalse; prev = GetPointer(cFF[i]); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) if(!cNODE[i].Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFF[i].Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, GetPointer(cQKV))) ReturnFalse; if(!LoadInsideLayer(file_handle, GetPointer(cW0))) ReturnFalse; for(int i = 0; i < 3; i++) if(!LoadInsideLayer(file_handle, GetPointer(cNODE[i]))) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, GetPointer(cFF[i]))) ReturnFalse; //--- iWindow = FileReadInteger(file_handle); iDimension = FileReadInteger(file_handle); iHeads = FileReadInteger(file_handle); iVariables = FileReadInteger(file_handle); iCount = FileReadInteger(file_handle); //--- if(!cdQKV.Init(0, 1, OpenCL, 3 * iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; if(iScore >= 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); if(iScore < 0) ReturnFalse; //--- if(!cAttentionOut.Init(0, 2, OpenCL, iDimension * iHeads * iVariables * iCount, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cFF[1].getGradient()) { DeleteObj(Gradient); Gradient = cFF[1].getGradient(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConformer::SetOpenCL(COpenCLMy * obj) { if(obj == OpenCL) return; if(iScore > 0 && !!OpenCL) { OpenCL.BufferFree(iScore); iScore = -1; } //--- CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cdQKV.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); for(int i = 0; i < 3; i++) cNODE[i].SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cFF[i].SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * iCount * iHeads * iVariables * iCount, CL_MEM_READ_WRITE); //--- if(getGradientIndex() != cFF[1].getGradientIndex()) SetGradientIndex(cFF[1].getGradientIndex()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRevINDenormOCL : public CNeuronBatchNormOCL { protected: int iBatchNormLayer; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronRevINDenormOCL(void) : iBatchNormLayer(-1) {}; ~CNeuronRevINDenormOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, int NormLayer, CNeuronBatchNormOCL *normLayer); virtual int GetNormLayer(void) { return iBatchNormLayer; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronRevInDenormOCL; }///< Identificator of class.@return Type of class //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override { return true; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRevINDenormOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, int NormLayer, CNeuronBatchNormOCL * normLayer) { if(NormLayer >= 0) { if(!normLayer) ReturnFalse; switch(normLayer.Type()) { case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: break; default: ReturnFalse; } if(BatchOptions != normLayer.BatchOptions) DeleteObj(BatchOptions); if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, numNeurons, normLayer.iBatchSize, normLayer.Optimization())) ReturnFalse; if(BatchOptions != normLayer.BatchOptions) { DeleteObj(BatchOptions); BatchOptions = normLayer.BatchOptions; } } else if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, numNeurons, 0, ADAM)) ReturnFalse; iBatchNormLayer = NormLayer; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRevINDenormOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- PrevLayer = NeuronOCL; //--- if(!BatchOptions) iBatchSize = 0; if(iBatchSize <= 1) { activation = (ENUM_ACTIVATION)NeuronOCL.Activation(); return true; } //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_RevInFeedForward, def_k_revffinputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_RevInFeedForward, def_k_revffoptions, BatchOptions.GetIndex()) setBuffer(def_k_RevInFeedForward, def_k_revffoutput, Output.GetIndex()) setArgument(def_k_RevInFeedForward, def_k_revffoptions_size, (int)BatchOptions.Total()) setArgument(def_k_RevInFeedForward, def_k_revffoptimization, (int)optimization) kernelExecute(def_k_RevInFeedForward, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRevINDenormOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- if(iBatchSize <= 1) return true; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {Neurons()}; ResetLastError(); setBuffer(def_k_RevInHiddenGraddient, def_k_revhginputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_RevInHiddenGraddient, def_k_revhginputs_gr, NeuronOCL.getGradientIndex()) setBuffer(def_k_RevInHiddenGraddient, def_k_revhgoptions, BatchOptions.GetIndex()) setBuffer(def_k_RevInHiddenGraddient, def_k_revhgoutput_gr, Gradient.GetIndex()) setArgument(def_k_RevInHiddenGraddient, def_k_revhgoptions_size, (int)BatchOptions.Total()) setArgument(def_k_RevInHiddenGraddient, def_k_revhgoptimization, (int)optimization) setArgument(def_k_RevInHiddenGraddient, def_k_revhgactivation, (int)NeuronOCL.Activation()) kernelExecute(def_k_RevInHiddenGraddient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRevINDenormOCL::Save(const int file_handle) { if(!CNeuronBatchNormOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iBatchNormLayer, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRevINDenormOCL::Load(const int file_handle) { if(!CNeuronBatchNormOCL::Load(file_handle)) ReturnFalse; iBatchNormLayer = FileReadInteger(file_handle, INT_VALUE); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNet::OpenCLInit(void) { if(!opencl) { opencl = new COpenCLMy(); if(!opencl) ReturnFalse; } if(!opencl.Initialize(cl_program, true)) { DeleteObj(opencl); ReturnFalse; } //--- create kernels opencl.SetKernelsCount(258); CreateKernel(def_k_FeedForward, "FeedForward") CreateKernel(def_k_CalcOutputGradient, "CalcOutputGradient") CreateKernel(def_k_CalcHiddenGradient, "CalcHiddenGradient") CreateKernel(def_k_UpdateWeightsMomentum, "UpdateWeightsMomentum") CreateKernel(def_k_UpdateWeightsAdam, "UpdateWeightsAdam") CreateKernel(def_k_UpdateWeightsLS, "UpdateWeightsLS") CreateKernel(def_k_AttentionGradients, "AttentionInsideGradients") CreateKernel(def_k_AttentionOut, "AttentionOut") CreateKernel(def_k_AttentionScore, "AttentionScore") CreateKernel(def_k_CalcHiddenGradientConv, "CalcHiddenGradientConv") CreateKernel(def_k_CalcInputGradientProof, "CalcInputGradientProof") CreateKernel(def_k_FeedForwardConv, "FeedForwardConv") CreateKernel(def_k_FeedForwardProof, "FeedForwardProof") CreateKernel(def_k_MatrixSum, "SumMatrix") CreateKernel(def_k_Matrix5Sum, "Sum5Matrix") CreateKernel(def_k_UpdateWeightsConvAdam, "UpdateWeightsConvAdam") CreateKernel(def_k_UpdateWeightsConvMomentum, "UpdateWeightsConvMomentum") CreateKernel(def_k_UpdateWeightsConvLS, "UpdateWeightsConvLS") CreateKernel(def_k_Normilize, "Normalize") CreateKernel(def_k_NormilizeWeights, "NormalizeWeights") CreateKernel(def_k_ConcatenateMatrix, "ConcatenateBuffers") CreateKernel(def_k_DeconcatenateMatrix, "DeconcatenateBuffers") CreateKernel(def_k_MHAttentionGradients, "MHAttentionInsideGradients") CreateKernel(def_k_MHAttentionScore, "MHAttentionScore") CreateKernel(def_k_MHAttentionOut, "MHAttentionOut") CreateKernel(def_k_Dropout, "Dropout") CreateKernel(def_k_BatchFeedForward, "BatchFeedForward") CreateKernel(def_k_CalcHiddenGradientBatch, "CalcHiddenGradientBatch") CreateKernel(def_k_UpdateBatchOptionsMomentum, "UpdateBatchOptionsMomentum") CreateKernel(def_k_UpdateBatchOptionsAdam, "UpdateBatchOptionsAdam") CreateKernel(def_k_VAEFeedForward, "VAE_FeedForward") CreateKernel(def_k_VAECalcHiddenGradient, "VAE_CalcHiddenGradient") CreateKernel(def_k_LSTM_ConcatenatedGradient, "LSTM_ConcatenatedGradient") CreateKernel(def_k_LSTM_HiddenGradient, "LSTM_HiddenGradient") CreateKernel(def_k_LSTM_FeedForward, "LSTM_FeedForward") CreateKernel(def_k_LSTM_UpdateWeightsAdam, "LSTM_UpdateWeightsAdam") CreateKernel(def_k_SoftMax_FeedForward, "SoftMax_FeedForward") CreateKernel(def_k_SoftMax_HiddenGradient, "SoftMax_HiddenGradient") CreateKernel(def_k_SoftMax_OutputGradient, "SoftMax_OutputGradient") CreateKernel(def_k_FQF_Cosine, "FQF_Cosine") CreateKernel(def_k_FQF_Output, "FQF_Output") CreateKernel(def_k_FQF_OutputGradient, "FQF_OutputGradient") CreateKernel(def_k_FQF_QuantileGradient, "FQF_QuantileGradient") CreateKernel(def_k_FQF_CosineGradient, "FQF_CosineGradient") CreateKernel(def_k_MHSparseAttentionScore, "MHSparseAttentionScore") CreateKernel(def_k_MHSparseAttentionOut, "MHSparseAttentionOut") CreateKernel(def_k_FFMultiModels, "FeedForwardMultiModels") CreateKernel(def_k_HGMultiModels, "CalcHiddenGradientMultiModels") CreateKernel(def_k_UWMultiModels, "UpdateWeightsAdamMultiModels") CreateKernel(def_k_ConcatFeedForward, "Concat_FeedForward") CreateKernel(def_k_ConcatCalcHiddenGradient, "Concat_HiddenGradient") CreateKernel(def_k_ConcatUpdWeightsMomentum, "Concat_UpdateWeightsMomentum") CreateKernel(def_k_ConcatUpdWeightsAdam, "Concat_UpdateWeightsAdam") CreateKernel(def_k_SoftUpdate, "SoftUpdate") CreateKernel(def_k_SoftUpdateAdam, "SoftUpdateAdam") CreateKernel(def_k_SAC_AlphaLogProbs, "SAC_AlphaLogProbs") CreateKernel(def_k_SAC_AlphaGradients, "SAC_AlphaGradients") CreateKernel(def_k_SAC_OutputGradient, "SAC_OutputGradient") CreateKernel(def_k_SAC_CalcLogProbs, "SAC_CalcLogProbs") CreateKernel(def_k_Embedding, "Embedding") CreateKernel(def_k_EmbeddingHiddenGradient, "EmbeddingHiddenGradient") CreateKernel(def_k_EmbeddingUpdateWeightsAdam, "EmbeddingUpdateWeightsAdam") CreateKernel(def_k_Transpose, "Transpose") CreateKernel(def_k_TransposeRCD, "TransposeRCD") CreateKernel(def_k_MH2AttentionOut, "MH2AttentionOut") CreateKernel(def_k_MH2AttentionInsideGradients, "MH2AttentionInsideGradients") CreateKernel(def_k_CGConv_HiddenGradient, "CGConv_HiddenGradient") CreateKernel(def_k_XCiTFeedForward, "XCiTFeedForward") CreateKernel(def_k_XCiTInsideGradients, "XCiTInsideGradients") CreateKernel(def_k_DOTFeedForward, "DOTFeedForward") CreateKernel(def_k_DOTInsideGradients, "DOTInsideGradients") CreateKernel(def_k_RPBUpdateAdam, "RPBUpdateAdam") CreateKernel(def_k_GTEFeedForward, "GTEFeedForward") CreateKernel(def_k_GTEInsideGradients, "GTEInsideGradients") CreateKernel(def_k_FeedForwardNODEF, "FeedForwardNODEF") CreateKernel(def_k_FeedForwardNODEInpK, "FeedForwardNODEInpK") CreateKernel(def_k_HiddenGradientNODEF, "HiddenGradientNODEF") CreateKernel(def_k_HiddenGradientNODEInpK, "HiddenGradientNODEInpK") CreateKernel(def_k_NODEF_UpdateWeightsAdam, "NODEF_UpdateWeightsAdam") CreateKernel(def_k_FeedForwardContAtt, "FeedForwardContAtt") CreateKernel(def_k_TimeDerivative, "TimeDerivative") CreateKernel(def_k_HGTimeDerivative, "HiddenGradientTimeDerivative") CreateKernel(def_k_HiddenGradientContAtt, "HiddenGradientContAtt") CreateKernel(def_k_RevInFeedForward, "RevInFeedForward") CreateKernel(def_k_RevInHiddenGraddient, "RevInHiddenGraddient") CreateKernel(def_k_DeActivation, "DeActivation") CreateKernel(def_k_PatchCreate, "PatchCreate") CreateKernel(def_k_PatchHiddenGradient, "PatchHiddenGradient") CreateKernel(def_k_PatchUpdateWeightsAdam, "PatchUpdateWeightsAdam") CreateKernel(def_k_MatMult, "MatMult") CreateKernel(def_k_FFT, "FFT") CreateKernel(def_k_ComplexLayer, "ComplexLayer") CreateKernel(def_k_ComplexLayerGradient, "ComplexLayerGradient") CreateKernel(def_k_GradientMSA, "GradientMSA") CreateKernel(def_k_CumulativeGradient, "CumulativeGradient") CreateKernel(def_k_FeedForwardComplexConv, "FeedForwardComplexConv") CreateKernel(def_k_CalcHiddenGradientComplexConv, "CalcHiddenGradientComplexConv") CreateKernel(def_k_UpdateWeightsComplexConvMomentum, "UpdateWeightsComplexConvMomentum") CreateKernel(def_k_UpdateWeightsComplexConvAdam, "UpdateWeightsComplexConvAdam") CreateKernel(def_k_ComplexMHAttentionGradients, "ComplexMHAttentionGradients") CreateKernel(def_k_ComplexMHAttentionScore, "ComplexMHAttentionScore") CreateKernel(def_k_ComplexMHAttentionOut, "ComplexMHAttentionOut") CreateKernel(def_k_ComplexSoftMax_FeedForward, "ComplexSoftMax_FeedForward") CreateKernel(def_k_ComplexSoftMax_HiddenGradient, "ComplexSoftMax_HiddenGradient") CreateKernel(def_k_ComplexSoftMax_OutputGradient, "ComplexSoftMax_OutputGradient") CreateKernel(def_k_ComplexNormalize, "ComplexNormalize") CreateKernel(def_k_ComplexUnNormalize, "ComplexUnNormalize") CreateKernel(def_k_ComplexNormalizeGradient, "ComplexNormalizeGradient") CreateKernel(def_k_ComplexUnNormalizeGradient, "ComplexUnNormalizeGradient") CreateKernel(def_k_MainFreqWeight, "MainFreqWeight") CreateKernel(def_k_WeightedSum, "WeightedSum") CreateKernel(def_k_WeightedSumGradient, "WeightedSumGradient") CreateKernel(def_k_FeedForwardS3, "FeedForwardS3") CreateKernel(def_k_InsideGradientS3, "InsideGradientS3") CreateKernel(def_k_WeightGradientS3, "WeightGradientS3") CreateKernel(def_k_MH2PyrAttentionOut, "MH2PyrAttentionOut") CreateKernel(def_k_PLR, "PLR") CreateKernel(def_k_PLRGrad, "PLRGradient") CreateKernel(def_k_PLRMultiAgents, "PLRMultiAgents") CreateKernel(def_k_PLRMultiAgentsGrad, "PLRMultiAgentsGradient") CreateKernel(def_k_UpdateWeightsAdamMini, "UpdateWeightsAdamMini") CreateKernel(def_k_UpdateWeightsConvAdamMini, "UpdateWeightsConvAdamMini") CreateKernel(def_k_CutTrendAndOther, "CutTrendAndOther") CreateKernel(def_k_CutTrendAndOtherGradient, "CutTrendAndOtherGradient") CreateKernel(def_k_CutOneFromAnother, "CutOneFromAnother") CreateKernel(def_k_CutOneFromAnotherGradient, "CutOneFromAnotherGradient") CreateKernel(def_k_UniTrajPrepare, "UniTrajPrepare") CreateKernel(def_k_UniTrajPrepareGrad, "UniTrajPrepareGrad") CreateKernel(def_k_UniTrajBTS, "UniTrajBTS") CreateKernel(def_k_HiVTPrepare, "HiVTPrepare") CreateKernel(def_k_GateElementMul, "GateElementMul") CreateKernel(def_k_GateElementMulGrad, "GateElementMulGrad") CreateKernel(def_k_MatMultGrad, "MatMultGrad") CreateKernel(def_k_OrthoganalLoss, "OrthoganalLoss") CreateKernel(def_k_CalcDistance, "CalcDistance") CreateKernel(def_k_FeedForwardLocalMax, "FeedForwardLocalMax") CreateKernel(def_k_CalcInputGradientLocalMax, "CalcInputGradientLocalMax") CreateKernel(def_k_MHMaskAttentionOut, "MHMaskAttentionOut") CreateKernel(def_k_MHMaskAttentionInsideGradients, "MHMaskAttentionInsideGradients") CreateKernel(def_k_MHPosBiasAttentionOut, "MHPosBiasAttentionOut") CreateKernel(def_k_MHPosBiasAttentionInsideGradients, "MHPosBiasAttentionInsideGradients") CreateKernel(def_k_CalcPositionBias, "CalcPositionBias") CreateKernel(def_k_DiversityLoss, "DiversityLoss") CreateKernel(def_k_MHRelativeAttentionOut, "MHRelativeAttentionOut") CreateKernel(def_k_MHRelativeAttentionInsideGradients, "MHRelativeAttentionInsideGradients") CreateKernel(def_k_CalcAlignmentGradient, "CalcAlignmentGradient") CreateKernel(def_k_FeatureSmoothing, "FeatureSmoothing") CreateKernel(def_k_FeatureSmoothingGradient, "FeatureSmoothingGradient") CreateKernel(def_k_BatchFeedForwardAddNoise, "BatchFeedForwardAddNoise") CreateKernel(def_k_HyperProjection, "HyperProjection") CreateKernel(def_k_HyperProjectionGrad, "HyperProjectionGrad") CreateKernel(def_k_LogMap, "LogMap") CreateKernel(def_k_LogMapGrad, "LogMapGrad") CreateKernel(def_k_CalcEpsilonWeights, "CalcEpsilonWeights") CreateKernel(def_k_CalcEpsilonWeightsConv, "CalcEpsilonWeightsConv") CreateKernel(def_k_CalcHiddenGradientMHConv, "CalcHiddenGradientMHConv") CreateKernel(def_k_FeedForwardMHConv, "FeedForwardMHConv") CreateKernel(def_k_UpdateWeightsMHConvAdam, "UpdateWeightsMHConvAdam") CreateKernel(def_k_MoreLessEqual, "MoreLessEqual") CreateKernel(def_k_MultiScaleRelativeAttentionOut, "MultiScaleRelativeAttentionOut") CreateKernel(def_k_SSM2D_FeedForward, "SSM2D_FeedForward") CreateKernel(def_k_SSM2D_CalcHiddenGradient, "SSM2D_CalcHiddenGradient") CreateKernel(def_k_PScan, "PScan") CreateKernel(def_k_PScan_CalcHiddenGradient, "PScan_CalcHiddenGradient") CreateKernel(def_k_DiagMatMult, "DiagMatMult") CreateKernel(def_k_DiagMatMultGrad, "DiagMatMultGrad") CreateKernel(def_k_TopKgates, "TopKgates") CreateKernel(def_k_TopKgatesGrad, "TopKgatesGrad") CreateKernel(def_k_MaskByDistance, "MaskByDistance") CreateKernel(def_k_MaskAttention, "MaskAttention") CreateKernel(def_k_MaskAttentionGradients, "MaskAttentionGradients") CreateKernel(def_k_FeedForwardMultWinConv, "FeedForwardMultWinConv") CreateKernel(def_k_CalcHiddenGradientMultWinConv, "CalcHiddenGradientMultWinConv") CreateKernel(def_k_UpdateWeightsMultWinConvAdam, "UpdateWeightsMultWinConvAdam") CreateKernel(def_k_MaskAttentionComplex, "MaskAttentionComplex") CreateKernel(def_k_MaskAttentionGradientsComplex, "MaskAttentionGradientsComplex") CreateKernel(def_k_CSLSTM_FeedForward, "CSLSTM_FeedForward") CreateKernel(def_k_CSLSTM_CalcHiddenGradient, "CSLSTM_CalcHiddenGradient") CreateKernel(def_k_ProbAttentionQeuryImp, "ProbAttentionQeuryImp") CreateKernel(def_k_TopKImportanceToIndex, "TopKImportanceToIndex") CreateKernel(def_k_QIndexAttention, "QIndexAttention") CreateKernel(def_k_QIndexAttentionGradients, "QIndexAttentionGradients") CreateKernel(def_k_TSPositonEncoder, "TSPositonEncoder") CreateKernel(def_k_FeedForwardMultWinConvWPad, "FeedForwardMultWinConvWPad") CreateKernel(def_k_CalcHiddenGradientMultWinConvWPad, "CalcHiddenGradientMultWinConvWPad") CreateKernel(def_k_UpdateWeightsMultWinConvAdamWPad, "UpdateWeightsMultWinConvAdamWPad") CreateKernel(def_k_ConcatDiff, "ConcatDiff") CreateKernel(def_k_FeedForwardMaskMultWinConv, "FeedForwardMaskMultWinConv") CreateKernel(def_k_CalcHiddenGradientMaskMultWinConv, "CalcHiddenGradientMaskMultWinConv") CreateKernel(def_k_UpdateWeightsMaskMultWinConvAdam, "UpdateWeightsMaskMultWinConvAdam") CreateKernel(def_k_MainFreq, "MainFreq") CreateKernel(def_k_FeedForwardAdaptConv, "FeedForwardAdaptConv") CreateKernel(def_k_CalcHiddenGradientAdaptConv, "CalcHiddenGradientAdaptConv") CreateKernel(def_k_UpdateWeightsAdaptConvAdam, "UpdateWeightsAdaptConvAdam") CreateKernel(def_k_RoPE, "RoPE") CreateKernel(def_k_CalcHiddenGradRoPE, "CalcHiddenGradRoPE") CreateKernel(def_k_MatrixDif, "DifMatrix") CreateKernel(def_k_MatrixDifGrad, "DifMatrixGrad") CreateKernel(def_k_IdentMatrixSum, "IdentitySumMatrix") CreateKernel(def_k_IdentMatrixDif, "IdentityDifMatrix") CreateKernel(def_k_IdentMatrixDifGrad, "IdentityDifMatrixGrad") CreateKernel(def_k_SumVecMatrix, "SumVecMatrix") CreateKernel(def_k_SumVecMatrixGrad, "SumVecMatrixGrad") CreateKernel(def_k_InterpolationAttention, "InterpolationAttention") CreateKernel(def_k_InterpolationAttentionGrad, "InterpolationAttentionGrad") CreateKernel(def_k_Activation, "Activation") CreateKernel(def_k_PeriodNorm, "PeriodNorm") CreateKernel(def_k_PeriodNormGrad, "PeriodNormGrad") CreateKernel(def_k_AdaptSpatialNorm, "AdaptSpatialNorm") CreateKernel(def_k_AdaptSpatialNormGrad, "AdaptSpatialNormGrad") CreateKernel(def_k_AttentNorm, "AttentNorm") CreateKernel(def_k_AttentNormGrad, "AttentNormGrad") CreateKernel(def_k_ChebStep, "ChebStep") CreateKernel(def_k_ChebStepGrad, "ChebStepGrad") CreateKernel(def_k_SignificantNeighborsSampling, "SignificantNeighborsSampling") CreateKernel(def_k_SparseMHScores, "SparseMHScores") CreateKernel(def_k_SparseMHScoresGrad, "SparseMHScoresGrad") CreateKernel(def_k_SparseMatMult, "SparseMatMult") CreateKernel(def_k_SparseMatMultGrad, "SparseMatMultGrad") CreateKernel(def_k_RandomWalk, "RandomWalk") CreateKernel(def_k_ConcatByLabel, "ConcatByLabel") CreateKernel(def_k_ConcatByLabelGrad, "ConcatByLabelGrad") CreateKernel(def_k_GlobalLocalAttention, "GlobalLocalAttention") CreateKernel(def_k_GlobalLocalAttentionGrad, "GlobalLocalAttentionGrad") CreateKernel(def_k_SparseSoftMax, "SparseSoftMax") CreateKernel(def_k_SparseSoftMaxGrad, "SparseSoftMaxGrad") CreateKernel(def_k_FloatToSpike, "FloatToSpike") CreateKernel(def_k_FloatToSpikeGrad, "FloatToSpikeGrad") CreateKernel(def_k_SpikeMHAttention, "SpikeMHAttention") CreateKernel(def_k_SpikeMHAttentionGrad, "SpikeMHAttentionGrad") CreateKernel(def_k_STFS, "STFS") CreateKernel(def_k_STFSGrad, "STFSGrad") CreateKernel(def_k_AddToStack, "AddToStack") CreateKernel(def_k_AggregationByTime, "AggregationByTime") CreateKernel(def_k_AggregationByTimeGrad, "AggregationByTimeGrad") CreateKernel(def_k_GRU, "GRU") CreateKernel(def_k_GRU_Grad, "GRU_Grad") CreateKernel(def_k_ScalarToVector, "ScalarToVector") CreateKernel(def_k_ScalarToVectorGrad, "ScalarToVectorGrad") CreateKernel(def_k_CalcFlow, "CalcFlow") CreateKernel(def_k_DilatedCorrelation, "DilatedCorrelation") CreateKernel(def_k_DilatedCorrelationGrad, "DilatedCorrelationGrad") CreateKernel(def_k_DilatedDifference, "DilatedDifference") CreateKernel(def_k_DilatedDifferenceGrad, "DilatedDifferenceGrad") CreateKernel(def_k_PerturbedMatrix, "PerturbedMatrix") CreateKernel(def_k_PerturbedMatrixGrad, "PerturbedMatrixGrad") CreateKernel(def_k_LinearUpsample, "LinearUpsample") CreateKernel(def_k_LinearUpsampleGrad, "LinearUpsampleGrad") CreateKernel(def_k_MixExpertsPredict, "MixExpertsPredict") CreateKernel(def_k_MixExpertsPredictGrad, "MixExpertsPredictGrad") CreateKernel(def_k_MHFAT, "MHFAT") CreateKernel(def_k_MHFATGrad, "MHFATGrad") CreateKernel(def_k_SparseConcatenate, "SparseConcatenate") CreateKernel(def_k_SparseConcatenateGrad, "SparseConcatenateGrad") CreateKernel(def_k_MHFlashAttention, "MHFlashAttention") CreateKernel(def_k_MHFlashAttentionGrad, "MHFlashAttentionGrad") CreateKernel(def_k_MHFlashSTCA, "MHFlashSTCA") CreateKernel(def_k_MHFlashSTCAGrad, "MHFlashSTCAGrad") //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronClientOCL : public CNeuronBaseOCL { protected: //--- Attention CNeuronMLMHAttentionOCL cTransformerEncoder; CNeuronConvOCL cProjection; //--- Linear model CNeuronConvOCL cLinearModel[]; //--- CNeuronBaseOCL cInput; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronClientOCL(void) {}; ~CNeuronClientOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint at_layers, uint count, uint &mlp[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronClientOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint at_layers, uint count, uint & mlp[], ENUM_OPTIMIZATION optimization_type, uint batch) { uint mlp_layers = mlp.Size(); if(mlp_layers == 0) ReturnFalse; if(ArrayResize(cLinearModel, mlp_layers + 1) != (mlp_layers + 1)) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, mlp[mlp_layers - 1] * count, optimization_type, batch)) ReturnFalse; if(!cTransformerEncoder.Init(0, 0, OpenCL, window, window_key, heads, count, at_layers, optimization, iBatch)) ReturnFalse; if(!cInput.Init(0, 1, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- uint w = window; for(uint i = 0; i < mlp_layers; i++) { if(!cLinearModel[i].Init(0, i + 2, OpenCL, w, w, mlp[i], count, optimization, iBatch)) ReturnFalse; cLinearModel[i].SetActivationFunction(LReLU); w = mlp[i]; } if(!cLinearModel[mlp_layers].Init(0, mlp_layers + 2, OpenCL, 1, 1, 1, w * count, optimization, iBatch)) ReturnFalse; cLinearModel[mlp_layers].SetActivationFunction(TANH); if(!cProjection.Init(0, mlp_layers + 3, OpenCL, window, window, w, count, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); SetActivationFunction(TANH); //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[mlp_layers].SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTransformerEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjection.FeedForward(GetPointer(cTransformerEncoder))) ReturnFalse; if(cInput.getOutputIndex() != NeuronOCL.getOutputIndex()) cInput.getOutput().BufferSet(NeuronOCL.getOutputIndex()); //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].FeedForward(neuron)) ReturnFalse; neuron = GetPointer(cLinearModel[i]); } //--- if(!SumAndNormilize(neuron.getOutput(), cProjection.getOutput(), Output, 1, false, 0, 0, 0, 0.5)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!cTransformerEncoder.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cTransformerEncoder.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *neuron = NULL; int total = (int)cLinearModel.Size() - 1; for(int i = total; i >= 0; i--) { neuron = (i > 0 ? cLinearModel[i - 1] : cInput).AsObject(); if(!neuron.CalcHiddenGradients(cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SumAndNormilize(neuron.getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cTransformerEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cProjection.UpdateInputWeights(cTransformerEncoder.AsObject())) ReturnFalse; //--- uint total = cLinearModel.Size(); CNeuronBaseOCL *neuron = NeuronOCL; for(uint i = 0; i < total; i++) { if(!cLinearModel[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = cLinearModel[i].AsObject(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronClientOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cTransformerEncoder.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cInput.SetOpenCL(OpenCL); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cTransformerEncoder.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cInput.Save(file_handle)) ReturnFalse; uint total = cLinearModel.Size(); if(FileWriteInteger(file_handle, int(total), INT_VALUE) != INT_VALUE) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::Load(const int file_handle) { ArrayFree(cLinearModel); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransformerEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInput.AsObject())) ReturnFalse; int total = FileReadInteger(file_handle); if(total <= 0) ReturnFalse; if(ArrayResize(cLinearModel, total) < total) ReturnFalse; for(int i = 0; i < total; i++) { cLinearModel[i].SetOpenCL(OpenCL); if(!LoadInsideLayer(file_handle, cLinearModel[i].AsObject())) ReturnFalse; } //--- if(!SetGradient(cProjection.getGradient())) ReturnFalse; if(!cLinearModel[total - 1].SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronClientOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cTransformerEncoder.TrainMode(bTrain); cProjection.TrainMode(bTrain); //--- uint total = cLinearModel.Size(); for(uint i = 0; i < total; i++) cLinearModel[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronClientOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronClientOCL *sour = source; if(!cTransformerEncoder.WeightsUpdate(sour.cTransformerEncoder.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(sour.cProjection.AsObject(), tau)) ReturnFalse; //--- uint total = cLinearModel.Size(); if(total != sour.cLinearModel.Size()) ReturnFalse; for(uint i = 0; i < total; i++) if(!cLinearModel[i].WeightsUpdate(sour.cLinearModel[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLearnabledPE : public CNeuronBaseOCL { protected: CNeuronBaseOCL cInputs; CNeuronBaseOCL cPositionEncoder; ///\ingroup neuron_base_ff virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); ///< \brief Feed Forward method of calling kernel ::FeedForward().@param NeuronOCL Pointer to previos layer. ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); ///< Method to transfer gradient to previous layer by calling kernel ::CalcHiddenGradient(). @param NeuronOCL Pointer to next layer. public: CNeuronLearnabledPE(void) {}; ~CNeuronLearnabledPE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle);///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle);///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual int Type(void) const { return defNeuronLearnabledPE; }///< Identificator of class.@return Type of class virtual void SetOpenCL(COpenCLMy *obj); //--- virtual CBufferFloat* GetPE(void) const { return cPositionEncoder.getOutput(); } virtual bool AddPEGradient(CBufferFloat *grad) { if(!grad || !SumAndNormilize(cPositionEncoder.getGradient(), grad, cPositionEncoder.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; return true; } virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; if(!cInputs.Init(numNeurons, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *inp = cInputs.getOutput(); inp.BufferInit(1, 1); inp.BufferWrite(); cInputs.SetActivationFunction(None); //--- if(!cPositionEncoder.Init(0, 1, OpenCL, numNeurons, optimization, iBatch)) ReturnFalse; cPositionEncoder.SetActivationFunction(SIGMOID); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::Activation(CBufferFloat *inputs, CBufferFloat *outputs, int activat) { if(!OpenCL || !inputs || !outputs) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {inputs.Total()}; ResetLastError(); setBuffer(def_k_Activation, def_k_act_inputs, inputs.GetIndex()) setBuffer(def_k_Activation, def_k_act_outputs, outputs.GetIndex()) setArgument(def_k_Activation, def_k_act_activation, (int)activat) //--- kernelExecute(def_k_Activation, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DeActivation(CBufferFloat * inputs, CBufferFloat * inputs_gr, CBufferFloat * output_gr, int activat) { if(!OpenCL || !inputs || !inputs_gr || !output_gr) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {inputs.Total()}; ResetLastError(); setBuffer(def_k_DeActivation, def_k_deact_inputs, inputs.GetIndex()) setBuffer(def_k_DeActivation, def_k_deact_inputs_gr, inputs_gr.GetIndex()) setBuffer(def_k_DeActivation, def_k_deact_output_gr, output_gr.GetIndex()) setArgument(def_k_DeActivation, def_k_deact_activation, (int)activat) //--- kernelExecute(def_k_DeActivation, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cPositionEncoder.FeedForward(cInputs.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cPositionEncoder.getOutput(), Output, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeActivation(cPositionEncoder.getOutput(), cPositionEncoder.getGradient(), Gradient, cPositionEncoder.Activation())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { return cPositionEncoder.UpdateInputWeights(cInputs.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronLearnabledPE *Source = source; return cPositionEncoder.WeightsUpdate(GetPointer(Source.cPositionEncoder), tau); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLearnabledPE::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cPositionEncoder.SetOpenCL(OpenCL); cInputs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cInputs.Save(file_handle)) ReturnFalse; if(!cPositionEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLearnabledPE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cInputs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPositionEncoder.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronUShapeAttention : public CNeuronBaseOCL { protected: CNeuronMLMHAttentionOCL cAttention[2]; CNeuronConvOCL cMergeSplit[2]; CNeuronBaseOCL *cNeck; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); //--- public: CNeuronUShapeAttention(void) { activation = None; } ~CNeuronUShapeAttention(void) { DeleteObj(cNeck); } //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch); ///< Method of initialization class.@param[in] numOutputs Number of connections to next layer.@param[in] myIndex Index of neuron in layer.@param[in] open_cl Pointer to #COpenCLMy object.@param[in] window Size of in/out window and step.@param[in] units_countNumber of neurons.@param[in] optimization_type Optimization type (#ENUM_OPTIMIZATION)@return Boolen result of operations. //--- virtual int Type(void) const { return defNeuronUShapeAttention; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, 4 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronUShapeAttention *temp = new CNeuronUShapeAttention(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, 2 * units_count, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window, window, 2 * units_count, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, 2 * units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, 2 * window, 2 * window, window, units_count, optimization, iBatch)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), Output, 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), Gradient, prevLayer.getGradient(), 1, false)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronUShapeAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) { cAttention[i].SetOpenCL(OpenCL); cMergeSplit[i].SetOpenCL(OpenCL); } if(!!cNeck) cNeck.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!cAttention[i].Save(file_handle)) ReturnFalse; if(!cMergeSplit[i].Save(file_handle)) ReturnFalse; } if(!cNeck.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronUShapeAttention: cNeck = new CNeuronUShapeAttention(); if(!cNeck) ReturnFalse; break; case defNeuronConvOCL: cNeck = new CNeuronConvOCL(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(Gradient != cMergeSplit[1].getGradient()) SetGradient(cMergeSplit[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUShapeAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) return true; //--- CNeuronUShapeAttention *temp = source; for(int i = 0; i < 2; i++) { if(!cAttention[i].WeightsUpdate(temp.cAttention[i].AsObject(), tau)) ReturnFalse; if(!cMergeSplit[i].WeightsUpdate(temp.cMergeSplit[i].AsObject(), tau)) ReturnFalse; } if(!cNeck.WeightsUpdate(temp.cNeck.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPatching : public CNeuronBaseOCL { protected: uint iWindowIn; uint iStep; uint iWindowOut; uint iVariables; uint iCount; //--- CBufferFloat cPatchWeights; CBufferFloat cPatchFirstMomentum; CBufferFloat cPatchSecondMomentum; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronPatching(void) {}; ~CNeuronPatching(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronPatchingOCL; } virtual void SetOpenCL(COpenCLMy *obj); //--- }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint step, uint window_out, uint count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count * variables, optimization_type, batch)) ReturnFalse; //--- iWindowIn = MathMax(window_in, 1); iWindowOut = MathMax(window_out, 1); iStep = MathMax(step, 1); iVariables = MathMax(variables, 1); iCount = MathMax(count, 1); //--- int total = int((window_in + 1) * window_out * variables); if(!cPatchWeights.Reserve(total)) ReturnFalse; float k = float(1 / sqrt(total)); for(int i = 0; i < total; i++) { if(!cPatchWeights.Add((2 * GenerateWeight()*k - k)*WeightsMultiplier)) ReturnFalse; } if(!cPatchWeights.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cPatchFirstMomentum.BufferInit(total, 0) || !cPatchFirstMomentum.BufferCreate(OpenCL)) ReturnFalse; if(!cPatchSecondMomentum.BufferInit(total, 0) || !cPatchSecondMomentum.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iWindowOut, iVariables}; ResetLastError(); setBuffer(def_k_PatchCreate, def_k_ptc_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_PatchCreate, def_k_ptc_weights, cPatchWeights.GetIndex()) setBuffer(def_k_PatchCreate, def_k_ptc_outputs, Output.GetIndex()) setArgument(def_k_PatchCreate, def_k_ptc_activation, (int)activation) setArgument(def_k_PatchCreate, def_k_ptc_inputs_total, (int)NeuronOCL.Neurons()) setArgument(def_k_PatchCreate, def_k_ptc_window_in, (int)iWindowIn) setArgument(def_k_PatchCreate, def_k_ptc_step, (int)iStep) //--- kernelExecute(def_k_PatchCreate, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {NeuronOCL.Neurons() / iVariables, iVariables}; ResetLastError(); setBuffer(def_k_PatchHiddenGradient, def_k_pthg_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_PatchHiddenGradient, def_k_pthg_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(def_k_PatchHiddenGradient, def_k_pthg_weights, cPatchWeights.GetIndex()) setBuffer(def_k_PatchHiddenGradient, def_k_pthg_outputs_gr, Gradient.GetIndex()) setArgument(def_k_PatchHiddenGradient, def_k_pthg_activation, (int)NeuronOCL.Activation()) setArgument(def_k_PatchHiddenGradient, def_k_pthg_outputs_total, (int)iCount) setArgument(def_k_PatchHiddenGradient, def_k_pthg_window_in, (int)iWindowIn) setArgument(def_k_PatchHiddenGradient, def_k_pthg_step, (int)iStep) setArgument(def_k_PatchHiddenGradient, def_k_pthg_window_out, (int)iWindowOut) //--- kernelExecute(def_k_PatchHiddenGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindowIn + 1, iWindowOut, iVariables}; ResetLastError(); setBuffer(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_outputs_gr, getGradientIndex()) setBuffer(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_weights, cPatchWeights.GetIndex()) setBuffer(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_weights_m, cPatchFirstMomentum.GetIndex()) setBuffer(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_weights_v, cPatchSecondMomentum.GetIndex()) setArgument(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_inputs_total, (int)NeuronOCL.Neurons()) setArgument(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_l, lr) setArgument(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_b1, b1) setArgument(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_step, (int)iStep) setArgument(def_k_PatchUpdateWeightsAdam, def_k_ptuwa_b2, b2) //--- kernelExecute(def_k_PatchUpdateWeightsAdam, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindowIn) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowOut) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iStep) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cPatchWeights.Save(file_handle)) ReturnFalse; if(!cPatchFirstMomentum.Save(file_handle)) ReturnFalse; if(!cPatchSecondMomentum.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPatching::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindowIn = (uint)FileReadInteger(file_handle); iWindowOut = (uint)FileReadInteger(file_handle); iStep = (uint)FileReadInteger(file_handle); iCount = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- if(!cPatchWeights.Load(file_handle)) ReturnFalse; if(!cPatchFirstMomentum.Load(file_handle)) ReturnFalse; if(!cPatchSecondMomentum.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPatching::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cPatchWeights.BufferCreate(OpenCL); cPatchFirstMomentum.BufferCreate(OpenCL); cPatchSecondMomentum.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTiDEOCL : public CNeuronBaseOCL { protected: uint iHistory; uint iForecast; uint iVariables; uint iFeatures; //--- CResidualConv acEncoderDecoder[]; CNeuronConvOCL cGlobalResidual; CResidualConv acFeatureProjection[2]; CResidualConv cTemporalDecoder; //--- CNeuronBaseOCL cHistoryInput; CNeuronBaseOCL cFeatureInput; CNeuronBaseOCL cEncoderInput; CNeuronBaseOCL cTemporalDecoderInput; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None); public: CNeuronTiDEOCL(void) {}; ~CNeuronTiDEOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint history, uint forecast, uint variables, uint features, uint &encoder_decoder[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronTiDEOCL; } virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint history, uint forecast, uint variables, uint features, uint & encoder_decoder[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- iHistory = MathMax(history, 1); iForecast = forecast; iVariables = variables; iFeatures = MathMax(features, 1); //--- if(!acFeatureProjection[0].Init(0, 0, OpenCL, iFeatures, iHistory * iVariables, 1, optimization, iBatch)) ReturnFalse; if(!acFeatureProjection[1].Init(0, 1, OpenCL, iFeatures, iForecast * iVariables, 1, optimization, iBatch)) ReturnFalse; //--- int total = ArraySize(encoder_decoder); if(ArrayResize(acEncoderDecoder, total + 1) < total + 1) ReturnFalse; if(total == 0) { if(!acEncoderDecoder[0].Init(0, 2, OpenCL, 2 * iHistory, iForecast, iVariables, optimization, iBatch)) ReturnFalse; } else { if(!acEncoderDecoder[0].Init(0, 2, OpenCL, 2 * iHistory, encoder_decoder[0], iVariables, optimization, iBatch)) ReturnFalse; for(int i = 1; i < total; i++) if(!acEncoderDecoder[i].Init(0, i + 2, OpenCL, encoder_decoder[i - 1], encoder_decoder[i], iVariables, optimization, iBatch)) ReturnFalse; if(!acEncoderDecoder[total].Init(0, total + 2, OpenCL, encoder_decoder[total - 1], iForecast, iVariables, optimization, iBatch)) ReturnFalse; } //--- if(!cGlobalResidual.Init(0, total + 3, OpenCL, iHistory, iHistory, iForecast, iVariables, optimization, iBatch)) ReturnFalse; cGlobalResidual.SetActivationFunction(TANH); if(!cTemporalDecoder.Init(0, total + 4, OpenCL, 2 * iForecast, iForecast, iVariables, optimization, iBatch)) ReturnFalse; //--- if(!cHistoryInput.Init(0, total + 5, OpenCL, iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cFeatureInput.Init(0, total + 6, OpenCL, iFeatures, optimization, iBatch)) ReturnFalse; if(!cEncoderInput.Init(0, total + 7, OpenCL, 2 * iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cTemporalDecoderInput.Init(0, total + 8, OpenCL, 2 * iForecast * iVariables, optimization, iBatch)) ReturnFalse; //--- if(cGlobalResidual.getGradient() != Gradient) if(!cGlobalResidual.SetGradient(Gradient)) ReturnFalse; if(cTemporalDecoder.getGradient() != getGradient()) if(!cTemporalDecoder.SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(cHistoryInput.getOutputIndex() != NeuronOCL.getOutputIndex()) { CBufferFloat *temp = cHistoryInput.getOutput(); if(!temp.BufferSet(NeuronOCL.getOutputIndex())) ReturnFalse; } if(cFeatureInput.getOutputIndex() != SecondInput.GetIndex()) { CBufferFloat *temp = cFeatureInput.getOutput(); if(!temp.BufferSet(SecondInput.GetIndex())) ReturnFalse; } //--- if(!cGlobalResidual.FeedForward(NeuronOCL)) ReturnFalse; if(!acFeatureProjection[0].FeedForward(NeuronOCL)) ReturnFalse; if(!acFeatureProjection[1].FeedForward(cFeatureInput.AsObject())) ReturnFalse; //--- if(!Concat(NeuronOCL.getOutput(), acFeatureProjection[0].getOutput(), cEncoderInput.getOutput(), iHistory, iHistory, iVariables)) ReturnFalse; uint total = acEncoderDecoder.Size(); CNeuronBaseOCL *prev = cEncoderInput.AsObject(); for(uint i = 0; i < total; i++) { if(!acEncoderDecoder[i].FeedForward(prev)) ReturnFalse; prev = acEncoderDecoder[i].AsObject(); } //--- if(!Concat(prev.getOutput(), acFeatureProjection[1].getOutput(), cTemporalDecoderInput.getOutput(), iForecast, iForecast, iVariables)) ReturnFalse; if(!cTemporalDecoder.FeedForward(cTemporalDecoderInput.AsObject())) ReturnFalse; if(!SumAndNormilize(cGlobalResidual.getOutput(), cTemporalDecoder.getOutput(), Output, iForecast, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!cTemporalDecoderInput.CalcHiddenGradients(cTemporalDecoder.AsObject())) ReturnFalse; //--- int total = (int)acEncoderDecoder.Size(); if(!DeConcat(acEncoderDecoder[total - 1].getGradient(), acFeatureProjection[1].getGradient(), cTemporalDecoderInput.getGradient(), iForecast, iForecast, iVariables)) ReturnFalse; for(int i = total - 2; i >= 0; i--) if(!acEncoderDecoder[i].CalcHiddenGradients(acEncoderDecoder[i + 1].AsObject())) ReturnFalse; if(!cEncoderInput.CalcHiddenGradients(acEncoderDecoder[0].AsObject())) ReturnFalse; //--- if(!DeConcat(cHistoryInput.getGradient(), acFeatureProjection[0].getGradient(), cEncoderInput.getGradient(), iHistory, iHistory, iVariables)) ReturnFalse; //--- if(cGlobalResidual.Activation() != None) { if(!DeActivation(cGlobalResidual.getOutput(), cGlobalResidual.getGradient(), cGlobalResidual.getGradient(), cGlobalResidual.Activation())) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(cGlobalResidual.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(cHistoryInput.getOutput(), cHistoryInput.getGradient(), cHistoryInput.getGradient(), SecondActivation)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cHistoryInput.getGradient(), NeuronOCL.getGradient(), iHistory, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cFeatureInput.CalcHiddenGradients(acFeatureProjection[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cFeatureInput.getGradient(), cFeatureInput.getGradient(), SecondGradient, iFeatures, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!cFeatureInput.CalcHiddenGradients(acFeatureProjection[1].AsObject())) ReturnFalse; if(!SumAndNormilize(SecondGradient, cFeatureInput.getGradient(), SecondGradient, iFeatures, false, 0, 0, 0, 1.0f)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- if(!cGlobalResidual.UpdateInputWeights(cHistoryInput.AsObject())) ReturnFalse; if(!acFeatureProjection[0].UpdateInputWeights(cHistoryInput.AsObject())) ReturnFalse; if(!acFeatureProjection[1].UpdateInputWeights(cFeatureInput.AsObject())) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); CNeuronBaseOCL *prev = cEncoderInput.AsObject(); for(uint i = 0; i < total; i++) { if(!acEncoderDecoder[i].UpdateInputWeights(prev)) ReturnFalse; prev = acEncoderDecoder[i].AsObject(); } //--- if(!cTemporalDecoder.UpdateInputWeights(cTemporalDecoderInput.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iHistory, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iForecast, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables, INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFeatures, INT_VALUE) < INT_VALUE) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); if(FileWriteInteger(file_handle, (int)total, INT_VALUE) < INT_VALUE) ReturnFalse; for(uint i = 0; i < total; i++) if(!acEncoderDecoder[i].Save(file_handle)) ReturnFalse; if(!cGlobalResidual.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!acFeatureProjection[i].Save(file_handle)) ReturnFalse; if(!cTemporalDecoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFeatures = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; //--- int total = FileReadInteger(file_handle); int prev_size = (int)acEncoderDecoder.Size(); if(prev_size != total) if(ArrayResize(acEncoderDecoder, total) < total) ReturnFalse; for(int i = 0; i < total; i++) { if(i >= prev_size) if(!acEncoderDecoder[i].Init(0, i + 2, OpenCL, 1, 1, 1, ADAM, 1)) ReturnFalse; if(!LoadInsideLayer(file_handle, acEncoderDecoder[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cGlobalResidual.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, acFeatureProjection[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalDecoder.AsObject())) ReturnFalse; //--- if(!cHistoryInput.Init(0, total + 5, OpenCL, iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cFeatureInput.Init(0, total + 6, OpenCL, iFeatures, optimization, iBatch)) ReturnFalse; if(!cEncoderInput.Init(0, total + 7, OpenCL, 2 * iHistory * iVariables, optimization, iBatch)) ReturnFalse; if(!cTemporalDecoderInput.Init(0, total + 8, OpenCL, 2 * iForecast * iVariables, optimization, iBatch)) ReturnFalse; //--- if(cGlobalResidual.getGradient() != Gradient) if(!cGlobalResidual.SetGradient(Gradient)) ReturnFalse; if(cTemporalDecoder.getGradient() != getGradient()) if(!cTemporalDecoder.SetGradient(Gradient)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTiDEOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- uint total = acEncoderDecoder.Size(); for(uint i = 0; i < total; i++) acEncoderDecoder[i].SetOpenCL(OpenCL); cGlobalResidual.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) acFeatureProjection[i].SetOpenCL(OpenCL); cTemporalDecoder.SetOpenCL(OpenCL); //--- cHistoryInput.SetOpenCL(OpenCL); cFeatureInput.SetOpenCL(OpenCL); cEncoderInput.SetOpenCL(OpenCL); cTemporalDecoderInput.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTiDEOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { //--- if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTiDEOCL *Source = source; if(!cGlobalResidual.WeightsUpdate(Source.cGlobalResidual.AsObject(), tau)) ReturnFalse; if(!acFeatureProjection[0].WeightsUpdate(Source.acFeatureProjection[0].AsObject(), tau)) ReturnFalse; if(!acFeatureProjection[1].WeightsUpdate(Source.acFeatureProjection[1].AsObject(), tau)) ReturnFalse; //--- uint total = acEncoderDecoder.Size(); for(uint i = 0; i < total; i++) if(!acEncoderDecoder[i].WeightsUpdate(Source.acEncoderDecoder[i].AsObject(), tau)) ReturnFalse; //--- if(!cTemporalDecoder.WeightsUpdate(Source.cTemporalDecoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLegendreWavelets : public CNeuronConvOCL { protected: virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } ///< Method for updating weights.@param NeuronOCL Pointer to previos layer. public: CNeuronLegendreWavelets(void) {}; ~CNeuronLegendreWavelets(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronLegendreWavelets; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLegendreWavelets::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint units_count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, step, filters, units_count, optimization_type, batch)) ReturnFalse; //--- WeightsConv.BufferInit(WeightsConv.Total(), 0); for(uint i = 0; i < iWindow; i++) { uint shift = i; float k = float(2.0 * i - 1.0) / iWindow; for(uint f = 1; f <= filters; f++) { float value = 0; switch(f) { case 1: value = k; break; case 2: value = (3 * k * k - 1) / 2; break; default: value = ((2 * f - 1) * k * WeightsConv.At(shift - (iWindow + 1)) - (f - 1) * WeightsConv.At(shift - 2 * (iWindow + 1))) / f; break; } if(!WeightsConv.Update(shift, value)) ReturnFalse; shift += iWindow + 1; } } if(!!OpenCL) if(!WeightsConv.BufferWrite()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFEDW : public CNeuronBaseOCL { protected: //--- uint iWindow; uint iCount; uint iFilters; //--- CNeuronLegendreWavelets cWavlets; CNeuronBatchNormOCL cNorm; CNeuronSoftMaxOCL cSoftMax; CNeuronConvOCL cFF[2]; CNeuronBaseOCL cReconstruct; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); //--- virtual bool Reconsruct(CBufferFloat* inputs, CBufferFloat *outputs); public: CNeuronFEDW(void) {}; ~CNeuronFEDW(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronFEDW; } virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint count, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; iFilters = MathMax(filters, 1); //--- if(!cWavlets.Init(0, 0, OpenCL, iWindow, iWindow, iCount, iFilters, optimization, iBatch)) ReturnFalse; cWavlets.SetActivationFunction(None); if(!cNorm.Init(0, 1, OpenCL, iFilters * iCount, 1000, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); if(!cSoftMax.Init(0, 1, OpenCL, iFilters * iCount, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iCount); cSoftMax.SetActivationFunction(None); if(!cReconstruct.Init(0, 2, OpenCL, iWindow, optimization, iBatch)) ReturnFalse; cReconstruct.SetActivationFunction(None); if(!cFF[0].Init(0, 3, OpenCL, iWindow, iWindow, 4 * iWindow, iCount, optimization, iBatch)) ReturnFalse; cFF[0].SetActivationFunction(LReLU); if(!cFF[1].Init(0, 4, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iCount, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- if(Gradient != cFF[1].getGradient()) SetGradient(cFF[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cWavlets.FeedForward(NeuronOCL.AsObject())) ReturnFalse; if(!cNorm.FeedForward(cWavlets.AsObject())) ReturnFalse; if(!cSoftMax.FeedForward(cNorm.AsObject())) ReturnFalse; if(!Reconsruct(cReconstruct.getOutput(), cSoftMax.getOutput())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cReconstruct.getOutput(), cReconstruct.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFF[0].FeedForward(cReconstruct.AsObject())) ReturnFalse; if(!cFF[1].FeedForward(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cFF[1].getOutput(), cReconstruct.getOutput(), getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::Reconsruct(CBufferFloat * sequence, CBufferFloat * probability) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = sequence.Total(); global_work_size[1] = 1; setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_w, cWavlets.GetWeightsConv().GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_g, probability.GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_o, probability.GetIndex()) setBuffer(def_k_CalcHiddenGradientConv, def_k_chgc_matrix_ig, sequence.GetIndex()) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_outputs, probability.Total()) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_step, (int)iWindow) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_in, (int)iWindow) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_window_out, (int)iFilters) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_activation, (int)None) setArgument(def_k_CalcHiddenGradientConv, def_k_chgc_shift_out, (int)0) kernelExecute(def_k_CalcHiddenGradientConv, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cFF[0].CalcHiddenGradients(cFF[1].AsObject())) ReturnFalse; if(!cReconstruct.CalcHiddenGradients(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(Gradient, cReconstruct.getGradient(), cReconstruct.getGradient(), iWindow, false)) ReturnFalse; //--- CBufferFloat *temp_r = cReconstruct.getOutput(); if(!cReconstruct.SetOutput(cReconstruct.getGradient(), false)) ReturnFalse; CBufferFloat *temp_w = cWavlets.getOutput(); if(!cWavlets.SetOutput(cSoftMax.getGradient(), false)) ReturnFalse; //--- if(!cWavlets.FeedForward(cReconstruct.AsObject())) ReturnFalse; //--- if(!cWavlets.SetOutput(temp_w, false)) ReturnFalse; if(!cReconstruct.SetOutput(temp_r, false)) ReturnFalse; //--- if(!cNorm.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!cWavlets.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cWavlets.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cReconstruct.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cFF[0].UpdateInputWeights(cReconstruct.AsObject())) ReturnFalse; if(!cFF[1].UpdateInputWeights(cFF[0].AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cWavlets.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFEDW::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cWavlets.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cFF[0].SetOpenCL(OpenCL); cFF[1].SetOpenCL(OpenCL); cReconstruct.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWavlets.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cFF[0].Save(file_handle)) ReturnFalse; if(!cFF[1].Save(file_handle)) ReturnFalse; if(!cReconstruct.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFilters) < INT_VALUE) ReturnFalse;//--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cWavlets.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSoftMax.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFF[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFF[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cReconstruct.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); iCount = (uint)FileReadInteger(file_handle); iFilters = (uint)FileReadInteger(file_handle); //--- if(Gradient != cFF[1].getGradient()) SetGradient(cFF[1].getGradient()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFEDW::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFEDW *Source = source; for(int i = 0; i < 2; i++) { if(!cFF[i].WeightsUpdate(Source.cFF[i].AsObject(), tau)) ReturnFalse; } if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFITSOCL : public CNeuronBaseOCL { protected: //--- uint iWindow; uint iWindowOut; uint iCount; uint iFFTin; uint iIFFTin; //--- CNeuronBaseOCL cInputsRe; CNeuronBaseOCL cInputsIm; CNeuronBaseOCL cFFTRe; CNeuronBaseOCL cFFTIm; CNeuronDropoutOCL cDropRe; CNeuronDropoutOCL cDropIm; CNeuronConvOCL cInsideRe1; CNeuronConvOCL cInsideIm1; CNeuronConvOCL cInsideRe2; CNeuronConvOCL cInsideIm2; CNeuronBaseOCL cComplexRe; CNeuronBaseOCL cComplexIm; CNeuronBaseOCL cIFFTRe; CNeuronBaseOCL cIFFTIm; CBufferFloat cClear; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false); virtual bool ComplexLayerOut(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im); virtual bool ComplexLayerGradient(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronFITSOCL(void) {}; ~CNeuronFITSOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint count, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronFITSOCL; } virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint count, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * count, optimization_type, batch)) ReturnFalse; //--- Save constants iWindow = window; iWindowOut = window_out; iCount = count; activation = None; //--- Calculate FFT and iFFT size int power = int(MathLog(iWindow) / M_LN2); if(MathPow(2, power) != iWindow) power++; iFFTin = uint(MathPow(2, power)); power = int(MathLog(iWindowOut) / M_LN2); if(MathPow(2, power) != iWindowOut) power++; iIFFTin = uint(MathPow(2, power)); //--- Init objects if(!cInputsRe.Init(0, 0, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cInputsIm.Init(0, 1, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTRe.Init(0, 2, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTIm.Init(0, 3, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cDropRe.Init(0, 4, OpenCL, iFFTin * iCount, dropout, optimization, iBatch)) ReturnFalse; if(!cDropIm.Init(0, 5, OpenCL, iFFTin * iCount, dropout, optimization, iBatch)) ReturnFalse; if(!cInsideRe1.Init(0, 6, OpenCL, iFFTin, iFFTin, 4 * iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideRe1.SetActivationFunction(TANH); if(!cInsideIm1.Init(0, 7, OpenCL, iFFTin, iFFTin, 4 * iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideIm1.SetActivationFunction(TANH); if(!cInsideRe2.Init(0, 8, OpenCL, 4 * iIFFTin, 4 * iIFFTin, iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideRe2.SetActivationFunction(None); if(!cInsideIm2.Init(0, 9, OpenCL, 4 * iIFFTin, 4 * iIFFTin, iIFFTin, iCount, optimization, iBatch)) ReturnFalse; cInsideIm2.SetActivationFunction(None); if(!cComplexRe.Init(0, 10, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexIm.Init(0, 11, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTRe.Init(0, 12, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTIm.Init(0, 13, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cClear.BufferInit(MathMax(iFFTin, iIFFTin)*iCount, 0)) ReturnFalse; cClear.BufferCreate(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iCount}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iCount)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iCount)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::ComplexLayerOut(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im) { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iCount, inp_re.Total() / iCount }; setBuffer(def_k_ComplexLayer, def_k_cl_inputs_re, inp_re.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_inputs_im, inp_im.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_outputs_re, out_re.GetIndex()) setBuffer(def_k_ComplexLayer, def_k_cl_outputs_im, out_im.GetIndex()) kernelExecute(def_k_ComplexLayer, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::ComplexLayerGradient(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im) { uint global_work_offset[2] = {0}; uint global_work_size[2] = { iCount, inp_re.Total() / iCount }; setBuffer(def_k_ComplexLayerGradient, def_k_cl_inputs_re, inp_re.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_inputs_im, inp_im.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_outputs_re, out_re.GetIndex()) setBuffer(def_k_ComplexLayerGradient, def_k_cl_outputs_im, out_im.GetIndex()) kernelExecute(def_k_ComplexLayerGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- FFT if(!FFT(NeuronOCL.getOutput(), NULL, cFFTRe.getOutput(), cFFTIm.getOutput(), false)) ReturnFalse; //--- DropOut if(!cDropRe.FeedForward(cFFTRe.AsObject())) ReturnFalse; if(!cDropIm.FeedForward(cFFTIm.AsObject())) ReturnFalse; //--- Complex Layer if(!cInsideRe1.FeedForward(cDropRe.AsObject())) ReturnFalse; if(!cInsideRe2.FeedForward(cInsideRe1.AsObject())) ReturnFalse; if(!cInsideIm1.FeedForward(cDropIm.AsObject())) ReturnFalse; if(!cInsideIm2.FeedForward(cInsideIm1.AsObject())) ReturnFalse; if(!ComplexLayerOut(cInsideRe2.getOutput(), cInsideIm2.getOutput(), cComplexRe.getOutput(), cComplexIm.getOutput())) ReturnFalse; //--- iFFT if(!FFT(cComplexRe.getOutput(), cComplexIm.getOutput(), cIFFTRe.getOutput(), cIFFTIm.getOutput(), true)) ReturnFalse; //--- To Output if(!DeConcat(Output, cIFFTRe.getGradient(), cIFFTRe.getOutput(), iWindowOut, iIFFTin - iWindowOut, iCount)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Copy Gradients if(!SumAndNormilize(cIFFTIm.getOutput(), GetPointer(cClear), cIFFTIm.getGradient(), 1, false, 0, 0, 0, -1)) ReturnFalse; if(!Concat(Gradient, GetPointer(cClear), cIFFTRe.getGradient(), iWindowOut, iIFFTin - iWindowOut, iCount)) ReturnFalse; //--- FFT if(!FFT(cIFFTRe.getGradient(), cIFFTIm.getGradient(), cComplexRe.getGradient(), cComplexIm.getGradient(), false)) ReturnFalse; //--- Complex Layer if(!ComplexLayerGradient(cInsideRe2.getGradient(), cInsideIm2.getGradient(), cComplexRe.getGradient(), cComplexIm.getGradient())) ReturnFalse; if(!cInsideRe1.CalcHiddenGradients(cInsideRe2.AsObject())) ReturnFalse; if(!cInsideIm1.CalcHiddenGradients(cInsideIm2.AsObject())) ReturnFalse; if(!cDropRe.CalcHiddenGradients(cInsideRe1.AsObject())) ReturnFalse; if(!cDropIm.CalcHiddenGradients(cInsideIm1.AsObject())) ReturnFalse; //--- Dropout if(!cFFTRe.CalcHiddenGradients(cDropRe.AsObject())) ReturnFalse; if(!cFFTIm.CalcHiddenGradients(cDropIm.AsObject())) ReturnFalse; //--- IFFT if(!FFT(cFFTRe.getGradient(), cFFTIm.getGradient(), cInputsRe.getGradient(), cInputsIm.getGradient(), true)) ReturnFalse; //--- To Input Layer if(!DeConcat(NeuronOCL.getGradient(), cFFTIm.getGradient(), cFFTRe.getGradient(), iWindow, iFFTin - iWindow, iCount)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cInsideRe1.UpdateInputWeights(cDropRe.AsObject())) ReturnFalse; if(!cInsideIm1.UpdateInputWeights(cDropIm.AsObject())) ReturnFalse; if(!cInsideRe2.UpdateInputWeights(cInsideRe1.AsObject())) ReturnFalse; if(!cInsideIm2.UpdateInputWeights(cInsideIm1.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowOut)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTin)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iIFFTin)) < INT_VALUE) ReturnFalse; //--- Save objects if(!cInsideRe1.Save(file_handle)) ReturnFalse; if(!cInsideIm1.Save(file_handle)) ReturnFalse; if(!cInsideRe2.Save(file_handle)) ReturnFalse; if(!cInsideIm2.Save(file_handle)) ReturnFalse; if(!cDropRe.Save(file_handle)) ReturnFalse; if(!cDropIm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iWindowOut = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iFFTin = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iIFFTin = uint(FileReadInteger(file_handle)); activation = None; //--- Load objects if(!LoadInsideLayer(file_handle, cInsideRe1.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideIm1.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideRe2.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideIm2.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDropRe.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDropIm.AsObject())) ReturnFalse; //--- Init objects if(!cInputsRe.Init(0, 0, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cInputsIm.Init(0, 1, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTRe.Init(0, 2, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cFFTIm.Init(0, 3, OpenCL, iFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexRe.Init(0, 8, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cComplexIm.Init(0, 9, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTRe.Init(0, 10, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cIFFTIm.Init(0, 11, OpenCL, iIFFTin * iCount, optimization, iBatch)) ReturnFalse; if(!cClear.BufferInit(MathMax(iFFTin, iIFFTin)*iCount, 0)) ReturnFalse; cClear.BufferCreate(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFITSOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cInputsRe.SetOpenCL(OpenCL); cInputsIm.SetOpenCL(OpenCL); cFFTRe.SetOpenCL(OpenCL); cFFTIm.SetOpenCL(OpenCL); cDropRe.SetOpenCL(OpenCL); cDropIm.SetOpenCL(OpenCL); cInsideRe1.SetOpenCL(OpenCL); cInsideIm1.SetOpenCL(OpenCL); cInsideRe2.SetOpenCL(OpenCL); cInsideIm2.SetOpenCL(OpenCL); cComplexRe.SetOpenCL(OpenCL); cComplexIm.SetOpenCL(OpenCL); cIFFTRe.SetOpenCL(OpenCL); cIFFTIm.SetOpenCL(OpenCL); cClear.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFITSOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cInputsRe.TrainMode(bTrain); cInputsIm.TrainMode(bTrain); cFFTRe.TrainMode(bTrain); cFFTIm.TrainMode(bTrain); cDropRe.TrainMode(bTrain); cDropIm.TrainMode(bTrain); cInsideRe1.TrainMode(bTrain); cInsideIm1.TrainMode(bTrain); cInsideRe2.TrainMode(bTrain); cInsideIm2.TrainMode(bTrain); cComplexRe.TrainMode(bTrain); cComplexIm.TrainMode(bTrain); cIFFTRe.TrainMode(bTrain); cIFFTIm.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFITSOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFITSOCL *Source = source; if(cInsideRe1.WeightsUpdate(Source.cInsideRe1.AsObject(), tau)) ReturnFalse; if(cInsideIm1.WeightsUpdate(Source.cInsideIm1.AsObject(), tau)) ReturnFalse; if(cInsideRe2.WeightsUpdate(Source.cInsideRe2.AsObject(), tau)) ReturnFalse; if(cInsideIm2.WeightsUpdate(Source.cInsideIm2.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFreDFOCL : public CNeuronBaseOCL { protected: uint iWindow; uint iCount; uint iFFTin; bool bTranspose; float fAlpha; //--- CBufferFloat cForecastFreRe; CBufferFloat cForecastFreIm; CBufferFloat cTargetFreRe; CBufferFloat cTargetFreIm; CBufferFloat cLossFreRe; CBufferFloat cLossFreIm; CBufferFloat cGradientFreRe; CBufferFloat cGradientFreIm; CBufferFloat cTranspose; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false); virtual bool Transpose(CBufferFloat *inputs, CBufferFloat *outputs, uint rows, uint cols); virtual bool FreqMSA(CBufferFloat *target, CBufferFloat *forecast, CBufferFloat *gradient); virtual bool CumulativeGradient(CBufferFloat *gradient1, CBufferFloat *gradient2, CBufferFloat *cummulative, float alpha); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronFreDFOCL(void) {}; ~CNeuronFreDFOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint count, float alpha, bool need_transpose = true, ENUM_OPTIMIZATION optimization_type = ADAM, uint batch = 1); virtual bool calcOutputGradients(CArrayFloat *Target, float &error); //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual int Type(void) const { return defNeuronFreDFOCL; } virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint count, float Alpha, bool need_transpose = true, ENUM_OPTIMIZATION optimization_type = ADAM, uint batch = 1) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * count, optimization_type, batch)) ReturnFalse; //--- bTranspose = need_transpose; iWindow = window; iCount = count; fAlpha = MathMax(0, MathMin(Alpha, 1)); activation = None; //--- Calculate FFTsize uint size = (bTranspose ? count : window); int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) != size) power++; iFFTin = uint(MathPow(2, power)); //--- uint n = (bTranspose ? iWindow : iCount); if(!cForecastFreRe.BufferInit(iFFTin * n, 0) || !cForecastFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastFreIm.BufferInit(iFFTin * n, 0) || !cForecastFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreRe.BufferInit(iFFTin * n, 0) || !cTargetFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreIm.BufferInit(iFFTin * n, 0) || !cTargetFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreRe.BufferInit(iFFTin * n, 0) || !cLossFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreIm.BufferInit(iFFTin * n, 0) || !cLossFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreRe.BufferInit(iFFTin * n, 0) || !cGradientFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreIm.BufferInit(iFFTin * n, 0) || !cGradientFreIm.BufferCreate(OpenCL)) ReturnFalse; if(bTranspose) if(!cTranspose.BufferInit(iWindow * iCount, 0) || !cTranspose.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {(bTranspose ? iWindow : iCount)}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / (!bTranspose ? iWindow : iCount))) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / (!bTranspose ? iWindow : iCount))) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; if(NeuronOCL.getOutput() != Output) { Output.BufferFree(); DeleteObj(Output); Output = NeuronOCL.getOutput(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Transpose(CBufferFloat * inputs, CBufferFloat * outputs, uint rows, uint cols) { if(!inputs || !outputs) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {rows, cols}; setBuffer(def_k_Transpose, def_k_tr_matrix_in, inputs.GetIndex()) setBuffer(def_k_Transpose, def_k_tr_matrix_out, outputs.GetIndex()) kernelExecute(def_k_Transpose, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::FreqMSA(CBufferFloat * target, CBufferFloat * forecast, CBufferFloat * gradient) { if(!target || !forecast || !gradient) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {gradient.Total()}; setBuffer(def_k_GradientMSA, def_k_gmsa_target, target.GetIndex()) setBuffer(def_k_GradientMSA, def_k_gmsa_forecast, forecast.GetIndex()) setBuffer(def_k_GradientMSA, def_k_gmsa_gradient, gradient.GetIndex()) kernelExecute(def_k_GradientMSA, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::CumulativeGradient(CBufferFloat * gradient1, CBufferFloat * gradient2, CBufferFloat * cummulative, float Alpha) { if(!gradient1 || !gradient2 || !cummulative) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {cummulative.Total()}; setBuffer(def_k_CumulativeGradient, def_k_cg_gradient1, gradient1.GetIndex()) setBuffer(def_k_CumulativeGradient, def_k_cg_gradient2, gradient2.GetIndex()) setBuffer(def_k_CumulativeGradient, def_k_cg_gradient_out, cummulative.GetIndex()) setArgument(def_k_CumulativeGradient, def_k_cg_alpha, float(Alpha)) kernelExecute(def_k_CumulativeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::calcOutputGradients(CArrayFloat * Target, float & error) { if(!Target) ReturnFalse; if(Target.Total() < Output.Total()) ReturnFalse; //--- if(Target.Total() == Output.Total()) { if(!Gradient.AssignArray(Target)) ReturnFalse; } else { for(int i = 0; i < Output.Total(); i++) { if(!Gradient.Update(i, Target.At(i))) ReturnFalse; } } if(!Gradient.BufferWrite()) ReturnFalse; //--- if(bTranspose) { if(!Transpose(Output, GetPointer(cTranspose), iWindow, iCount)) ReturnFalse; if(!FFT(GetPointer(cTranspose), NULL, GetPointer(cForecastFreRe), GetPointer(cForecastFreIm), false)) ReturnFalse; if(!Transpose(Gradient, GetPointer(cTranspose), iWindow, iCount)) ReturnFalse; if(!FFT(GetPointer(cTranspose), NULL, GetPointer(cTargetFreRe), GetPointer(cTargetFreIm), false)) ReturnFalse; } else { if(!FFT(Output, NULL, GetPointer(cForecastFreRe), GetPointer(cForecastFreIm), false)) ReturnFalse; if(!FFT(Gradient, NULL, GetPointer(cTargetFreRe), GetPointer(cTargetFreIm), false)) ReturnFalse; } //--- if(!FreqMSA(GetPointer(cTargetFreRe), GetPointer(cForecastFreRe), GetPointer(cLossFreRe))) ReturnFalse; if(!FreqMSA(GetPointer(cTargetFreIm), GetPointer(cForecastFreIm), GetPointer(cLossFreIm))) ReturnFalse; if(!FreqMSA(Gradient, Output, Gradient)) ReturnFalse; //--- if(!FFT(GetPointer(cLossFreRe), GetPointer(cLossFreIm), GetPointer(cGradientFreRe), GetPointer(cGradientFreIm), true)) ReturnFalse; if(bTranspose) { if(!Transpose(GetPointer(cGradientFreRe), GetPointer(cTranspose), iCount, iWindow)) ReturnFalse; if(!CumulativeGradient(GetPointer(cTranspose), Gradient, Gradient, fAlpha)) ReturnFalse; } else if(!CumulativeGradient(GetPointer(cGradientFreRe), Gradient, Gradient, fAlpha)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- return DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, NeuronOCL.Activation()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTin)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(bTranspose)) < INT_VALUE) ReturnFalse; if(FileWriteFloat(file_handle, fAlpha) < sizeof(float)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFreDFOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iFFTin = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; bTranspose = bool(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; fAlpha = FileReadFloat(file_handle); //--- uint n = (bTranspose ? iWindow : iCount); if(!cForecastFreRe.BufferInit(iFFTin * n, 0) || !cForecastFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastFreIm.BufferInit(iFFTin * n, 0) || !cForecastFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreRe.BufferInit(iFFTin * n, 0) || !cTargetFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cTargetFreIm.BufferInit(iFFTin * n, 0) || !cTargetFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreRe.BufferInit(iFFTin * n, 0) || !cLossFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cLossFreIm.BufferInit(iFFTin * n, 0) || !cLossFreIm.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreRe.BufferInit(iFFTin * n, 0) || !cGradientFreRe.BufferCreate(OpenCL)) ReturnFalse; if(!cGradientFreIm.BufferInit(iFFTin * n, 0) || !cGradientFreIm.BufferCreate(OpenCL)) ReturnFalse; if(bTranspose) { if(!cTranspose.BufferInit(iWindow * iCount, 0) || !cTranspose.BufferCreate(OpenCL)) ReturnFalse; } else { cTranspose.BufferFree(); cTranspose.Clear(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFreDFOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cForecastFreRe.BufferCreate(OpenCL); cForecastFreIm.BufferCreate(OpenCL); cTargetFreRe.BufferCreate(OpenCL); cTargetFreIm.BufferCreate(OpenCL); cLossFreRe.BufferCreate(OpenCL); cLossFreIm.BufferCreate(OpenCL); cGradientFreRe.BufferCreate(OpenCL); cGradientFreIm.BufferCreate(OpenCL); if(bTranspose) cTranspose.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronComplexMLMHAttention : public CNeuronMLMHAttentionOCL { protected: virtual bool ConvolutionForward(CBufferFloat *weights, CBufferFloat *inputs, CBufferFloat *outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0); virtual bool AttentionScore(CBufferFloat *qkv, CBufferFloat *scores, bool mask = false); virtual bool AttentionOut(CBufferFloat *qkv, CBufferFloat *scores, CBufferFloat *out); virtual bool ConvolutuionUpdateWeights(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *momentum1, CBufferFloat *momentum2, uint window, uint window_out, uint step = 0); virtual bool ConvolutionInputGradients(CBufferFloat *weights, CBufferFloat *gradient, CBufferFloat *inputs, CBufferFloat *inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0); virtual bool AttentionInsideGradients(CBufferFloat *qkv, CBufferFloat *qkv_g, CBufferFloat *scores, CBufferFloat *gradient); virtual bool SumAndNormilize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f); public: CNeuronComplexMLMHAttention(void) {}; ~CNeuronComplexMLMHAttention(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronComplexMLMHAttentionOCL; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); //--- uint num = 2 * 3 * iWindowKey * iHeads * iUnits; //Size of QKV tensor uint qkv_weights = 2 * 3 * (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of QKV tenzor uint scores = 2 * iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = 2 * iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = 2 * iWindow * iUnits; //Size of our tensore uint w0 = 2 * (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = 4 * (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize QKV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize QKV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(qkv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < qkv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(qkv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(w0, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_1, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(ff_2, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionForward(CBufferFloat * weights, CBufferFloat * inputs, CBufferFloat * outputs, uint window, uint window_out, ENUM_ACTIVATION activ, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(outputs) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(outputs.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = outputs.Total() / (2 * window_out); global_work_size[1] = window_out; global_work_size[2] = 1; //--- setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_w, weights.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_i, inputs.GetIndex()) setBuffer(def_k_FeedForwardComplexConv, def_k_ffc_matrix_o, outputs.GetIndex()) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_inputs, (int)(inputs.Total() / 2)) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_step, (int)step) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_window_in, (int)window) setArgument(def_k_FeedForwardComplexConv, def_k_ffс_window_out, (int)window_out) setArgument(def_k_FeedForwardComplexConv, def_k_ffc_activation - 1, (int)activ) //--- kernelExecute(def_k_FeedForwardComplexConv, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionScore(CBufferFloat * qkv, CBufferFloat * scores, bool mask = false) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID) ReturnFalse; //--- if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionScore, def_k_mhas_score, scores.GetIndex()) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_dimension, (int)iWindowKey) setArgument(def_k_ComplexMHAttentionScore, def_k_mhas_mask, (int)mask) kernelExecute(def_k_ComplexMHAttentionScore, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionOut(CBufferFloat * qkv, CBufferFloat * scores, CBufferFloat * out) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iHeads}; if(qkv.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionOut, def_k_mhao_out, out.GetIndex()) setArgument(def_k_ComplexMHAttentionOut, def_k_mhao_dimension, (int)iWindowKey) //--- kernelExecute(def_k_ComplexMHAttentionOut, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutuionUpdateWeights(CBufferFloat * weights, CBufferFloat * gradient, CBufferFloat * inputs, CBufferFloat * momentum1, CBufferFloat * momentum2, uint window, uint window_out, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(momentum1) == POINTER_INVALID) ReturnFalse; if(step == 0) step = window; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = weights.Total() / 2; global_work_size[1] = weights.Total() / 2; if(weights.GetIndex() < 0) ReturnFalse; if(optimization == SGD) { if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_matrix_dw, momentum1.GetIndex()) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_learning_rates, lr) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_momentum, alpha) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvMomentum, def_k_uwcm_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvMomentum, global_work_offset, global_work_size) } else { if(CheckPointer(momentum2) == POINTER_INVALID) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(momentum1.GetIndex() < 0) ReturnFalse; if(momentum2.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_w, weights.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_g, gradient.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_i, inputs.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_m, momentum1.GetIndex()) setBuffer(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_matrix_v, momentum2.GetIndex()) float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_inputs, int(inputs.Total() / 2)) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_l, lt) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b1, b1) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_b2, b2) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_in, (int)window) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_window_out, (int)window_out) setArgument(def_k_UpdateWeightsComplexConvAdam, def_k_uwca_step, (int)step) ResetLastError(); kernelExecute(def_k_UpdateWeightsComplexConvAdam, global_work_offset, global_work_size) t++; } //--- return true;; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::ConvolutionInputGradients(CBufferFloat * weights, CBufferFloat * gradient, CBufferFloat * inputs, CBufferFloat * inp_gradient, uint window, uint window_out, uint activ, uint shift_out = 0, uint step = 0) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(weights) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID || CheckPointer(inputs) == POINTER_INVALID || CheckPointer(inp_gradient) == POINTER_INVALID) ReturnFalse; //--- if(weights.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; if(inputs.GetIndex() < 0) ReturnFalse; if(inp_gradient.GetIndex() < 0) ReturnFalse; if(step == 0) step = window; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = inputs.Total() / 2; global_work_size[1] = 1; //--- setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_w, weights.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_g, gradient.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_o, inputs.GetIndex()) setBuffer(def_k_CalcHiddenGradientComplexConv, def_k_chgc_matrix_ig, inp_gradient.GetIndex()) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_outputs, int(Neurons() / 2) - shift_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_step, (int)step) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_in, (int)window) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_window_out, (int)window_out) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_activation, (int)activ) setArgument(def_k_CalcHiddenGradientComplexConv, def_k_chgc_shift_out, (int)shift_out) kernelExecute(def_k_CalcHiddenGradientComplexConv, global_work_offset, global_work_size) //--- return true;; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::AttentionInsideGradients(CBufferFloat * qkv, CBufferFloat * qkv_g, CBufferFloat * scores, CBufferFloat * gradient) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(qkv) == POINTER_INVALID || CheckPointer(qkv_g) == POINTER_INVALID || CheckPointer(scores) == POINTER_INVALID || CheckPointer(gradient) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = iUnits; global_work_size[1] = iHeads; global_work_size[2] = iWindowKey; if(qkv.GetIndex() < 0) ReturnFalse; if(qkv_g.GetIndex() < 0) ReturnFalse; if(scores.GetIndex() < 0) ReturnFalse; if(gradient.GetIndex() < 0) ReturnFalse; //--- setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv, qkv.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_qkv_g, qkv_g.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_score, scores.GetIndex()) setBuffer(def_k_ComplexMHAttentionGradients, def_k_mhag_gradient, gradient.GetIndex()) //--- kernelExecute(def_k_ComplexMHAttentionGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMLMHAttention::SumAndNormilize(CBufferFloat * tensor1, CBufferFloat * tensor2, CBufferFloat * out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; setBuffer(def_k_MatrixSum, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(def_k_MatrixSum, def_k_sum_matrix_out, out.GetIndex()) setArgument(def_k_MatrixSum, def_k_sum_dimension, 2 * dimension) setArgument(def_k_MatrixSum, def_k_sum_shift_in1, 2 * shift_in1) setArgument(def_k_MatrixSum, def_k_sum_shift_in2, 2 * shift_in2) setArgument(def_k_MatrixSum, def_k_sum_shift_out, 2 * shift_out) setArgument(def_k_MatrixSum, def_k_sum_multiplyer, multiplyer) kernelExecute(def_k_MatrixSum, global_work_offset, global_work_size) //--- if(!normilize) return true; //--- setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, out.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, dimension) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronATFNetOCL : public CNeuronBaseOCL { protected: uint iHistory; uint iForecast; uint iVariables; uint iFFT; //--- T-Block CNeuronBatchNormOCL cNorm; CNeuronTransposeOCL cTranspose; CNeuronPositionEncoder cPositionEncoder; CNeuronPatching cPatching; CLayer caAttention; CLayer caProjection; CNeuronRevINDenormOCL cRevIN; //--- F-Block CBufferFloat *cInputs; CBufferFloat cInputFreqRe; CBufferFloat cInputFreqIm; CNeuronBaseOCL cInputFreqComplex; CBufferFloat cMainFreqWeights; CNeuronBaseOCL cNormFreqComplex; CBufferFloat cMeans; CBufferFloat cVariances; CNeuronComplexMLMHAttention cFreqAtteention; CNeuronBaseOCL cUnNormFreqComplex; CBufferFloat cOutputFreqRe; CBufferFloat cOutputFreqIm; CBufferFloat cOutputTimeSeriasRe; CBufferFloat cOutputTimeSeriasIm; CBufferFloat cOutputTimeSeriasReGrad; CBufferFloat cReconstructInput; CBufferFloat cForecast; CBufferFloat cReconstructInputGrad; CBufferFloat cForecastGrad; CBufferFloat cZero; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool ComplexNormalize(void); virtual bool ComplexUnNormalize(void); virtual bool ComplexNormalizeGradient(void); virtual bool ComplexUnNormalizeGradient(void); virtual bool MainFreqWeights(void); virtual bool WeightedSum(void); virtual bool WeightedSumGradient(void); virtual bool calcReconstructGradient(void); public: CNeuronATFNetOCL(void) {}; ~CNeuronATFNetOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint history, uint forecast, uint variables, uint heads, uint layers, uint &patch[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronATFNetOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); virtual CBufferFloat *getWeights(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint history, uint forecast, uint variables, uint heads, uint layers, uint & patch[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- iHistory = MathMax(history, 1); iForecast = forecast; iVariables = variables; //--- Calculate FFTsize uint size = iHistory + iForecast; int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) < size) power++; iFFT = uint(MathPow(2, power)); //--- T-Block if(!cNorm.Init(0, 0, OpenCL, iHistory * iVariables, batch, optimization)) ReturnFalse; if(!cPositionEncoder.Init(0, 1, OpenCL, iVariables, iHistory, optimization, batch)) ReturnFalse; if(!cTranspose.Init(0, 2, OpenCL, iHistory, iVariables, optimization, batch)) ReturnFalse; cTranspose.SetActivationFunction(None); uint count = (iHistory - patch[0] + 2 * patch[1] - 1) / patch[1]; if(!cPatching.Init(0, 3, OpenCL, patch[0], patch[1], patch[2], count, iVariables, optimization, batch)) ReturnFalse; caAttention.SetOpenCL(OpenCL); for(uint l = 0; l < layers; l++) { CNeuronConformer *temp = new CNeuronConformer(); if(!temp) ReturnFalse; if(!temp.Init(0, 4 + l, OpenCL, patch[2], 32, heads, iVariables, count, optimization, batch)) { DeleteObj(temp); ReturnFalse; } if(!caAttention.Add(temp)) { DeleteObj(temp); ReturnFalse; } } int total = 3; caProjection.SetOpenCL(OpenCL); uint window = patch[2] * count; for(int l = 0; l < total; l++) { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 4 + layers + l, OpenCL, window, window, (total - l)*iForecast, iVariables, optimization, batch)) { DeleteObj(temp); ReturnFalse; } temp.SetActivationFunction(TANH); if(!caProjection.Add(temp)) { DeleteObj(temp); ReturnFalse; } window = (total - l) * iForecast; } if(!cRevIN.Init(0, 4 + layers + total, OpenCL, iForecast * iVariables, 1, cNorm.AsObject())) ReturnFalse; //--- F-Block if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 0, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMainFreqWeights.BufferInit(iVariables, 0) || !cMainFreqWeights.BufferCreate(OpenCL)) ReturnFalse; if(!cNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cFreqAtteention.Init(0, 2, OpenCL, iFFT, 32, heads, iVariables, layers, optimization, batch)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasReGrad.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasReGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cReconstructInput.BufferInit(iHistory * iVariables, 0) || !cReconstructInput.BufferCreate(OpenCL)) ReturnFalse; if(!cForecast.BufferInit(iForecast * iVariables, 0) || !cForecast.BufferCreate(OpenCL)) ReturnFalse; if(!cReconstructInputGrad.BufferInit(iHistory * iVariables, 0) || !cReconstructInputGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cForecastGrad.BufferInit(iForecast * iVariables, 0) || !cForecastGrad.BufferCreate(OpenCL)) ReturnFalse; if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexUnNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalize, def_k_cn_inputs, cFreqAtteention.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_outputs, cUnNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::ComplexUnNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_inputs_gr, cFreqAtteention.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_outputs_gr, cUnNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalizeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::MainFreqWeights(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_MainFreqWeight, def_k_mfw_freq, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_MainFreqWeight, def_k_mfw_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_MainFreqWeight, def_k_mfw_dimension, (int)(iFFT)) kernelExecute(def_k_MainFreqWeight, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::WeightedSum(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_WeightedSum, def_k_ws_inputs1, cForecast.GetIndex()) setBuffer(def_k_WeightedSum, def_k_ws_inputs2, cRevIN.getOutputIndex()) setBuffer(def_k_WeightedSum, def_k_ws_outputs, getOutputIndex()) setBuffer(def_k_WeightedSum, def_k_ws_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_WeightedSum, def_k_ws_dimension, (int)(iForecast)) kernelExecute(def_k_WeightedSum, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::WeightedSumGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_WeightedSumGradient, def_k_ws_inputs1, cForecastGrad.GetIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_inputs2, cRevIN.getGradientIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_outputs, getGradientIndex()) setBuffer(def_k_WeightedSumGradient, def_k_ws_weight, cMainFreqWeights.GetIndex()) setArgument(def_k_WeightedSumGradient, def_k_ws_dimension, (int)(iForecast)) kernelExecute(def_k_WeightedSumGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::calcReconstructGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = iHistory * iVariables; setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_t, cInputs.GetIndex()) setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_o, cReconstructInput.GetIndex()) setBuffer(def_k_CalcOutputGradient, def_k_cog_matrix_ig, cReconstructInputGrad.GetIndex()) setArgument(def_k_CalcOutputGradient, def_k_cog_activation, (int)None) setArgument(def_k_CalcOutputGradient, def_k_cog_error, 1) ResetLastError(); kernelExecute(def_k_CalcOutputGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; if(cInputs != NeuronOCL.getOutput()) cInputs = NeuronOCL.getOutput(); //--- T-Block if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse;; if(!cPositionEncoder.FeedForward(cNorm.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cPositionEncoder.AsObject())) ReturnFalse; if(!cPatching.FeedForward(cTranspose.AsObject())) ReturnFalse; int total = caAttention.Total(); CNeuronBaseOCL *prev = cPatching.AsObject(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *att = caAttention.At(i); if(!att.FeedForward(prev)) ReturnFalse; prev = att; } total = caProjection.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *proj = caProjection.At(i); if(!proj.FeedForward(prev)) ReturnFalse; prev = proj; } if(!cRevIN.FeedForward(prev)) ReturnFalse; //--- F-Block if(!FFT(cInputs, cInputs, GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexNormalize()) ReturnFalse; if(!MainFreqWeights()) ReturnFalse; if(!cFreqAtteention.FeedForward(cNormFreqComplex.AsObject())) ReturnFalse; if(!ComplexUnNormalize()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cOutputTimeSeriasRe), GetPointer(cOutputTimeSeriasIm), true)) ReturnFalse; if(!DeConcat(GetPointer(cReconstructInput), GetPointer(cForecast), GetPointer(cOutputTimeSeriasReGrad), GetPointer(cOutputTimeSeriasRe), iHistory, iForecast, iFFT - iHistory - iForecast, iVariables)) ReturnFalse; //--- Output if(!WeightedSum()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getGradient() || !cInputs) ReturnFalse; //--- Output if(!WeightedSumGradient()) ReturnFalse; //--- T-Block if(cRevIN.Activation() != None && !DeActivation(cRevIN.getOutput(), cRevIN.getGradient(), cRevIN.getGradient(), cRevIN.Activation())) ReturnFalse; CNeuronBaseOCL *next = cRevIN.AsObject(); for(int i = caProjection.Total() - 1; i >= 0; i--) { CNeuronBaseOCL *proj = caProjection.At(i); if(!proj || !proj.CalcHiddenGradients((CObject *)next)) ReturnFalse; next = proj; } for(int i = caAttention.Total() - 1; i >= 0; i--) { CNeuronBaseOCL *att = caAttention.At(i); if(!att || !att.CalcHiddenGradients((CObject *)next)) ReturnFalse; next = att; } if(!cPatching.CalcHiddenGradients((CObject*)next)) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; if(!cPositionEncoder.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cPositionEncoder.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- F-Block if(!CNeuronBaseOCL::SumAndNormilize(GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), iFFT * iVariables, false, 0, 0, 0, -0.5)) ReturnFalse; if(!calcReconstructGradient()) ReturnFalse; if(!Concat(GetPointer(cReconstructInputGrad), GetPointer(cForecastGrad), GetPointer(cZero), GetPointer(cOutputTimeSeriasReGrad), iHistory, iForecast, iFFT - iHistory - iForecast, iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputTimeSeriasReGrad), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexUnNormalizeGradient()) ReturnFalse; if(!cNormFreqComplex.CalcHiddenGradients(cFreqAtteention.AsObject())) ReturnFalse; if(!ComplexNormalizeGradient()) ReturnFalse; if(!DeConcat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), GetPointer(cOutputTimeSeriasRe), GetPointer(cOutputTimeSeriasIm), false)) ReturnFalse; if(!DeConcat(GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasRe), iHistory, iFFT - iHistory, iVariables)) ReturnFalse; //--- if(!CNeuronBaseOCL::SumAndNormilize(NeuronOCL.getGradient(), GetPointer(cInputFreqRe), NeuronOCL.getGradient(), iHistory * iVariables, false, 0, 0, 0, 0.5)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- F-Block if(!cFreqAtteention.UpdateInputWeights(cNormFreqComplex.AsObject())) ReturnFalse; //--- T-Block if(!cPatching.UpdateInputWeights(cPositionEncoder.AsObject())) ReturnFalse; int total = caAttention.Total(); CNeuronBaseOCL *prev = cPatching.AsObject(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *att = caAttention.At(i); if(!att.UpdateInputWeights(prev)) ReturnFalse; prev = att; } total = caProjection.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *proj = caProjection.At(i); if(!proj.UpdateInputWeights(prev)) ReturnFalse; prev = proj; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iHistory) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iForecast) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iFFT) < INT_VALUE) ReturnFalse; //--- T-Block if(!cNorm.Save(file_handle)) ReturnFalse; if(!cPositionEncoder.Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!caAttention.Save(file_handle)) ReturnFalse; if(!caProjection.Save(file_handle)) ReturnFalse; //--- F-Block if(!cFreqAtteention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronATFNetOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFT = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; //--- T-Block if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPositionEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!caAttention.Load(file_handle)) ReturnFalse; if(!caProjection.Load(file_handle)) ReturnFalse; if(!cTranspose.Init(0, 2, OpenCL, iHistory, iVariables, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); if(!cRevIN.Init(0, 4 + caAttention.Total() + caProjection.Total(), OpenCL, iForecast * iVariables, 1, cNorm.AsObject())) ReturnFalse; //--- F-Block if(!LoadInsideLayer(file_handle, cFreqAtteention.AsObject())) ReturnFalse; cInputFreqRe.BufferFree(); if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cInputFreqIm.BufferFree(); if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 0, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMainFreqWeights.BufferFree(); if(!cMainFreqWeights.BufferInit(iVariables, 0) || !cMainFreqWeights.BufferCreate(OpenCL)) ReturnFalse; if(!cNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMeans.BufferFree(); if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; cVariances.BufferFree(); if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 1, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cOutputFreqRe.BufferFree(); if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cOutputFreqIm.BufferFree(); if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasRe.BufferCreate(OpenCL)) ReturnFalse; cOutputTimeSeriasIm.BufferFree(); if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; cOutputTimeSeriasReGrad.BufferFree(); if(!cOutputTimeSeriasReGrad.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasReGrad.BufferCreate(OpenCL)) ReturnFalse; cReconstructInput.BufferFree(); if(!cReconstructInput.BufferInit(iHistory * iVariables, 0) || !cReconstructInput.BufferCreate(OpenCL)) ReturnFalse; cForecast.BufferFree(); if(!cForecast.BufferInit(iForecast * iVariables, 0) || !cForecast.BufferCreate(OpenCL)) ReturnFalse; cReconstructInputGrad.BufferFree(); if(!cReconstructInputGrad.BufferInit(iHistory * iVariables, 0) || !cReconstructInputGrad.BufferCreate(OpenCL)) ReturnFalse; cForecastGrad.BufferFree(); if(!cForecastGrad.BufferInit(iForecast * iVariables, 0) || !cForecastGrad.BufferCreate(OpenCL)) ReturnFalse; cZero.BufferFree(); if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronATFNetOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- T-Block cNorm.SetOpenCL(OpenCL); cPositionEncoder.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cPatching.SetOpenCL(OpenCL); for(int l = 0; l < caAttention.Total(); l++) { CNeuronBaseOCL *temp = caAttention.At(l); if(!temp) continue; temp.SetOpenCL(OpenCL); } for(int l = 0; l < caProjection.Total(); l++) { CNeuronBaseOCL *temp = caProjection.At(l); if(!temp) continue; temp.SetOpenCL(OpenCL); } cRevIN.SetOpenCL(OpenCL); //--- F-Block cInputFreqRe.BufferCreate(OpenCL); cInputFreqIm.BufferCreate(OpenCL); cInputFreqComplex.SetOpenCL(OpenCL); cMainFreqWeights.BufferCreate(OpenCL); cNormFreqComplex.SetOpenCL(OpenCL); cMeans.BufferCreate(OpenCL); cVariances.BufferCreate(OpenCL); cFreqAtteention.SetOpenCL(OpenCL); cUnNormFreqComplex.SetOpenCL(OpenCL); cOutputFreqRe.BufferCreate(OpenCL); cOutputFreqIm.BufferCreate(OpenCL); cOutputTimeSeriasRe.BufferCreate(OpenCL); cOutputTimeSeriasIm.BufferCreate(OpenCL); cOutputTimeSeriasReGrad.BufferCreate(OpenCL); cReconstructInput.BufferCreate(OpenCL); cForecast.BufferCreate(OpenCL); cReconstructInputGrad.BufferCreate(OpenCL); cForecastGrad.BufferCreate(OpenCL); cZero.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronATFNetOCL::getWeights(void) { CNeuronConvOCL *conv = caProjection.At(caProjection.Total() - 1); return conv.GetWeightsConv(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronS3 : public CNeuronBaseOCL { protected: uint iWindow; uint iSegments; //--- CNeuronBaseOCL cOne; CNeuronConvOCL cShufle; CNeuronSoftMaxOCL cProbability; CNeuronConvOCL cWeights; CBufferFloat cPositions; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardS3(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradientsS3(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronS3(void) {}; ~CNeuronS3(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronS3; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; //--- iWindow = MathMax(window, 1); iSegments = (numNeurons + window - 1) / window; //--- if(!cOne.Init(0, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buffer = cOne.getOutput(); if(!buffer || !buffer.BufferInit(buffer.Total(), 1)) ReturnFalse; if(!buffer.BufferWrite()) ReturnFalse; //--- if(!cShufle.Init(0, 1, OpenCL, 1, 1, iSegments, 1, optimization, iBatch)) ReturnFalse; cShufle.SetActivationFunction(None); if(!cProbability.Init(0, 2, OpenCL, iSegments, optimization, iBatch)) ReturnFalse; cProbability.SetActivationFunction(None); cProbability.SetHeads(1); if(!cWeights.Init(0, 3, OpenCL, 1, 1, 2, 1, optimization, iBatch)) ReturnFalse; cWeights.SetActivationFunction(SIGMOID); if(!cPositions.BufferInit(iSegments, 0) || !cPositions.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::feedForwardS3(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSegments}; setBuffer(def_k_FeedForwardS3, def_k_s3_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_outputs, getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_positions, cPositions.GetIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_probability, cProbability.getOutputIndex()) setBuffer(def_k_FeedForwardS3, def_k_s3_weights, cWeights.getOutputIndex()) setArgument(def_k_FeedForwardS3, def_k_s3_window, (int)(iWindow)) setArgument(def_k_FeedForwardS3, def_k_s3_total, (int)(Neurons())) kernelExecute(def_k_FeedForwardS3, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::calcInputGradientsS3(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSegments}; setBuffer(def_k_InsideGradientS3, def_k_s3g_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_outputs_gr, getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_positions, cPositions.GetIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_probability, cProbability.getOutputIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_probability_gr, cProbability.getGradientIndex()) setBuffer(def_k_InsideGradientS3, def_k_s3g_weights, cWeights.getOutputIndex()) setArgument(def_k_InsideGradientS3, def_k_s3g_window, (int)(iWindow)) setArgument(def_k_InsideGradientS3, def_k_s3g_total, (int)(Neurons())) kernelExecute(def_k_InsideGradientS3, global_work_offset, global_work_size) } //--- { uint global_work_offset[2] = {0, 0}; uint dimension = MathMin(Neurons(), 64); uint global_work_size[2] = {dimension, 2}; uint local_work_size[2] = {dimension, 1}; setBuffer(def_k_WeightGradientS3, def_k_s3w_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_outputs_gr, getGradientIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_positions, cPositions.GetIndex()) setBuffer(def_k_WeightGradientS3, def_k_s3w_weights_gr, cWeights.getGradientIndex()) setArgument(def_k_WeightGradientS3, def_k_s3w_window, (int)(iWindow)) setArgument(def_k_WeightGradientS3, def_k_s3w_total, (int)(Neurons())) kernelExecuteLoc(def_k_WeightGradientS3, global_work_offset, global_work_size, local_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::feedForward(CNeuronBaseOCL * NeuronOCL) { if(bTrain) { if(!cWeights.FeedForward(cOne.AsObject())) ReturnFalse; if(!cShufle.FeedForward(cOne.AsObject())) ReturnFalse; if(!cProbability.FeedForward(cShufle.AsObject())) ReturnFalse; } if(!feedForwardS3(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!calcInputGradientsS3(NeuronOCL)) ReturnFalse; if(!cShufle.CalcHiddenGradients(cProbability.AsObject())) ReturnFalse; if(cWeights.Activation() != None) if(!DeActivation(cWeights.getOutput(), cWeights.getGradient(), cWeights.getGradient(), cWeights.Activation())) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cWeights.UpdateInputWeights(cOne.AsObject())) ReturnFalse; if(!cShufle.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cShufle.Save(file_handle)) ReturnFalse; if(!cWeights.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow), INT_VALUE) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSegments), INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cShufle.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWeights.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iSegments = uint(FileReadInteger(file_handle)); //--- if(cProbability.Neurons() != iSegments) { if(!cProbability.Init(0, 2, OpenCL, iSegments, optimization, iBatch)) ReturnFalse; cProbability.SetActivationFunction(None); } if(!cProbability.FeedForward(cShufle.AsObject())) ReturnFalse; if(cPositions.Total() != iSegments) { cPositions.BufferFree(); if(!cPositions.BufferInit(iSegments, 0) || !cPositions.BufferCreate(OpenCL)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronS3::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cShufle.SetOpenCL(OpenCL); cProbability.SetOpenCL(OpenCL); cWeights.SetOpenCL(OpenCL); cPositions.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronS3::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronS3 *Source = source; if(!cWeights.WeightsUpdate(Source.cWeights.AsObject(), tau)) ReturnFalse; if(!cShufle.WeightsUpdate(Source.cShufle.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMLMHAttentionMLKV : public CNeuronMLMHAttentionOCL { protected: uint iLayersToOneKV; ///< Number of inner layers uint iHeadsKV; ///< Number of heads CCollection KV_Tensors; ///< The collection of tensors of Keys and Values CCollection KV_Weights; ///< The collection of Matrix of weights to previous layer CBufferFloat Temp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out); virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. public: CNeuronMLMHAttentionMLKV(void) {}; ~CNeuronMLMHAttentionMLKV(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMLMHAttentionMLKV; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindow * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize KV weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, kv, iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, CBufferFloat * scores, CBufferFloat * outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), kv_g, inp, GetPointer(Temp), iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindow, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); KV_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle) || !KV_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronMLMHAttentionOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMLMHAttentionMLKV *temp = source; if(iLayers != temp.iLayers) ReturnFalse; for(uint l = 0; l < ((iLayers + iLayersToOneKV - 1) / iLayersToOneKV); l++) { if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l * (optimization == SGD ? 2 : 3)), temp.KV_Weights.At(l * (temp.optimization == SGD ? 2 : 3)), (optimization == SGD ? KV_Weights.At(l * 2 + 1) : KV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMLCrossAttentionMLKV : public CNeuronMLMHAttentionMLKV { protected: uint iWindowKV; uint iUnitsKV; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override; virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } public: CNeuronMLCrossAttentionMLKV(void) {}; ~CNeuronMLCrossAttentionMLKV(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMLCrossAttentionMLKV; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual uint GetSecondBufferSize(void) { return (iWindowKV * iUnitsKV); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = fmax(window_kv, 1); iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnitsKV; //Size of KV tensor uint q_weights = (iWindow + 1) * iWindowKey * iHeads; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindowKV + 1) * iWindowKey * iHeadsKV; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnitsKV * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of our tensore uint w0 = (iWindowKey + 1) * iHeads * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : 2 * iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!NeuronOCL || !Context) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, kv, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, CBufferFloat * scores, CBufferFloat * outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, SecondActivation)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(i > 0) out_grad = temp; } //--- if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(0), KV_Tensors.At(1), SecondInput, SecondGradient, iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; for(int i = 1; i < KV_Tensors.Total() / 2; i++) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i * (optimization == SGD ? 2 : 3)), KV_Tensors.At(i * 2 + 1), SecondInput, GetPointer(Temp), iWindowKV, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle) || !KV_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL) || !KV_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindowKV, 2 * iWindowKey * iHeadsKV, 0, 2 * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, iWindow)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCSCMOCL : public CNeuronBaseOCL { protected: uint i_Count; uint i_Variables; bool b_NeedTranspose; //--- CArrayInt ia_Windows; CArrayObj caTranspose; CArrayObj caConvolutions; CArrayObj caMLP; CArrayObj caTemp; CArrayObj caConvOutputs; CArrayObj caConvGradients; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronCSCMOCL(void) {}; ~CNeuronCSCMOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronCSCMOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint variables, uint inputs_count, bool need_transpose, ENUM_OPTIMIZATION optimization_type, uint batch) { const uint layers = windows.Size(); if(layers <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs_count * variables, optimization_type, batch)) ReturnFalse; //--- ia_Windows.Clear(); ia_Windows.Reserve(layers); for(uint i = 0; i < layers; i++) if(!ia_Windows.Add((int)windows[i])) ReturnFalse; i_Variables = variables; i_Count = inputs_count / ia_Windows[0]; b_NeedTranspose = need_transpose; //--- if(b_NeedTranspose) { CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, 0, OpenCL, inputs_count, i_Variables, optimization, iBatch)) { DeleteObj(transp); ReturnFalse; } if(!caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, 1, OpenCL, i_Variables, inputs_count, optimization, iBatch)) { DeleteObj(transp); ReturnFalse; } if(!caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } if(!SetOutput(transp.getOutput()) || !SetGradient(transp.getGradient()) ) ReturnFalse; } //--- uint total = ia_Windows[0] * i_Count; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, 0, OpenCL, inputs_count, inputs_count, total, 1, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SIGMOID); if(!caConvolutions.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- total = 0; for(uint i = 0; i < layers; i++) { conv = new CNeuronConvOCL(); if(!conv.Init(0, i + 1, OpenCL, ia_Windows[i], ia_Windows[i], (i < (layers - 1) ? ia_Windows[i + 1] : 1), i_Count, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SIGMOID); if(!caConvolutions.Add(conv)) { DeleteObj(conv); ReturnFalse; } if(!caConvOutputs.Add(conv.getOutput()) || !caConvGradients.Add(conv.getGradient()) ) ReturnFalse; total += conv.Neurons(); } //--- CNeuronBaseOCL *comul = new CNeuronBaseOCL(); if(!comul.Init(0, 0, OpenCL, total, optimization, iBatch)) { DeleteObj(comul); ReturnFalse; } if(!caMLP.Add(comul)) { DeleteObj(comul); ReturnFalse; } if(layers == 1) { comul.SetOutput(conv.getOutput()); comul.SetGradient(conv.getGradient()); } conv = new CNeuronConvOCL(); if(!conv.Init(0, 0, OpenCL, total / i_Variables, total / i_Variables, inputs_count, 1, i_Variables, optimization, iBatch)) { DeleteObj(conv); ReturnFalse; } if(!caMLP.Add(conv)) { DeleteObj(conv); ReturnFalse; } if(!b_NeedTranspose) { if(!SetOutput(conv.getOutput()) || !SetGradient(conv.getGradient()) ) ReturnFalse; } //--- CBufferFloat *buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } buf = new CBufferFloat(); if(!buf) ReturnFalse; if(!buf.BufferInit(total, 0) || !buf.BufferCreate(OpenCL) || !caTemp.Add(buf)) { DeleteObj(buf); ReturnFalse; } //--- caConvOutputs.FreeMode(false); caConvGradients.FreeMode(false); caTranspose.FreeMode(true); caConvolutions.FreeMode(true); caMLP.FreeMode(true); caTemp.FreeMode(true); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- if(b_NeedTranspose) { current = caTranspose.At(0); if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- int layers = caConvolutions.Total() - 1; for(int l = 0; l <= layers; l++) { current = caConvolutions.At(l); if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- current = caMLP.At(0); if(!current) ReturnFalse; switch(layers) { case 0: ReturnFalse; case 1: break; case 2: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), current.getOutput(), ia_Windows[1], 1, i_Variables * i_Count)) ReturnFalse; break; case 3: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), current.getOutput(), ia_Windows[1], ia_Windows[2], 1, i_Variables * i_Count)) ReturnFalse; break; case 4: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), caConvOutputs.At(3), current.getOutput(), ia_Windows[1], ia_Windows[2], ia_Windows[3], 1, i_Variables * i_Count)) ReturnFalse; break; default: if(!Concat(caConvOutputs.At(0), caConvOutputs.At(1), caConvOutputs.At(2), caConvOutputs.At(3), caTemp.At(0), ia_Windows[1], ia_Windows[2], ia_Windows[3], ia_Windows[4], i_Variables * i_Count)) ReturnFalse; break; } uint last_buf = 0; for(int i = 4; i < layers; i += 3) { uint buf_size = 0; for(int j = 1; j <= i; j++) buf_size += ia_Windows[j]; switch(layers - i) { case 1: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), current.getOutput(), buf_size, 1, i_Variables * i_Count)) ReturnFalse; break; case 2: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), current.getOutput(), buf_size, ia_Windows[i + 1], 1, i_Variables * i_Count)) ReturnFalse; break; case 3: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), caConvOutputs.At(i + 2), current.getOutput(), buf_size, ia_Windows[i + 1], ia_Windows[i + 2], 1, i_Variables * i_Count)) ReturnFalse; break; default: if(!Concat(caTemp.At(last_buf), caConvOutputs.At(i), caConvOutputs.At(i + 1), caConvOutputs.At(i + 2), caTemp.At((last_buf + 1) % 2), buf_size, ia_Windows[i + 1], ia_Windows[i + 2], ia_Windows[i + 3], i_Variables * i_Count)) ReturnFalse; break; } last_buf = (last_buf + 1) % 2; } //--- inp = current; current = caMLP.At(1); if(!current || !current.FeedForward(inp)) ReturnFalse; //--- if(b_NeedTranspose) { inp = current; current = caTranspose.At(1); if(!current || !current.FeedForward(inp)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *current = caMLP.At(0); CNeuronBaseOCL *next = caMLP.At(1); if(b_NeedTranspose) { if(!next.CalcHiddenGradients(caTranspose.At(1))) ReturnFalse; } if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; //--- int layers = caConvGradients.Total(); if(layers == 1) { next = caConvolutions.At(1); if(next.Activation() != None) { if(!DeActivation(next.getOutput(), next.getGradient(), next.getGradient(), next.Activation())) ReturnFalse; } } else { int prev_window = 0; for(int i = 1; i < layers; i++) prev_window += int(ia_Windows[i]); if(!DeConcat(caTemp.At(0), caConvGradients.At(layers - 1), next.getGradient(), prev_window, 1, i_Variables * i_Count)) ReturnFalse; next = caConvolutions.At(layers); int current_buf = 0; for(int l = layers; l > 1; l--) { current = caConvolutions.At(l - 1); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; int window = int(ia_Windows[l - 1]); prev_window -= window; if(!DeConcat(caTemp.At((current_buf + 1) % 2), caTemp.At(2), caTemp.At(current_buf), prev_window, window, i_Variables * i_Count)) ReturnFalse; if(current.Activation() != None) { if(!DeActivation(current.getOutput(), caTemp.At(2), caTemp.At(2), current.Activation())) ReturnFalse; } if(!SumAndNormilize(current.getGradient(), caTemp.At(2), current.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; next = current; current_buf = (current_buf + 1) % 2; } } current = caConvolutions.At(0); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; //--- if(b_NeedTranspose) { current = caTranspose.At(0); if(!current.CalcHiddenGradients(next.AsObject())) ReturnFalse; next = current; } //--- if(!prevLayer.CalcHiddenGradients(next.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CObject *prev = (b_NeedTranspose ? caTranspose.At(0) : NeuronOCL); CNeuronBaseOCL *current = NULL; for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(1); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current; } current = caMLP.At(1); if(!current || !current.UpdateInputWeights(caMLP.At(0)) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(i_Variables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(i_Count)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(b_NeedTranspose)) < INT_VALUE) ReturnFalse; //--- Objects if(!ia_Windows.Save(file_handle)) ReturnFalse; if(b_NeedTranspose) { for(int i = 0; i < 2; i++) if(!caTranspose.At(i).Save(file_handle)) ReturnFalse; } int layers = caConvolutions.Total(); if(FileWriteInteger(file_handle, layers) < INT_VALUE) ReturnFalse; for(int i = 0; i < layers; i++) { if(!caConvolutions.At(i).Save(file_handle)) ReturnFalse; } for(int i = 0; i < 2; i++) if(!caMLP.At(i).Save(file_handle)) ReturnFalse; for(int i = 0; i < 3; i++) if(!caTemp.At(i).Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::Load(const int file_handle) { CBufferFloat *temp = new CBufferFloat(); if(!temp) ReturnFalse; if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) || !SetOutput(temp, false)) { DeleteObj(temp); ReturnFalse; } temp = new CBufferFloat(); if(!temp) ReturnFalse; if(!temp.BufferInit(Neurons(), 0) || !temp.BufferCreate(OpenCL) || !SetGradient(temp, false)) { DeleteObj(temp); ReturnFalse; } //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; i_Variables = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; i_Count = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; b_NeedTranspose = bool(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; //--- Objects if(!ia_Windows.Load(file_handle)) ReturnFalse; if(b_NeedTranspose) { caTranspose.Clear(); CNeuronTransposeOCL *transp = NULL; for(int i = 0; i < 2; i++) { transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, i, OpenCL, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, transp) || !caTranspose.Add(transp)) { DeleteObj(transp); ReturnFalse; } } } int layers = FileReadInteger(file_handle); caConvolutions.Clear(); caConvOutputs.Clear(); caConvGradients.Clear(); CNeuronConvOCL *conv = NULL; for(int i = 0; i < layers; i++) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, i, OpenCL, 1, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caConvolutions.Add(conv) ) { DeleteObj(conv); ReturnFalse; } if(i > 0) { if(!caConvOutputs.Add(conv.getOutput()) || !caConvGradients.Add(conv.getGradient()) ) ReturnFalse; } } for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, caMLP.At(i))) ReturnFalse; for(int i = 0; i < 3; i++) if(!caTemp.At(i).Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = (b_NeedTranspose ? caTranspose.At(1) : caMLP.At(1)); if(!SetOutput(neuron.getOutput()) || !SetGradient(neuron.getGradient()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCSCMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); CNeuronBaseOCL *current = NULL; if(b_NeedTranspose) for(int i = 0; i < 2; i++) { current = caTranspose.At(i); current.SetOpenCL(OpenCL); } //--- for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(i); current.SetOpenCL(OpenCL); } //--- for(int i = 0; i < 2; i++) { current = caMLP.At(i); current.SetOpenCL(OpenCL); } for(int i = 0; i < 3; i++) { CBufferFloat *buf = caTemp.At(i); buf.BufferCreate(OpenCL); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCSCMOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronCSCMOCL *Source = source; if(caConvolutions.Total() != Source.caConvolutions.Total()) ReturnFalse; CNeuronBaseOCL *current = NULL; for(int i = 0; i < caConvolutions.Total(); i++) { current = caConvolutions.At(i); if(!current || !current.WeightsUpdate(Source.caConvolutions.At(i), tau) ) ReturnFalse; } current = caMLP.At(1); if(!current || !current.WeightsUpdate(Source.caMLP.At(1), tau) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSPyrAttentionOCL : public CNeuronBaseOCL { protected: uint iWindowIn; uint iWindowKey; uint iHeads; uint iHeadsKV; uint iCount; uint iPAMLayers; //--- CArrayObj caS3; CArrayObj caQuery; CArrayObj caKV; CArrayInt caScore; CArrayObj caAttentionOut; CArrayObj caW0; CNeuronConvOCL cFF1; CNeuronConvOCL cFF2; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, int scores, CBufferFloat *out, int window); virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, int scores, CBufferFloat *gradient); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); ///< Method to transfer gradients to previous layer @param[in] prevLayer Pointer to previous layer. virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. virtual void ArraySetOpenCL(CArrayObj *array, COpenCLMy *obj); public: CNeuronSPyrAttentionOCL(void) {}; ~CNeuronSPyrAttentionOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_key, uint heads, uint heads_kv, uint units_count, uint pam_layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSPyrAttentionMLKV; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_key, uint heads, uint heads_kv, uint units_count, uint pam_layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_in * units_count, optimization_type, batch)) ReturnFalse; //--- iWindowIn = window_in; iWindowKey = MathMax(window_key, 1); iHeads = MathMax(heads, 1); iHeadsKV = MathMax(heads_kv, 1); iCount = units_count; iPAMLayers = MathMax(pam_layers, 2); //--- caS3.Clear(); caQuery.Clear(); caKV.Clear(); caScore.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3) ReturnFalse; if(!s3.Init(0, l, OpenCL, iWindowIn, iCount, optimization, iBatch) || !caS3.Add(s3)) ReturnFalse; s3.SetActivationFunction(None); //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, iWindowKey * iHeads, iCount, optimization, iBatch) || !caQuery.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- KV conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 2 * iWindowKey * iHeadsKV, iCount, optimization, iBatch) || !caKV.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- MH Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, 0, OpenCL, iWindowKey * iHeadsKV, iWindowKey * iHeadsKV, iWindowIn, iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!cFF1.Init(0, 0, OpenCL, iWindowIn, iWindowIn, 4 * iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF1.SetActivationFunction(LReLU); if(!cFF2.Init(0, 0, OpenCL, 4 * iWindowIn, 4 * iWindowIn, iWindowIn, iCount, optimization, iBatch)) ReturnFalse; cFF2.SetActivationFunction(None); if(!SetGradient(cFF2.getGradient())) ReturnFalse; //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionOut(CBufferFloat * q, CBufferFloat * kv, int scores, CBufferFloat * out, int window) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount/*Q units*/, iCount/*K units*/, iHeads}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_q, q.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_kv, kv.GetIndex()) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_score, scores) setBuffer(def_k_MH2PyrAttentionOut, def_k_pam_out, out.GetIndex()) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_dimension, (int)iWindowKey) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2PyrAttentionOut, def_k_pam_window, window) kernelExecuteLoc(def_k_MH2PyrAttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * q_g, CBufferFloat * kv, CBufferFloat * kv_g, int scores, CBufferFloat * gradient) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iWindowKey, iHeads}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, q_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kv_g.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, gradient.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iCount) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)iHeadsKV) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; CBufferFloat *q = NULL; CBufferFloat *kv = NULL; //--- for(uint l = 0; l < iPAMLayers; l++) { //--- Mix current = caS3.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; //--- Query current = caQuery.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; q = current.getOutput(); //--- Key and Value current = caKV.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; kv = current.getOutput(); //--- PAM current = caAttentionOut.At(l); if(!current || !AttentionOut(q, kv, caScore.At(l), current.getOutput(), iPAMLayers - l)) ReturnFalse; prev = current; //--- W0 current = caW0.At(l); if(!current || !current.FeedForward(prev.AsObject()) ) ReturnFalse; prev = current; } //--- Residual current = caW0.At(iPAMLayers); if(!SumAndNormilize(NeuronOCL.getOutput(), prev.getOutput(), current.getOutput(), iWindowIn, true)) ReturnFalse; //---FeedForward if(!cFF1.FeedForward(current.AsObject()) || !cFF2.FeedForward(cFF1.AsObject()) ) ReturnFalse; //--- Residual if(!SumAndNormilize(current.getOutput(), cFF2.getOutput(), getOutput(), iWindowIn, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; CNeuronBaseOCL *q = NULL; CNeuronBaseOCL *kv = NULL; //--- FeedForward current = caW0.At(iPAMLayers); if(!current || !cFF1.CalcHiddenGradients(cFF2.AsObject()) || !current.CalcHiddenGradients(cFF1.AsObject()) ) ReturnFalse; next = current; //--- Residual current = caW0.At(iPAMLayers - 1); if(!SumAndNormilize(getGradient(), next.getGradient(), current.getGradient(), iWindowIn, false)) ReturnFalse; CBufferFloat *residual = next.getGradient(); next = current; //--- for(int l = int(iPAMLayers - 1); l >= 0; l--) { //--- W0 current = caAttentionOut.At(l); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; //--- MH Attention q = caQuery.At(l); kv = caKV.At(l); if(!q || !kv || !AttentionInsideGradients(q.getOutput(), q.getGradient(), kv.getOutput(), kv.getGradient(), caScore.At(l), current.getGradient()) ) ReturnFalse; //--- Query current = caS3.At(l); if(!current || !current.CalcHiddenGradients(q.AsObject()) || !Concat(current.getGradient(), current.getGradient(), residual, iWindowIn, 0, iCount) ) ReturnFalse; //--- Key and Value if(!current || !current.CalcHiddenGradients(kv.AsObject()) || !SumAndNormilize(current.getGradient(), residual, current.getGradient(), iWindowIn, false) ) ReturnFalse; next = current; //--- S3 current = (l == 0 ? prevLayer : caW0.At(l - 1)); if(!current || !current.CalcHiddenGradients(next.AsObject()) ) ReturnFalse; next = current; } //--- Residual current = caW0.At(iPAMLayers - 1); if(!DeActivation(prevLayer.getOutput(), residual, current.getGradient(), prevLayer.Activation()) || !SumAndNormilize(prevLayer.getGradient(), residual, prevLayer.getGradient(), iWindowIn, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 current = caS3.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Query prev = current; current = caQuery.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- Key and Value current = caKV.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; //--- W0 prev = caAttentionOut.At(l); current = caW0.At(l); if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current; } //--- FeedForward prev = caW0.At(iPAMLayers); if(!cFF1.UpdateInputWeights(prev) || !cFF2.UpdateInputWeights(cFF1.AsObject()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::ArraySetOpenCL(CArrayObj * array, COpenCLMy * obj) { if(!array || array.Total() <= 0) return; //--- CNeuronBaseOCL *neuron = NULL; for(int l = 0; l < array.Total(); l++) { neuron = array.At(l); if(!neuron) continue; neuron.SetOpenCL(obj); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPyrAttentionOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); ArraySetOpenCL(GetPointer(caS3), OpenCL); ArraySetOpenCL(GetPointer(caQuery), OpenCL); ArraySetOpenCL(GetPointer(caKV), OpenCL); ArraySetOpenCL(GetPointer(caAttentionOut), OpenCL); ArraySetOpenCL(GetPointer(caW0), OpenCL); cFF1.SetOpenCL(OpenCL); cFF2.SetOpenCL(OpenCL); //--- Score for(uint l = 0; l < iPAMLayers; l++) { int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) return; if(!caScore.Add(temp)) return; } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSPyrAttentionOCL *Source = source; if(iPAMLayers != Source.iPAMLayers) ReturnFalse; CNeuronBaseOCL *s = NULL; CNeuronBaseOCL *c = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 c = caS3.At(l); s = Source.caS3.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Query c = caQuery.At(l); s = Source.caQuery.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- Key and Value c = caKV.At(l); s = Source.caKV.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; //--- W0 c = caW0.At(l); s = Source.caW0.At(l); if(!c || !c.WeightsUpdate(s, tau)) ReturnFalse; } //--- FeedForward if(!cFF1.WeightsUpdate(Source.cFF1.AsObject(), tau) || !cFF2.WeightsUpdate(Source.cFF2.AsObject(), tau) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iWindowIn)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeadsKV)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPAMLayers)) < INT_VALUE) ReturnFalse; //--- Save objects CNeuronBaseOCL *neuron = NULL; for(uint l = 0; l < iPAMLayers; l++) { neuron = caS3.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caQuery.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caKV.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; neuron = caW0.At(l); if(!neuron || !neuron.Save(file_handle)) ReturnFalse; } //--- if(!cFF1.Save(file_handle) || !cFF2.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPyrAttentionOCL::Load(const int file_handle) { //--- Clear Arrays if(!!OpenCL) { for(int l = 0; l < caScore.Total(); l++) OpenCL.BufferFree(caScore[l]); } caScore.Clear(); caS3.Clear(); caQuery.Clear(); caKV.Clear(); caAttentionOut.Clear(); caW0.Clear(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iWindowIn = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iHeadsKV = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iCount = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iPAMLayers = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; //--- Load objects CNeuronBaseOCL *base = NULL; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint l = 0; l < iPAMLayers; l++) { //--- S3 s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, l, OpenCL, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, s3) || !caS3.Add(s3) ) ReturnFalse; //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caQuery.Add(conv) ) ReturnFalse; //--- Key and Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caKV.Add(conv) ) ReturnFalse; //--- Score int temp = OpenCL.AddBuffer(sizeof(float) * iCount * iCount * iHeads, CL_MEM_READ_WRITE); if(temp < 0) ReturnFalse; if(!caScore.Add(temp)) ReturnFalse; //--- Attention Out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowKey * iHeadsKV * iCount, optimization, iBatch) || !caAttentionOut.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- W0 conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, l, OpenCL, 1, 1, 1, 1, optimization, iBatch) || !LoadInsideLayer(file_handle, conv) || !caW0.Add(conv) ) ReturnFalse; } //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, 0, OpenCL, iWindowIn * iCount, optimization, iBatch) || !caW0.Add(conv)) { DeleteObj(base); ReturnFalse; } base.SetActivationFunction(None); //--- FeedForward if(!LoadInsideLayer(file_handle, cFF1.AsObject()) || !LoadInsideLayer(file_handle, cFF2.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPLROCL : public CNeuronBaseOCL { protected: bool bTranspose; int icIsTTP; int iVariables; int iCount; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } public: CNeuronPLROCL(void) : bTranspose(false) {}; ~CNeuronPLROCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint units_count, bool transpose, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronPLROCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLROCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint units_count, bool transpose, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_in * units_count, optimization_type, batch)) ReturnFalse; //--- iVariables = (int)window_in; iCount = (int)units_count; bTranspose = transpose; //--- icIsTTP = OpenCL.AddBuffer(sizeof(int) * Neurons(), CL_MEM_READ_WRITE); if(icIsTTP < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLROCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iVariables}; uint local_work_size[2] = {iCount, 1}; ResetLastError(); setBuffer(def_k_PLR, def_k_plr_inputs, NeuronOCL.getOutputIndex()) setBuffer(def_k_PLR, def_k_plr_outputs, getOutputIndex()) setBuffer(def_k_PLR, def_k_plt_isttp, icIsTTP) setArgument(def_k_PLR, def_k_plr_transpose, (int)bTranspose) setArgument(def_k_PLR, def_k_plr_step, (float)0.2) //--- kernelExecuteLoc(def_k_PLR, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLROCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!OpenCL || !prevLayer || !prevLayer.getGradient()) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iVariables}; ResetLastError(); setBuffer(def_k_PLRGrad, def_k_plrg_inputs_gr, prevLayer.getGradientIndex()) setBuffer(def_k_PLRGrad, def_k_plrg_outputs, getOutputIndex()) setBuffer(def_k_PLRGrad, def_k_plrg_outputs_gr, getGradientIndex()) setArgument(def_k_PLRGrad, def_k_plr_transpose, (int)bTranspose) //--- kernelExecute(def_k_PLRGrad, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLROCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(bTranspose)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLROCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; bTranspose = (bool)FileReadInteger(file_handle); //--- OpenCL.BufferFree(icIsTTP); icIsTTP = OpenCL.AddBuffer(sizeof(int) * Neurons(), CL_MEM_READ_WRITE); if(icIsTTP < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPLROCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && icIsTTP >= 0) OpenCL.BufferFree(icIsTTP); //--- CNeuronBaseOCL::SetOpenCL(obj); icIsTTP = OpenCL.AddBuffer(sizeof(float) * Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTPMEncoder : public CNeuronLSTMOCL { protected: bool bTSinRow; //--- CNeuronCSCMOCL cFeatureExtraction; CNeuronBaseOCL cMemAndHidden; CNeuronConcatenate cConcatenated; CNeuronSoftMaxOCL cSoftMax; CNeuronBaseOCL cAttentionOut; CNeuronTransposeOCL cTranspose; CBufferFloat cTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTPMEncoder(void) {}; ~CNeuronTPMEncoder(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronTPMEncoder; } virtual void SetOpenCL(COpenCLMy *obj) override; //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronLSTMOCL::Init(numOutputs, myIndex, open_cl, hidden_size, optimization_type, batch)) ReturnFalse; if(!SetInputs(variables * lenth)) ReturnFalse; //--- uint windows[] = {variables, 6, 5, 4}; if(!cFeatureExtraction.Init(0, 0, OpenCL, windows, lenth, variables, ts_in_row, optimization, batch)) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 1, OpenCL, hidden_size * 2, optimization, batch)) ReturnFalse; if(!cConcatenated.Init(0, 2, OpenCL, variables * lenth, variables * lenth, hidden_size * 2, optimization, batch)) ReturnFalse; cConcatenated.SetActivationFunction(TANH); if(!cSoftMax.Init(0, 3, OpenCL, variables * lenth, optimization, batch)) ReturnFalse; cSoftMax.SetHeads(variables); if(!cAttentionOut.Init(0, 4, OpenCL, variables * lenth, optimization, batch)) ReturnFalse; //--- bTSinRow = ts_in_row; if(!bTSinRow) { if(!cTranspose.Init(0, 5, OpenCL, variables, lenth, optimization, iBatch)) ReturnFalse; } //--- if(!cTemp.BufferInit(variables * lenth, 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- FEATURE EXTRACTION if(!cFeatureExtraction.FeedForward(NeuronOCL)) ReturnFalse; //--- Memory and Hidden if(!Concat(m_iHiddenState, m_iMemory, m_iHiddenState, m_iMemory, cMemAndHidden.getOutputIndex(), 1, 1, 0, 0, Neurons())) ReturnFalse; //--- Attention if(!cConcatenated.FeedForward(cFeatureExtraction.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cSoftMax.FeedForward(cConcatenated.AsObject())) ReturnFalse; CBufferFloat *map = cSoftMax.getOutput(); if(!bTSinRow) { if(!cTranspose.FeedForward(cSoftMax.AsObject())) ReturnFalse; map = cTranspose.getOutput(); } //--- if(!ElementMult(cFeatureExtraction.getOutput(), map, cAttentionOut.getOutput())) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- LSTM if(!CNeuronLSTMOCL::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronLSTMOCL::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; //--- if(!ElementMultGrad(cFeatureExtraction.getOutput(), GetPointer(cTemp), (bTSinRow ? cSoftMax.getOutput() : cTranspose.getOutput()), (bTSinRow ? cSoftMax.getGradient() : cTranspose.getGradient()), cAttentionOut.getGradient(), NeuronOCL.Activation(), int(None))) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- if(bTSinRow) { if(!cSoftMax.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; } //--- if(!cConcatenated.CalcHiddenGradients((CObject*)cSoftMax.AsObject(), NULL) || !DeActivation(cConcatenated.getOutput(), cConcatenated.getGradient(), cConcatenated.getGradient(), cConcatenated.Activation())) ReturnFalse; if(!cFeatureExtraction.CalcHiddenGradients(cConcatenated.AsObject(), cMemAndHidden.getOutput(), cMemAndHidden.getGradient())) ReturnFalse; if(!DeActivation(cFeatureExtraction.getOutput(), GetPointer(cTemp), GetPointer(cTemp), NeuronOCL.Activation()) || !SumAndNormilize(cFeatureExtraction.getGradient(), GetPointer(cTemp), cFeatureExtraction.getGradient(), 1, false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cFeatureExtraction.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronLSTMOCL::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; if(!cFeatureExtraction.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConcatenated.UpdateInputWeights(cFeatureExtraction.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTPMEncoder::SetOpenCL(COpenCLMy * obj) { if(!OpenCL) cTemp.BufferFree(); //--- CNeuronLSTMOCL::SetOpenCL(obj); cFeatureExtraction.SetOpenCL(OpenCL); cMemAndHidden.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); if(!bTSinRow) cTranspose.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Save(const int file_handle) { if(!CNeuronLSTMOCL::Save(file_handle)) ReturnFalse; if(!cFeatureExtraction.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(bTSinRow)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPMEncoder::Load(const int file_handle) { if(!CNeuronLSTMOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureExtraction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSoftMax.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; bTSinRow = (bool)FileReadInteger(file_handle); //--- if(!cMemAndHidden.Init(0, 1, OpenCL, Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cAttentionOut.Init(0, 3, OpenCL, cSoftMax.Neurons(), optimization, iBatch)) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(cSoftMax.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTPM : public CNeuronLSTMOCL { protected: CNeuronTPMEncoder cEncoder; CNeuronPLROCL cFeatureExtraction; CNeuronBaseOCL cMemAndHidden; CNeuronConcatenate cConcatenated; CNeuronSoftMaxOCL cSoftMax; CNeuronBaseOCL cAttentionOut; CNeuronConcatenate cAttAndFeature; CBufferFloat cTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTPM(void) {}; ~CNeuronTPM(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronTPM; } virtual void SetOpenCL(COpenCLMy *obj); //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint lenth, uint hidden_size, bool ts_in_row, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronLSTMOCL::Init(numOutputs, myIndex, open_cl, hidden_size, optimization_type, batch)) ReturnFalse; if(!SetInputs(hidden_size)) ReturnFalse; //--- if(!cEncoder.Init(0, 0, OpenCL, variables, lenth, hidden_size, ts_in_row, optimization, iBatch)) ReturnFalse; //--- if(!cFeatureExtraction.Init(0, 1, OpenCL, variables, lenth, !ts_in_row, optimization, iBatch)) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 2, OpenCL, hidden_size * 2, optimization, iBatch)) ReturnFalse; if(!cConcatenated.Init(0, 3, OpenCL, hidden_size, hidden_size, hidden_size * 2, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(TANH); if(!cSoftMax.Init(0, 4, OpenCL, hidden_size, optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(1); if(!cAttentionOut.Init(0, 5, OpenCL, hidden_size, optimization, iBatch)) ReturnFalse; if(!cAttAndFeature.Init(0, 6, OpenCL, hidden_size, hidden_size, variables * lenth, optimization, iBatch)) ReturnFalse; //--- if(!cTemp.BufferInit(variables * lenth, 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Encoder if(!cEncoder.FeedForward(NeuronOCL)) ReturnFalse; //--- FEATURE EXTRACTION if(!cFeatureExtraction.FeedForward(NeuronOCL)) ReturnFalse; //--- Memory and Hidden if(!Concat(m_iHiddenState, m_iMemory, m_iHiddenState, m_iMemory, cMemAndHidden.getOutputIndex(), 1, 1, 0, 0, Neurons())) ReturnFalse; //--- Attention if(!cConcatenated.FeedForward(cEncoder.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cSoftMax.FeedForward(cConcatenated.AsObject())) ReturnFalse; //--- if(!ElementMult(cEncoder.getOutput(), cSoftMax.getOutput(), cAttentionOut.getOutput())) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- Attention and Features if(!cAttAndFeature.FeedForward(cAttentionOut.AsObject(), cFeatureExtraction.getOutput())) ReturnFalse; //--- LSTM if(!CNeuronLSTMOCL::feedForward(cAttAndFeature.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronLSTMOCL::calcInputGradients(cAttAndFeature.AsObject())) ReturnFalse; //--- if(!cAttentionOut.CalcHiddenGradients(cAttAndFeature.AsObject(), cFeatureExtraction.getOutput(), cFeatureExtraction.getGradient(), (ENUM_ACTIVATION)cFeatureExtraction.Activation() )) ReturnFalse; //--- if(!ElementMultGrad(cEncoder.getOutput(), cEncoder.getGradient(), cSoftMax.getOutput(), cSoftMax.getGradient(), cAttentionOut.getGradient(), cEncoder.Activation(), int(None))) { printf("Error of execution kernel %s: %d", __FUNCTION__, GetLastError()); ReturnFalse; } //--- if(!cConcatenated.CalcHiddenGradients(cSoftMax.AsObject(), NULL) || !DeActivation(cConcatenated.getOutput(), cConcatenated.getGradient(), cConcatenated.getGradient(), cConcatenated.Activation())) ReturnFalse; if(!cFeatureExtraction.CalcHiddenGradients(cConcatenated.AsObject(), cMemAndHidden.getOutput(), cMemAndHidden.getGradient())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cFeatureExtraction.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), GetPointer(cTemp), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cEncoder.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), GetPointer(cTemp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronLSTMOCL::updateInputWeights(cAttAndFeature.AsObject())) ReturnFalse; if(!cEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeatureExtraction.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConcatenated.UpdateInputWeights(cEncoder.AsObject(), cMemAndHidden.getOutput())) ReturnFalse; if(!cAttAndFeature.UpdateInputWeights(cAttentionOut.AsObject(), cFeatureExtraction.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTPM::SetOpenCL(COpenCLMy * obj) { if(!OpenCL) cTemp.BufferFree(); //--- CNeuronLSTMOCL::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); cFeatureExtraction.SetOpenCL(OpenCL); cMemAndHidden.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cAttAndFeature.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::Save(const int file_handle) { if(!CNeuronLSTMOCL::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cFeatureExtraction.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cAttAndFeature.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTPM::Load(const int file_handle) { if(!CNeuronLSTMOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureExtraction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttAndFeature.AsObject())) ReturnFalse; //--- if(!cMemAndHidden.Init(0, 2, OpenCL, Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cSoftMax.Init(0, 4, OpenCL, cConcatenated.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(1); if(!cAttentionOut.Init(0, 5, OpenCL, cSoftMax.Neurons(), optimization, iBatch)) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(cFeatureExtraction.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTNNEncoder : public CNeuronMLMHAttentionMLKV { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTNNEncoder(void) {}; ~CNeuronSTNNEncoder(void) {}; //--- virtual int Type(void) override const { return defNeuronSTNNEncoder; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 1) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Feed Forward CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; inputs = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, inputs, 4 * iWindow, iWindow, None)) ReturnFalse; //--- Calculate Queries, Keys, Values CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(KV_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, kv, iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = FF_Tensors.At(i * 6); CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !ConvolutionInputGradients(KV_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), kv_g, inp, GetPointer(Temp), iWindow, 2 * iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; inp = (i > 0 ? FF_Tensors.At(i * 6 - 4) : prevLayer.getOutput()); temp = (i > 0 ? FF_Tensors.At(i * 6 - 1) : prevLayer.getGradient()); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), inp, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNEncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), inputs, (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 3), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), FF_Tensors.At(l * 6), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(KV_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), KV_Tensors.At(l_kv * 2 + 1), FF_Tensors.At(l * 6), (optimization == SGD ? KV_Weights.At(l_kv * 2 + 1) : KV_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : KV_Weights.At(l_kv * 3 + 2)), iWindow, 2 * iWindowKey * iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 5), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow)) ReturnFalse; //--- inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTNNDecoder : public CNeuronMLCrossAttentionMLKV { protected: CNeuronSTNNEncoder cEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; public: CNeuronSTNNDecoder(void) {}; ~CNeuronSTNNDecoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSTNNDecoder; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!cEncoder.Init(0, 0, open_cl, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, window_kv, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)iHeadsKV) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 1) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!cEncoder.FeedForward(NeuronOCL, Context)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(cEncoder.AsObject(), Context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(cEncoder.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEncoder.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!cEncoder.UpdateInputWeights(NeuronOCL, Context)) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(cEncoder.AsObject(), Context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTNNDecoder::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronMLCrossAttentionMLKV::WeightsUpdate(source, tau)) ReturnFalse; if(!cEncoder.WeightsUpdate(((CNeuronSTNNDecoder*)source).cEncoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTNNDecoder::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSparseTSF : public CNeuronBaseOCL { protected: CNeuronConvOCL cConvolution; CNeuronTransposeOCL acTranspose[4]; CNeuronConvOCL cForecast; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSparseTSF(void) {}; ~CNeuronSparseTSF(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint sequence, uint variables, uint period, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSparseTSF; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint sequence, uint variables, uint period, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; if(!acTranspose[0].Init(0, 0, OpenCL, sequence, variables, optimization, iBatch)) ReturnFalse; if(!cConvolution.Init(0, 1, OpenCL, 2 * period, period, period, sequence / period, variables, optimization, iBatch)) ReturnFalse; cConvolution.SetActivationFunction(None); if(!acTranspose[1].Init(0, 2, OpenCL, variables * sequence / period, period, optimization, iBatch)) ReturnFalse; if(!cForecast.Init(0, 3, OpenCL, sequence / period, sequence / period, forecast / period, variables, period, optimization, iBatch)) ReturnFalse; cForecast.SetActivationFunction(TANH); if(!acTranspose[2].Init(0, 4, OpenCL, period, variables * forecast / period, optimization, iBatch)) ReturnFalse; if(!acTranspose[3].Init(0, 5, OpenCL, variables, forecast, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(acTranspose[3].getOutput()) || !SetGradient(acTranspose[3].getGradient()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!acTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cConvolution.FeedForward(acTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cConvolution.getOutput(), acTranspose[0].getOutput(), cConvolution.getOutput(), 1, false)) ReturnFalse; if(!acTranspose[1].FeedForward(cConvolution.AsObject())) ReturnFalse; if(!cForecast.FeedForward(acTranspose[1].AsObject())) ReturnFalse; if(!acTranspose[2].FeedForward(cForecast.AsObject())) ReturnFalse; if(!acTranspose[3].FeedForward(acTranspose[2].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!acTranspose[2].CalcHiddenGradients(acTranspose[3].AsObject())) ReturnFalse; if(!cForecast.CalcHiddenGradients(acTranspose[2].AsObject())) ReturnFalse; if(cForecast.Activation() != None && !DeActivation(cForecast.getOutput(), cForecast.getGradient(), cForecast.getGradient(), cForecast.Activation())) ReturnFalse; if(!acTranspose[1].CalcHiddenGradients(cForecast.AsObject())) ReturnFalse; if(!cConvolution.CalcHiddenGradients(acTranspose[1].AsObject())) ReturnFalse; if(!acTranspose[0].CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; if(!SumAndNormilize(cConvolution.getGradient(), acTranspose[0].getGradient(), acTranspose[0].getGradient(), 1, false)) ReturnFalse; if(!NeuronOCL || !NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None && !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cConvolution.UpdateInputWeights(acTranspose[0].AsObject())) ReturnFalse; if(!cForecast.UpdateInputWeights(acTranspose[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseTSF::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(int i = 0; i < 4; i++) acTranspose[i].SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cForecast.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSparseTSF *Source = source; if(!cConvolution.WeightsUpdate(Source.cConvolution.AsObject(), tau)) ReturnFalse; if(!cForecast.WeightsUpdate(Source.cForecast.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!acTranspose[i].Save(file_handle)) ReturnFalse;; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cForecast.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseTSF::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject())) ReturnFalse;; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cForecast.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTEMPOOCL : public CNeuronBaseOCL { protected: //--- constants uint iVariables; uint iSequence; uint iForecast; uint iFFT; //--- Trend CNeuronPLROCL cPLR; CNeuronBaseOCL cTrend; //--- Seasons CNeuronBaseOCL cInputSeasons; CNeuronTransposeOCL cTranspose[2]; CBufferFloat cInputFreqRe; CBufferFloat cInputFreqIm; CNeuronBaseOCL cInputFreqComplex; CNeuronBaseOCL cNormFreqComplex; CBufferFloat cMeans; CBufferFloat cVariances; CNeuronComplexMLMHAttention cFreqAtteention; CNeuronBaseOCL cUnNormFreqComplex; CBufferFloat cOutputFreqRe; CBufferFloat cOutputFreqIm; CNeuronBaseOCL cOutputTimeSeriasRe; CBufferFloat cOutputTimeSeriasIm; CBufferFloat cZero; //--- Noise CNeuronBaseOCL cResidual; //--- Forecast CNeuronBaseOCL cConcatInput; CNeuronBatchNormOCL cNormalize; CNeuronPatching cPatching; CNeuronBatchNormOCL cNormalizePLR; CNeuronPatching cPatchingPLR; CNeuronPositionEncoder acPE[2]; CNeuronMLCrossAttentionMLKV cAttention; CNeuronTransposeOCL cTransposeAtt; CNeuronConvOCL acForecast[2]; CNeuronTransposeOCL cTransposeFrc; CNeuronRevINDenormOCL cRevIn; CNeuronConvOCL cSum; //--- Complex functions virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false); virtual bool ComplexNormalize(void); virtual bool ComplexUnNormalize(void); virtual bool ComplexNormalizeGradient(void); virtual bool ComplexUnNormalizeGradient(void); //--- bool CutTrendAndOther(CBufferFloat *inputs); bool CutTrendAndOtherGradient(CBufferFloat *inputs_gr); bool CutOneFromAnother(void); bool CutOneFromAnotherGradient(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronTEMPOOCL(void) {}; ~CNeuronTEMPOOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint sequence, uint variables, uint forecast, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronTEMPOOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual CBufferFloat *getWeights(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint sequence, uint variables, uint forecast, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- base if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, forecast * variables, optimization_type, batch)) ReturnFalse; //--- constants iVariables = variables; iForecast = forecast; iSequence = MathMax(sequence, 1); //--- Calculate FFTsize uint size = iSequence; int power = int(MathLog(size) / M_LN2); if(MathPow(2, power) < size) power++; iFFT = uint(MathPow(2, power)); //--- trend if(!cPLR.Init(0, 0, OpenCL, iVariables, iSequence, true, optimization, iBatch)) ReturnFalse; if(!cTrend.Init(0, 1, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- seasons if(!cInputSeasons.Init(0, 2, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[0].Init(0, 3, OpenCL, iSequence, iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[1].Init(0, 4, OpenCL, iVariables, iSequence, optimization, iBatch)) ReturnFalse; if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 5, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cNormFreqComplex.Init(0, 6, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cFreqAtteention.Init(0, 7, OpenCL, iFFT, 32, heads, iVariables, layers, optimization, batch)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 8, OpenCL, iFFT * iVariables * 2, optimization, batch)) ReturnFalse; if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.Init(0, 9, OpenCL, iFFT * iVariables, optimization, iBatch)) ReturnFalse; if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- Noise if(!cResidual.Init(0, 10, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- Forecast if(!cConcatInput.Init(0, 11, OpenCL, 3 * iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cNormalize.Init(0, 12, OpenCL, 3 * iSequence * iVariables, iBatch, optimization)) ReturnFalse; int window = MathMin(5, (int)iSequence - 1); int patches = (int)iSequence - window + 1; if(!cPatching.Init(0, 13, OpenCL, window, 1, 8, patches, 3 * iVariables, optimization, iBatch)) ReturnFalse; if(!acPE[0].Init(0, 14, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; int plr = cPLR.Neurons(); if(!cNormalizePLR.Init(0, 15, OpenCL, plr, iBatch, optimization)) ReturnFalse; plr = MathMax(plr / (3 * (int)iVariables), 1); if(!cPatchingPLR.Init(0, 16, OpenCL, 3, 3, 8, plr, iVariables, optimization, iBatch)) ReturnFalse; if(!acPE[1].Init(0, 17, OpenCL, plr, 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 18, OpenCL, 3 * 8 * iVariables, 3 * iVariables, MathMax(heads, 1), 8 * iVariables, MathMax(heads / 2, 1), patches, plr, MathMax(layers, 1), 2, optimization, iBatch)) ReturnFalse; if(!cTransposeAtt.Init(0, 19, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!acForecast[0].Init(0, 20, OpenCL, patches, patches, iForecast, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; acForecast[0].SetActivationFunction(LReLU); if(!acForecast[1].Init(0, 21, OpenCL, 8 * iForecast, 8 * iForecast, iForecast, 3 * iVariables, optimization, iBatch)) ReturnFalse; acForecast[1].SetActivationFunction(TANH); if(!cTransposeFrc.Init(0, 22, OpenCL, 3 * iVariables, iForecast, optimization, iBatch)) ReturnFalse; if(!cRevIn.Init(0, 23, OpenCL, 3 * iVariables * iForecast, 11, GetPointer(cNormalize))) ReturnFalse; if(!cSum.Init(0, 24, OpenCL, 3, 3, 1, iVariables, iForecast, optimization, iBatch)) ReturnFalse; cSum.SetActivationFunction(None); //--- SetActivationFunction(None); SetOutput(cSum.getOutput(), true); SetGradient(cSum.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_FFT, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(def_k_FFT, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(def_k_FFT, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(def_k_FFT, def_k_fft_input_window, (int)(inp_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_input_complex, int(!!inp_im)) setArgument(def_k_FFT, def_k_fft_output_window, (int)(out_re.Total() / iVariables)) setArgument(def_k_FFT, def_k_fft_reverse, int(reverse)) kernelExecute(def_k_FFT, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexUnNormalize(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalize, def_k_cn_inputs, cFreqAtteention.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_outputs, cUnNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexUnNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalize, def_k_cn_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::ComplexUnNormalizeGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iVariables}; setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_inputs_gr, cFreqAtteention.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_outputs_gr, cUnNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexUnNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexUnNormalizeGradient, def_k_cng_dimension, (int)(iFFT)) kernelExecute(def_k_ComplexUnNormalizeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutTrendAndOther(CBufferFloat * inputs) { if(!inputs) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iSequence, iVariables}; setBuffer(def_k_CutTrendAndOther, def_k_ct_inputs, inputs.GetIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_plr, cPLR.getOutputIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_trend, cTrend.getOutputIndex()) setBuffer(def_k_CutTrendAndOther, def_k_ct_other, cInputSeasons.getOutputIndex()) kernelExecute(def_k_CutTrendAndOther, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutTrendAndOtherGradient(CBufferFloat * inputs_gr) { if(!inputs_gr) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iSequence, iVariables}; setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_inputs_gr, inputs_gr.GetIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_plr, cPLR.getOutputIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_plr_gr, cPLR.getGradientIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_trend_gr, cTrend.getGradientIndex()) setBuffer(def_k_CutTrendAndOtherGradient, def_k_ctg_other_gr, cInputSeasons.getGradientIndex()) kernelExecute(def_k_CutTrendAndOtherGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutOneFromAnother(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSequence * iVariables}; setBuffer(def_k_CutOneFromAnother, def_k_ctofa_inputs, cInputSeasons.getOutputIndex()) setBuffer(def_k_CutOneFromAnother, def_k_ctofa_cut, cTranspose[1].getOutputIndex()) setBuffer(def_k_CutOneFromAnother, def_k_ctofa_other, cResidual.getOutputIndex()) kernelExecute(def_k_CutOneFromAnother, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::CutOneFromAnotherGradient(void) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iSequence * iVariables}; setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_inputs, cInputSeasons.getGradientIndex()) setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_cut, cOutputTimeSeriasRe.getGradientIndex()) setBuffer(def_k_CutOneFromAnotherGradient, def_k_ctofa_other, cResidual.getGradientIndex()) kernelExecute(def_k_CutOneFromAnother, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- trend if(!cPLR.FeedForward(NeuronOCL)) ReturnFalse; if(!CutTrendAndOther(NeuronOCL.getOutput())) ReturnFalse; //--- seasons if(!cTranspose[0].FeedForward(cInputSeasons.AsObject())) ReturnFalse; if(!FFT(cTranspose[0].getOutput(), NULL, GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), cInputFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexNormalize()) ReturnFalse; if(!cFreqAtteention.FeedForward(cNormFreqComplex.AsObject())) ReturnFalse; if(!ComplexUnNormalize()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getOutput(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), true)) ReturnFalse; if(!DeConcat(cOutputTimeSeriasRe.getOutput(), cOutputTimeSeriasRe.getGradient(), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!cTranspose[1].FeedForward(cOutputTimeSeriasRe.AsObject())) ReturnFalse; //--- Noise if(!CutOneFromAnother()) ReturnFalse; //--- Forecast if(!Concat(cTrend.getOutput(), cTranspose[1].getOutput(), cResidual.getOutput(), cConcatInput.getOutput(), 1, 1, 1, 3 * iSequence * iVariables)) ReturnFalse; if(!cNormalize.FeedForward(cConcatInput.AsObject())) ReturnFalse; if(!cPatching.FeedForward(cNormalize.AsObject())) ReturnFalse; if(!acPE[0].FeedForward(cPatching.AsObject())) ReturnFalse; if(!cNormalizePLR.FeedForward(cPLR.AsObject())) ReturnFalse; if(!cPatchingPLR.FeedForward(cPatchingPLR.AsObject())) ReturnFalse; if(!acPE[1].FeedForward(cPatchingPLR.AsObject())) ReturnFalse; if(!cAttention.FeedForward(acPE[0].AsObject(), acPE[1].getOutput())) ReturnFalse; if(!cTransposeAtt.FeedForward(cAttention.AsObject())) ReturnFalse; if(!acForecast[0].FeedForward(cTransposeAtt.AsObject())) ReturnFalse; if(!acForecast[1].FeedForward(acForecast[0].AsObject())) ReturnFalse; if(!cTransposeFrc.FeedForward(acForecast[1].AsObject())) ReturnFalse; if(!cRevIn.FeedForward(cTransposeFrc.AsObject())) ReturnFalse; if(!cSum.FeedForward(cRevIn.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Devide to Trend, Seasons and Noise if(!cRevIn.CalcHiddenGradients(cSum.AsObject())) ReturnFalse; //--- Forecast gradient if(!cTransposeFrc.CalcHiddenGradients(cRevIn.AsObject())) ReturnFalse; if(!acForecast[1].CalcHiddenGradients(cTransposeFrc.AsObject())) ReturnFalse; if(acForecast[1].Activation() != None && !DeActivation(acForecast[1].getOutput(), acForecast[1].getGradient(), acForecast[1].getGradient(), acForecast[1].Activation()) ) ReturnFalse; if(!acForecast[0].CalcHiddenGradients(acForecast[1].AsObject())) ReturnFalse; //--- Attention gradient if(!cTransposeAtt.CalcHiddenGradients(acForecast[0].AsObject())) ReturnFalse; if(!cAttention.CalcHiddenGradients(cTransposeAtt.AsObject())) ReturnFalse; if(!acPE[0].CalcHiddenGradients(cAttention.AsObject(), acPE[1].getOutput(), acPE[1].getGradient(), (ENUM_ACTIVATION)acPE[1].Activation())) ReturnFalse; //--- Gradient to PLR if(!cPatchingPLR.CalcHiddenGradients(acPE[1].AsObject())) ReturnFalse; if(!cNormalizePLR.CalcHiddenGradients(cPatchingPLR.AsObject())) ReturnFalse; if(!cPLR.CalcHiddenGradients(cNormalizePLR.AsObject())) ReturnFalse; //--- Gradient to Concatenate buffer of Trend, Season and Noise if(!cPatching.CalcHiddenGradients(acPE[0].AsObject())) ReturnFalse; if(!cNormalize.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; if(!cConcatInput.CalcHiddenGradients(cNormalize.AsObject())) ReturnFalse; //--- DeConcatenate if(!DeConcat(cTrend.getGradient(), cOutputTimeSeriasRe.getGradient(), cResidual.getGradient(), cConcatInput.getGradient(), 1, 1, 1, 3 * iSequence * iVariables)) ReturnFalse; //--- Seasons if(!CutOneFromAnotherGradient()) ReturnFalse; if(!SumAndNormilize(cOutputTimeSeriasRe.getGradient(), cTranspose[1].getGradient(), cTranspose[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cOutputTimeSeriasRe.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!Concat(cOutputTimeSeriasRe.getGradient(), GetPointer(cZero), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!SumAndNormilize(GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputTimeSeriasIm), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!FFT(GetPointer(cInputFreqRe), GetPointer(cOutputTimeSeriasIm), GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), false)) ReturnFalse; if(!Concat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cUnNormFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!ComplexUnNormalizeGradient()) ReturnFalse; if(!cNormFreqComplex.CalcHiddenGradients(cFreqAtteention.AsObject())) ReturnFalse; if(!ComplexNormalizeGradient()) ReturnFalse; if(!DeConcat(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), cInputFreqComplex.getGradient(), 1, 1, iFFT * iVariables)) ReturnFalse; if(!FFT(GetPointer(cOutputFreqRe), GetPointer(cOutputFreqIm), GetPointer(cInputFreqRe), GetPointer(cInputFreqIm), true)) ReturnFalse; if(!DeConcat(cTranspose[0].getGradient(), GetPointer(cInputFreqIm), GetPointer(cInputFreqRe), iSequence, iFFT - iSequence, iVariables)) ReturnFalse; if(!cInputSeasons.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cInputSeasons.getGradient(), cResidual.getGradient(), cInputSeasons.getGradient(), 1, 1, false, 0, 0, 1)) ReturnFalse; //--- trend if(!CutTrendAndOtherGradient(NeuronOCL.getGradient())) ReturnFalse; //--- input gradient if(!NeuronOCL.CalcHiddenGradients(cPLR.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cInputSeasons.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- trend if(!cPLR.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- seasons if(!cFreqAtteention.UpdateInputWeights(cNormFreqComplex.AsObject())) ReturnFalse; //--- Forecast if(!cNormalize.UpdateInputWeights(cConcatInput.AsObject())) ReturnFalse; if(!cPatching.UpdateInputWeights(cNormalize.AsObject())) ReturnFalse; if(!acPE[0].UpdateInputWeights(cPatching.AsObject())) ReturnFalse; if(!cNormalizePLR.UpdateInputWeights(cPLR.AsObject())) ReturnFalse; if(!cPatchingPLR.UpdateInputWeights(cPatchingPLR.AsObject())) ReturnFalse; if(!acPE[1].UpdateInputWeights(cPatchingPLR.AsObject())) ReturnFalse; if(!cAttention.UpdateInputWeights(acPE[0].AsObject(), acPE[1].getOutput())) ReturnFalse; if(!acForecast[0].UpdateInputWeights(cTransposeAtt.AsObject())) ReturnFalse; if(!acForecast[1].UpdateInputWeights(acForecast[0].AsObject())) ReturnFalse; if(!cSum.UpdateInputWeights(cRevIn.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- constants if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSequence)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iForecast)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFT)) < INT_VALUE) ReturnFalse; //--- trend if(!cPLR.Save(file_handle)) ReturnFalse; //--- seasons if(!cFreqAtteention.Save(file_handle)) ReturnFalse; //--- Forecast if(!cNormalize.Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!acPE[0].Save(file_handle)) ReturnFalse; if(!cNormalizePLR.Save(file_handle)) ReturnFalse; if(!cPatchingPLR.Save(file_handle)) ReturnFalse; if(!acPE[1].Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(!acForecast[0].Save(file_handle)) ReturnFalse; if(!acForecast[1].Save(file_handle)) ReturnFalse; if(!cSum.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTEMPOOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- constants if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSequence = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFT = (uint)FileReadInteger(file_handle); //--- trend if(!LoadInsideLayer(file_handle, cPLR.AsObject())) ReturnFalse; //--- seasons if(!LoadInsideLayer(file_handle, cFreqAtteention.AsObject())) ReturnFalse; //--- Forecast if(!LoadInsideLayer(file_handle, cNormalize.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acPE[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizePLR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingPLR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acPE[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acForecast[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, acForecast[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSum.AsObject())) ReturnFalse; //--- ReInit Objects //--- trend if(!cTrend.Init(0, 1, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- seasons if(!cInputSeasons.Init(0, 2, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[0].Init(0, 3, OpenCL, iSequence, iVariables, optimization, iBatch)) ReturnFalse; if(!cTranspose[1].Init(0, 4, OpenCL, iVariables, iSequence, optimization, iBatch)) ReturnFalse; cInputFreqRe.BufferFree(); if(!cInputFreqRe.BufferInit(iFFT * iVariables, 0) || !cInputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cInputFreqIm.BufferFree(); if(!cInputFreqIm.BufferInit(iFFT * iVariables, 0) || !cInputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cInputFreqComplex.Init(0, 5, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; if(!cNormFreqComplex.Init(0, 6, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cMeans.BufferFree(); if(!cMeans.BufferInit(iVariables, 0) || !cMeans.BufferCreate(OpenCL)) ReturnFalse; cVariances.BufferFree(); if(!cVariances.BufferInit(iVariables, 0) || !cVariances.BufferCreate(OpenCL)) ReturnFalse; if(!cUnNormFreqComplex.Init(0, 8, OpenCL, iFFT * iVariables * 2, optimization, iBatch)) ReturnFalse; cOutputFreqRe.BufferFree(); if(!cOutputFreqRe.BufferInit(iFFT * iVariables, 0) || !cOutputFreqRe.BufferCreate(OpenCL)) ReturnFalse; cOutputFreqIm.BufferFree(); if(!cOutputFreqIm.BufferInit(iFFT * iVariables, 0) || !cOutputFreqIm.BufferCreate(OpenCL)) ReturnFalse; if(!cOutputTimeSeriasRe.Init(0, 9, OpenCL, iFFT * iVariables, optimization, iBatch)) ReturnFalse; cOutputTimeSeriasIm.BufferFree(); if(!cOutputTimeSeriasIm.BufferInit(iFFT * iVariables, 0) || !cOutputTimeSeriasIm.BufferCreate(OpenCL)) ReturnFalse; cZero.BufferFree(); if(!cZero.BufferInit(iFFT * iVariables, 0) || !cZero.BufferCreate(OpenCL)) ReturnFalse; //--- Noise if(!cResidual.Init(0, 10, OpenCL, iSequence * iVariables, optimization, iBatch)) ReturnFalse; //--- Forecast if(!cConcatInput.Init(0, 11, OpenCL, 3 * iSequence * iVariables, optimization, iBatch)) ReturnFalse; uint patches = cAttention.Neurons() / (3 * 8 * iVariables); if(!cTransposeAtt.Init(0, 19, OpenCL, patches, 3 * 8 * iVariables, optimization, iBatch)) ReturnFalse; if(!cTransposeFrc.Init(0, 22, OpenCL, 3 * iVariables, iForecast, optimization, iBatch)) ReturnFalse; if(!cRevIn.Init(0, 23, OpenCL, 3 * iVariables * iForecast, 11, GetPointer(cNormalize))) ReturnFalse; //--- SetOutput(cSum.getOutput(), true); SetGradient(cSum.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTEMPOOCL::SetOpenCL(COpenCLMy * obj) { //--- base CNeuronBaseOCL::SetOpenCL(obj); //--- trend cPLR.SetOpenCL(OpenCL); cTrend.SetOpenCL(OpenCL); //--- seasons cInputSeasons.SetOpenCL(OpenCL); cTranspose[0].SetOpenCL(OpenCL); cTranspose[1].SetOpenCL(OpenCL); cInputFreqRe.BufferCreate(OpenCL); cInputFreqIm.BufferCreate(OpenCL); cInputFreqComplex.SetOpenCL(OpenCL); cNormFreqComplex.SetOpenCL(OpenCL); cMeans.BufferCreate(OpenCL); cVariances.BufferCreate(OpenCL); cFreqAtteention.SetOpenCL(OpenCL); cUnNormFreqComplex.SetOpenCL(OpenCL); cOutputFreqRe.BufferCreate(OpenCL); cOutputFreqIm.BufferCreate(OpenCL); cOutputTimeSeriasRe.SetOpenCL(OpenCL); cOutputTimeSeriasIm.BufferCreate(OpenCL); cZero.BufferCreate(OpenCL); //--- Noise cResidual.SetOpenCL(OpenCL); //--- Forecast cConcatInput.SetOpenCL(OpenCL); cNormalize.SetOpenCL(OpenCL); cPatching.SetOpenCL(OpenCL); acPE[0].SetOpenCL(OpenCL); cNormalizePLR.SetOpenCL(OpenCL); cPatchingPLR.SetOpenCL(OpenCL); acPE[1].SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cTransposeAtt.SetOpenCL(OpenCL); acForecast[0].SetOpenCL(OpenCL); acForecast[1].SetOpenCL(OpenCL); cTransposeFrc.SetOpenCL(OpenCL); cRevIn.SetOpenCL(OpenCL); cSum.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronTEMPOOCL::getWeights(void) { CBufferFloat *result = CNeuronBaseOCL::getWeights(); if(!result) result = cSum.GetWeightsConv(); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMVMHAttentionMLKV : public CNeuronMLMHAttentionOCL { protected: uint iLayersToOneKV; ///< Number of inner layers uint iHeadsKV; ///< Number of heads uint iVariables; ///< Number of variables CCollection KV_Tensors; ///< The collection of tensors of Keys and Values CCollection K_Tensors; ///< The collection of tensors of Keys CCollection K_Weights; ///< The collection of Matrix of weights to previous layer CCollection V_Tensors; ///< The collection of tensors of Values CCollection V_Weights; ///< The collection of Matrix of weights to previous layer CBufferFloat Temp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out); virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMVMHAttentionMLKV(void) {}; ~CNeuronMVMHAttentionMLKV(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMVMHAttentionMLKV; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables; //--- uint num_q = iWindowKey * iHeads * iUnits * iVariables; //Size of Q tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariables; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = (iWindow * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint scores = iUnits * iUnits * iHeads * iVariables; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //Size of multi-heads self-attention uint out = iWindow * iUnits * iVariables; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnits/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnits, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariables)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, k, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, v, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, CBufferFloat * scores, CBufferFloat * outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnits) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariables)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronMVMHAttentionMLKV *Source = source; for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), Source.QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), tau)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), Source.V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), tau)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9)), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), tau)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), Source.FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Save(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, iLayersToOneKV, INT_VALUE) || !FileWriteInteger(file_handle, iHeadsKV, INT_VALUE) || !FileWriteInteger(file_handle, iVariables, INT_VALUE)) ReturnFalse; //--- Saving objects if(!KV_Tensors.Save(file_handle)) ReturnFalse; if(!K_Tensors.Save(file_handle) || !K_Weights.Save(file_handle)) ReturnFalse; if(!V_Tensors.Save(file_handle) || !V_Weights.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVMHAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnits * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMVMHAttentionMLKV::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionOCL::SetOpenCL(obj); KV_Tensors.SetOpenCL(OpenCL); K_Tensors.SetOpenCL(OpenCL); K_Weights.SetOpenCL(OpenCL); V_Tensors.SetOpenCL(OpenCL); V_Weights.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMVCrossAttentionMLKV : public CNeuronMVMHAttentionMLKV { protected: uint iWindowKV; uint iUnitsKV; uint iVariablesKV; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context); virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, CBufferFloat *scores, CBufferFloat *out); virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, CBufferFloat *scores, CBufferFloat *gradient); //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context); ///< Method for updating weights.\details Calling one of kernels ::UpdateWeightsMomentum() or ::UpdateWeightsAdam() in depends of optimization type (#ENUM_OPTIMIZATION).@param NeuronOCL Pointer to previos layer. public: CNeuronMVCrossAttentionMLKV(void) {}; ~CNeuronMVCrossAttentionMLKV(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, uint variables_q, uint variables_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMVCrossMHAttentionMLKV; } //--- virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- virtual CBufferFloat* getWeights(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint window_kv, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, uint variables_q, uint variables_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables_q, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iWindowKV = window_kv; iUnitsKV = fmax(units_count_kv, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables_q; iVariablesKV = variables_kv; //--- uint num_q = iWindowKey * iHeads * iUnits * iVariables; //Size of Q tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariablesKV; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = (iWindowKV * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint scores = iUnits * iUnitsKV * iHeads * iVariables; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //Size of multi-heads self-attention uint out = iWindow * iUnits * iVariables; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionOut(CBufferFloat * q, CBufferFloat * kv, CBufferFloat * scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iUnitsKV/*K units*/, iHeads * iVariables}; uint local_work_size[3] = {1, iUnitsKV, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, (int)iWindowKey) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (int)(iHeadsKV * iVariablesKV)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, k, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), Context, v, iWindowKV, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariablesKV, iWindowKey * iHeadsKV * iVariablesKV, iUnitsKV)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //out.BufferRead(); //--- Attention out calculation temp = FF_Tensors.At(i * 6); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors.At(i * 6 + 1); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors.At(i * 6 + 2); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, CBufferFloat * scores, CBufferFloat * outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iWindowKey, iHeads * iVariables}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iUnitsKV) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (int)(iHeadsKV * iVariablesKV)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); if(!SecondGradient.BufferInit(SecondGradient.Total(), 0) || !SecondGradient.BufferWrite()) ReturnFalse; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), out_grad, FF_Tensors.At(i * 6 + 1), FF_Tensors.At(i * 6 + 4), 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors.At(i * 6 + 3); if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out_grad, AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = NeuronOCL.getOutput(); temp = NeuronOCL.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, temp, iWindow, iWindowKey * iHeads, None)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), SecondGradient, SecondGradient, iWindowKV, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = temp; } //--- Deactivation7 if(SecondActivation != None && !DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(NeuronOCL.Activation() != None && !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * Context) { if(!NeuronOCL || !Context) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), Context, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 6 + 3), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 6 + 4), FF_Tensors.At(l * 6), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 6 + 5), FF_Tensors.At(l * 6 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 6 + 2); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Save(const int file_handle) { if(!CNeuronMVMHAttentionMLKV::Save(file_handle)) ReturnFalse; //--- Saving constants if(!FileWriteInteger(file_handle, int(iWindowKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iUnitsKV), INT_VALUE) || !FileWriteInteger(file_handle, int(iVariablesKV), INT_VALUE)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMVCrossAttentionMLKV::Load(const int file_handle) { if(!CNeuronMLMHAttentionOCL::Load(file_handle)) ReturnFalse; //--- Loading constants iLayersToOneKV = (uint)FileReadInteger(file_handle); iHeadsKV = (uint)FileReadInteger(file_handle); iVariables = (uint)FileReadInteger(file_handle); //--- loading objects if(!KV_Tensors.Load(file_handle)) ReturnFalse; if(!K_Tensors.Load(file_handle) || !K_Weights.Load(file_handle)) ReturnFalse; if(!V_Tensors.Load(file_handle) || !V_Weights.Load(file_handle)) ReturnFalse; if(!KV_Tensors.SetOpenCL(OpenCL)) ReturnFalse; if(!K_Tensors.SetOpenCL(OpenCL) || !K_Weights.SetOpenCL(OpenCL)) ReturnFalse; if(!V_Tensors.SetOpenCL(OpenCL) || !V_Weights.SetOpenCL(OpenCL)) ReturnFalse; //--- Loading constants iWindowKV = (uint)FileReadInteger(file_handle); iUnitsKV = (uint)FileReadInteger(file_handle); iVariablesKV = (uint)FileReadInteger(file_handle); //--- Temp.BufferFree(); if(!Temp.BufferInit(MathMin(iWindow * iUnits, 2 * iWindowKey * iUnitsKV * iHeadsKV), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronMVCrossAttentionMLKV::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = FF_Weights.At((iLayers - 1) * (optimization == SGD ? 6 : 9) + 2); return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronInjectTST : public CNeuronBaseOCL { protected: CNeuronPatching cPatching; CNeuronLearnabledPE cCIPosition; CNeuronLearnabledPE cCMPosition; CNeuronMVMHAttentionMLKV cChanelIndependentAttention; CNeuronMLMHAttentionMLKV cChanelMixAttention; CNeuronMVCrossAttentionMLKV cGlobalInjectionAttention; CBufferFloat cTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronInjectTST(void) {}; ~CNeuronInjectTST(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronInjectTST; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual CBufferFloat *getWeights(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); if(!cPatching.Init(0, 0, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; cPatching.SetActivationFunction(None); if(!cCIPosition.Init(0, 1, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCIPosition.SetActivationFunction(None); if(!cCMPosition.Init(0, 2, OpenCL, window * units_count * variables, optimization, iBatch)) ReturnFalse; cCMPosition.SetActivationFunction(None); if(!cChanelIndependentAttention.Init(0, 3, OpenCL, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, variables, optimization, iBatch)) ReturnFalse; cChanelIndependentAttention.SetActivationFunction(None); if(!cChanelMixAttention.Init(0, 4, OpenCL, window * variables, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, optimization, iBatch)) ReturnFalse; cChanelMixAttention.SetActivationFunction(None); if(!cGlobalInjectionAttention.Init(0, 5, OpenCL, window, window_key, heads, window * variables, heads_kv, units_count, units_count, layers, layers_to_one_kv, variables, 1, optimization, iBatch)) ReturnFalse; cGlobalInjectionAttention.SetActivationFunction(None); if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cPatching.FeedForward(NeuronOCL)) ReturnFalse; if(!cCIPosition.FeedForward(cPatching.AsObject()) || !cCMPosition.FeedForward(cPatching.AsObject()) ) ReturnFalse; if(!cChanelIndependentAttention.FeedForward(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.FeedForward(cCMPosition.AsObject())) ReturnFalse; if(!cGlobalInjectionAttention.FeedForward(cCIPosition.AsObject(), cCMPosition.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cChanelIndependentAttention.CalcHiddenGradients(cGlobalInjectionAttention.AsObject(), cChanelMixAttention.getOutput(), cChanelMixAttention.getGradient(), (ENUM_ACTIVATION)cChanelMixAttention.Activation()) ) ReturnFalse; //--- if(!cCMPosition.CalcHiddenGradients(cChanelMixAttention.AsObject())) ReturnFalse; if(!cCIPosition.CalcHiddenGradients(cChanelIndependentAttention.AsObject())) ReturnFalse; //--- if(!cPatching.CalcHiddenGradients(cCIPosition.AsObject())) ReturnFalse; if(!SumAndNormilize(cPatching.getGradient(), cPatching.getGradient(), GetPointer(cTemp), 1, false)) ReturnFalse; if(!cPatching.CalcHiddenGradients(cCMPosition.AsObject())) ReturnFalse; if(!SumAndNormilize(cPatching.getGradient(), GetPointer(cTemp), cPatching.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cPatching.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cPatching.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cCIPosition.UpdateInputWeights(cPatching.AsObject()) || !cCMPosition.UpdateInputWeights(cPatching.AsObject()) ) ReturnFalse; //--- if(!cChanelIndependentAttention.UpdateInputWeights(cCIPosition.AsObject())) ReturnFalse; if(!cChanelMixAttention.UpdateInputWeights(cCMPosition.AsObject())) ReturnFalse; //--- if(!cGlobalInjectionAttention.UpdateInputWeights(cChanelIndependentAttention.AsObject(), cChanelMixAttention.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cPatching.Save(file_handle)) ReturnFalse; if(!cCIPosition.Save(file_handle)) ReturnFalse; if(!cCMPosition.Save(file_handle)) ReturnFalse; if(!cChanelIndependentAttention.Save(file_handle)) ReturnFalse; if(!cChanelMixAttention.Save(file_handle)) ReturnFalse; if(!cGlobalInjectionAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInjectTST::Load(const int file_handle) { cTemp.BufferFree(); //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatching.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCIPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCMPosition.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelIndependentAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChanelMixAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGlobalInjectionAttention.AsObject())) ReturnFalse; //--- if(!SetOutput(cGlobalInjectionAttention.getOutput(), true) || !SetGradient(cGlobalInjectionAttention.getGradient(), true) ) ReturnFalse; //--- if(!cTemp.BufferInit(cPatching.Neurons(), 0) || !cTemp.BufferCreate(OpenCL) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInjectTST::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cPatching.SetOpenCL(OpenCL); cCIPosition.SetOpenCL(OpenCL); cCMPosition.SetOpenCL(OpenCL); cChanelIndependentAttention.SetOpenCL(OpenCL); cChanelMixAttention.SetOpenCL(OpenCL); cGlobalInjectionAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronInjectTST::getWeights(void) { CBufferFloat *result = Weights; if(!result) result = cGlobalInjectionAttention.getWeights(); //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSSMOCL : public CNeuronBaseOCL { protected: uint iWindowHidden; CNeuronBaseOCL cHiddenStates; CNeuronConvOCL cA; CNeuronConvOCL cB; CNeuronBaseOCL cAB; CNeuronConvOCL cC; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronSSMOCL(void) {}; ~CNeuronSSMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSSMOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- if(!cHiddenStates.Init(0, 0, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cHiddenStates.SetActivationFunction(None); iWindowHidden = window_key; //--- if(!cA.Init(0, 1, OpenCL, iWindowHidden, iWindowHidden, iWindowHidden, units_count, 1, optimization, iBatch)) ReturnFalse; cA.SetActivationFunction(SIGMOID); //--- if(!cB.Init(0, 2, OpenCL, window, window, iWindowHidden, units_count, 1, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(SIGMOID); //--- if(!cAB.Init(0, 3, OpenCL, 2 * iWindowHidden * units_count, optimization, iBatch)) ReturnFalse; cAB.SetActivationFunction(None); //--- if(!cC.Init(0, 4, OpenCL, 2 * iWindowHidden, 2 * iWindowHidden, window, units_count, 1, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); //--- SetActivationFunction(None); if(!SetOutput(cC.getOutput()) || !SetGradient(cC.getGradient())) ReturnFalse; //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cA.FeedForward(cHiddenStates.AsObject())) ReturnFalse; if(!cB.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cA.getOutput(), cB.getOutput(), cAB.getOutput(), iWindowHidden, iWindowHidden, cA.Neurons() / iWindowHidden)) ReturnFalse; if(!cC.FeedForward(cAB.AsObject())) ReturnFalse; //--- if(!bTrain) if(!SumAndNormilize(cA.getOutput(), cB.getOutput(), cHiddenStates.getOutput(), iWindowHidden, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cAB.CalcHiddenGradients(cC.AsObject())) ReturnFalse; //--- if(!DeConcat(cA.getGradient(), cB.getGradient(), cAB.getGradient(), iWindowHidden, iWindowHidden, cA.Neurons() / iWindowHidden)) ReturnFalse; if(cA.Activation() != None && !DeActivation(cA.getOutput(), cA.getGradient(), cA.getGradient(), cA.Activation())) ReturnFalse; if(cB.Activation() != None && !DeActivation(cB.getOutput(), cB.getGradient(), cB.getGradient(), cB.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cB.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cA.UpdateInputWeights(cHiddenStates.AsObject())) ReturnFalse; if(!SumAndNormilize(cA.getOutput(), cB.getOutput(), cHiddenStates.getOutput(), iWindowHidden, true)) ReturnFalse; //--- if(!cB.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cC.UpdateInputWeights(cAB.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cHiddenStates.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cB.SetOpenCL(OpenCL); cAB.SetOpenCL(OpenCL); cC.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowHidden)) < INT_VALUE) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cB.Save(file_handle)) ReturnFalse; if(!cC.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindowHidden = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC.AsObject())) ReturnFalse; //--- if(!cHiddenStates.Init(0, 0, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; if(!cAB.Init(0, 3, OpenCL, 2 * cA.Neurons(), optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cC.getOutput()) || !SetGradient(cC.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; CBufferFloat *output = cHiddenStates.getOutput(); if(!output || !output.Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMambaOCL : public CNeuronBaseOCL { protected: CNeuronConvOCL cXProject; CNeuronConvOCL cZProject; CNeuronConvOCL cInsideConv; CNeuronSSMOCL cSSM; CNeuronBaseOCL cZSSM; CNeuronConvOCL cOutProject; CBufferFloat Temp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronMambaOCL(void) {}; ~CNeuronMambaOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMambaOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cXProject.Init(0, 0, OpenCL, window, window, window_key + 2, units_count, 1, optimization, iBatch)) ReturnFalse; cXProject.SetActivationFunction(None); if(!cZProject.Init(0, 1, OpenCL, window, window, window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cZProject.SetActivationFunction(SIGMOID); if(!cInsideConv.Init(0, 2, OpenCL, 3, 1, 1, window_key, units_count, optimization, iBatch)) ReturnFalse; cInsideConv.SetActivationFunction(SIGMOID); if(!cSSM.Init(0, 3, OpenCL, window_key, window_key, units_count, optimization, iBatch)) ReturnFalse; if(!cZSSM.Init(0, 4, OpenCL, 2 * window_key * units_count, optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); if(!cOutProject.Init(0, 5, OpenCL, 2 * window_key, 2 * window_key, window, units_count, 1, optimization, iBatch)) ReturnFalse; cOutProject.SetActivationFunction(None); //--- if(!Temp.BufferInit(window * units_count, 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SetOutput(cOutProject.getOutput())) ReturnFalse; if(!SetGradient(cOutProject.getGradient())) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cXProject.FeedForward(NeuronOCL)) ReturnFalse; if(!cZProject.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!cInsideConv.FeedForward(cXProject.AsObject())) ReturnFalse; if(!cSSM.FeedForward(cInsideConv.AsObject())) ReturnFalse; if(!Concat(cSSM.getOutput(), cZProject.getOutput(), cZSSM.getOutput(), 1, 1, cSSM.Neurons())) ReturnFalse; if(!cOutProject.FeedForward(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cZSSM.CalcHiddenGradients(cOutProject.AsObject())) ReturnFalse; if(!DeConcat(cSSM.getGradient(), cZProject.getGradient(), cZSSM.getGradient(), 1, 1, cSSM.Neurons())) ReturnFalse; if(cZProject.Activation() != None && !DeActivation(cZProject.getOutput(), cZProject.getGradient(), cZProject.getGradient(), cZProject.Activation())) ReturnFalse; if(!cInsideConv.CalcHiddenGradients(cSSM.AsObject())) ReturnFalse; if(!cXProject.CalcHiddenGradients(cInsideConv.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cZProject.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), GetPointer(Temp), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cXProject.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), GetPointer(Temp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cXProject.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cZProject.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cInsideConv.UpdateInputWeights(cXProject.AsObject())) ReturnFalse; if(!cSSM.UpdateInputWeights(cInsideConv.AsObject())) ReturnFalse; if(!cOutProject.UpdateInputWeights(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMambaOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cXProject.SetOpenCL(OpenCL); cZProject.SetOpenCL(OpenCL); cInsideConv.SetOpenCL(OpenCL); cSSM.SetOpenCL(OpenCL); cZSSM.SetOpenCL(OpenCL); cOutProject.SetOpenCL(OpenCL); Temp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cXProject.Save(file_handle)) ReturnFalse; if(!cZProject.Save(file_handle)) ReturnFalse; if(!cInsideConv.Save(file_handle)) ReturnFalse; if(!cSSM.Save(file_handle)) ReturnFalse; if(!cOutProject.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cXProject.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cZProject.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cInsideConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSSM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cOutProject.AsObject())) ReturnFalse; //--- if(!cZSSM.Init(0, 3, OpenCL, 2 * cZProject.Neurons(), optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); //--- if(!SetOutput(cOutProject.getOutput())) ReturnFalse; if(!SetGradient(cOutProject.getGradient())) ReturnFalse; //--- if(!Temp.BufferInit(Neurons(), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaOCL::Clear(void) { return cSSM.Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMambaBlockOCL : public CNeuronBaseOCL { protected: uint iWindow; CNeuronMambaOCL cMamba; CNeuronBaseOCL cMambaResidual; CNeuronConvOCL cFF[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronMambaBlockOCL(void) {}; ~CNeuronMambaBlockOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMambaBlockOCL; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; //--- if(!cMamba.Init(0, 0, OpenCL, window, window_key, units_count, optimization, iBatch)) ReturnFalse; if(!cMambaResidual.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cMambaResidual.SetActivationFunction(None); if(!cFF[0].Init(0, 2, OpenCL, window, window, 4 * window, units_count, 1, optimization, iBatch)) ReturnFalse; cFF[0].SetActivationFunction(LReLU); if(!cFF[1].Init(0, 2, OpenCL, 4 * window, 4 * window, window, units_count, 1, optimization, iBatch)) ReturnFalse; cFF[1].SetActivationFunction(None); //--- SetActivationFunction(None); SetGradient(cFF[1].getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cMamba.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(cMamba.getOutput(), NeuronOCL.getOutput(), cMambaResidual.getOutput(), iWindow, true)) ReturnFalse; if(!cFF[0].FeedForward(cMambaResidual.AsObject())) ReturnFalse; if(!cFF[1].FeedForward(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cMambaResidual.getOutput(), cFF[1].getOutput(), getOutput(), iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cFF[0].CalcHiddenGradients(cFF[1].AsObject())) ReturnFalse; if(!cMambaResidual.CalcHiddenGradients(cFF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cMambaResidual.getGradient(), getGradient(), cMamba.getGradient(), iWindow, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMamba.AsObject())) ReturnFalse; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), cMambaResidual.getGradient(), cMamba.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(cMambaResidual.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; } else if(!SumAndNormilize(cMamba.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), iWindow, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cMamba.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFF[0].UpdateInputWeights(cMambaResidual.AsObject())) ReturnFalse; if(!cFF[1].UpdateInputWeights(cFF[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; //--- if(!cMamba.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFF[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMambaBlockOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cMamba.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cFF[i].AsObject())) ReturnFalse; if(!cMambaResidual.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- SetGradient(cFF[1].getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMambaBlockOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cMamba.SetOpenCL(OpenCL); cMambaResidual.SetOpenCL(OpenCL); cFF[0].SetOpenCL(OpenCL); cFF[1].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTrajLLMOCL : public CNeuronBaseOCL { protected: //--- State Encoder CNeuronLSTMOCL cStateRNN; CNeuronConvOCL cStateMLP[2]; //--- Variables Encoder CNeuronTransposeOCL cTranspose; CNeuronLSTMOCL cVariablesRNN; CNeuronConvOCL cVariablesMLP[2]; //--- Context Encoder CNeuronLearnabledPE cStatePE; CNeuronLearnabledPE cVariablesPE; CNeuronMLMHAttentionMLKV cStateToState; CNeuronMLCrossAttentionMLKV cVariableToState; CNeuronMLCrossAttentionMLKV cStateToVariable; CNeuronBaseOCL cContext; CNeuronConvOCL cContextMLP[2]; //--- CNeuronMLMHAttentionMLKV cHighLevelInteraction; CNeuronMambaBlockOCL caMamba[3]; CNeuronMLCrossAttentionMLKV cLaneAware; CNeuronConvOCL caForecastMLP[2]; CNeuronTransposeOCL cTransposeOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronTrajLLMOCL(void) {}; ~CNeuronTrajLLMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronTrajLLMOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * forecast, optimization_type, batch)) ReturnFalse; //--- State Encoder if(!cStateRNN.Init(0, 0, OpenCL, window_key, units_count, optimization, iBatch) || !cStateRNN.SetInputs(window)) ReturnFalse; if(!cStateMLP[0].Init(0, 1, OpenCL, window_key, window_key, 4 * window_key, units_count, optimization, iBatch)) ReturnFalse; cStateMLP[0].SetActivationFunction(LReLU); if(!cStateMLP[1].Init(0, 2, OpenCL, 4 * window_key, 4 * window_key, window_key, units_count, optimization, iBatch)) ReturnFalse; //--- Variables Encoder if(!cTranspose.Init(0, 3, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; if(!cVariablesRNN.Init(0, 4, OpenCL, window_key, window, optimization, iBatch) || !cVariablesRNN.SetInputs(units_count)) ReturnFalse; if(!cVariablesMLP[0].Init(0, 5, OpenCL, window_key, window_key, 4 * window_key, window, optimization, iBatch)) ReturnFalse; cVariablesMLP[0].SetActivationFunction(LReLU); if(!cVariablesMLP[1].Init(0, 6, OpenCL, 4 * window_key, 4 * window_key, window_key, window, optimization, iBatch)) ReturnFalse; //--- Position Encoder if(!cStatePE.Init(0, 7, OpenCL, cStateMLP[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cVariablesPE.Init(0, 8, OpenCL, cVariablesMLP[1].Neurons(), optimization, iBatch)) ReturnFalse; //--- Context if(!cStateToState.Init(0, 9, OpenCL, window_key, window_key, heads, heads / 2, units_count, 2, 1, optimization, iBatch)) ReturnFalse; if(!cStateToVariable.Init(0, 10, OpenCL, window_key, window_key, heads, window_key, heads / 2, units_count, window, 2, 1, optimization, iBatch)) ReturnFalse; if(!cVariableToState.Init(0, 11, OpenCL, window_key, window_key, heads, window_key, heads / 2, window, units_count, 2, 1, optimization, iBatch)) ReturnFalse; if(!cContext.Init(0, 12, OpenCL, window_key * (units_count + window), optimization, iBatch)) ReturnFalse; if(!cContextMLP[0].Init(0, 13, OpenCL, window_key, window_key, 4 * window_key, window + units_count, optimization, iBatch)) ReturnFalse; cContextMLP[0].SetActivationFunction(LReLU); if(!cContextMLP[1].Init(0, 14, OpenCL, 4 * window_key, 4 * window_key, window_key, window + units_count, optimization, iBatch)) ReturnFalse; //--- if(!cHighLevelInteraction.Init(0, 15, OpenCL, window_key, window_key, heads, heads / 2, window + units_count, 4, 2, optimization, iBatch)) ReturnFalse; for(int i = 0; i < int(caMamba.Size()); i++) { if(!caMamba[i].Init(0, 16 + i, OpenCL, window_key, 2 * window_key, window + units_count, optimization, iBatch)) ReturnFalse; } if(!cLaneAware.Init(0, 19, OpenCL, window_key, window_key, heads, window_key, heads / 2, window, window + units_count, 2, 1, optimization, iBatch)) ReturnFalse; //--- if(!caForecastMLP[0].Init(0, 20, OpenCL, window_key, window_key, 4 * forecast, window, optimization, iBatch)) ReturnFalse; caForecastMLP[0].SetActivationFunction(LReLU); if(!caForecastMLP[1].Init(0, 21, OpenCL, 4 * forecast, 4 * forecast, forecast, window, optimization, iBatch)) ReturnFalse; caForecastMLP[1].SetActivationFunction(TANH); if(!cTransposeOut.Init(0, 22, OpenCL, window, forecast, optimization, iBatch)) ReturnFalse; //--- SetOutput(cTransposeOut.getOutput(), true); SetGradient(cTransposeOut.getGradient(), true); SetActivationFunction((ENUM_ACTIVATION)caForecastMLP[1].Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- State Encoder if(!cStateRNN.FeedForward(NeuronOCL)) ReturnFalse; if(!cStateMLP[0].FeedForward(cStateRNN.AsObject())) ReturnFalse; if(!cStateMLP[1].FeedForward(cStateMLP[0].AsObject())) ReturnFalse; //--- Variables Encoder if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cVariablesRNN.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cVariablesMLP[0].FeedForward(cVariablesRNN.AsObject())) ReturnFalse; if(!cVariablesMLP[1].FeedForward(cVariablesMLP[0].AsObject())) ReturnFalse; //--- Position Encoder if(!cStatePE.FeedForward(cStateMLP[1].AsObject())) ReturnFalse; if(!cVariablesPE.FeedForward(cVariablesMLP[1].AsObject())) ReturnFalse; //--- Context if(!cStateToState.FeedForward(cStatePE.AsObject())) ReturnFalse; if(!cStateToVariable.FeedForward(cStateToState.AsObject(), cVariablesPE.getOutput())) ReturnFalse; if(!cVariableToState.FeedForward(cVariablesPE.AsObject(), cStateToVariable.getOutput())) ReturnFalse; if(!Concat(cStateToVariable.getOutput(), cVariableToState.getOutput(), cContext.getOutput(), cStateToVariable.Neurons(), cVariableToState.Neurons(), 1)) ReturnFalse; if(!cContextMLP[0].FeedForward(cContext.AsObject())) ReturnFalse; if(!cContextMLP[1].FeedForward(cContextMLP[0].AsObject())) ReturnFalse; //--- Lane aware if(!cHighLevelInteraction.FeedForward(cContextMLP[1].AsObject())) ReturnFalse; if(!caMamba[0].FeedForward(cHighLevelInteraction.AsObject())) ReturnFalse; for(int i = 1; i < int(caMamba.Size()); i++) { if(!caMamba[i].FeedForward(caMamba[i - 1].AsObject())) ReturnFalse; } if(!cLaneAware.FeedForward(cVariablesPE.AsObject(), caMamba[caMamba.Size() - 1].getOutput())) ReturnFalse; //--- Forecast if(!caForecastMLP[0].FeedForward(cLaneAware.AsObject())) ReturnFalse; if(!caForecastMLP[1].FeedForward(caForecastMLP[0].AsObject())) ReturnFalse; if(!cTransposeOut.FeedForward(caForecastMLP[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Forecast if(!caForecastMLP[1].CalcHiddenGradients(cTransposeOut.AsObject())) ReturnFalse; if(!caForecastMLP[0].CalcHiddenGradients(caForecastMLP[1].AsObject())) ReturnFalse; //--- Lane aware if(!cLaneAware.CalcHiddenGradients(caForecastMLP[0].AsObject())) ReturnFalse; if(!cVariablesPE.CalcHiddenGradients(cLaneAware.AsObject(), caMamba[caMamba.Size() - 1].getOutput(), caMamba[caMamba.Size() - 1].getGradient(), (ENUM_ACTIVATION)caMamba[caMamba.Size() - 1].Activation())) ReturnFalse; //--- for(int i = int(caMamba.Size()) - 2; i >= 0; i--) if(!caMamba[i].CalcHiddenGradients(caMamba[i + 1].AsObject())) ReturnFalse; if(!cHighLevelInteraction.CalcHiddenGradients(caMamba[0].AsObject())) ReturnFalse; //--- Context if(!cContextMLP[1].CalcHiddenGradients(cHighLevelInteraction.AsObject())) ReturnFalse; if(!cContextMLP[0].CalcHiddenGradients(cContextMLP[1].AsObject())) ReturnFalse; if(!cContext.CalcHiddenGradients(cContextMLP[0].AsObject())) ReturnFalse; if(!DeConcat(cStatePE.getGradient(), cVariableToState.getGradient(), cContext.getGradient(), cStateToVariable.Neurons(), cVariableToState.Neurons(), 1)) ReturnFalse; if(!SumAndNormilize(cVariablesPE.getGradient(), cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), 1, false)) ReturnFalse; if(!cVariablesPE.CalcHiddenGradients(cVariableToState.AsObject(), cStateToVariable.getOutput(), cStateToVariable.getGradient(), (ENUM_ACTIVATION)cStateToVariable.Activation())) ReturnFalse; if(!SumAndNormilize(cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), cVariablesMLP[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cStateToVariable.getGradient(), cStatePE.getGradient(), cStateToVariable.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cStateToState.CalcHiddenGradients(cStateToVariable.AsObject(), cVariablesPE.getOutput(), cVariablesPE.getGradient(), (ENUM_ACTIVATION)cVariablesPE.Activation())) ReturnFalse; if(!SumAndNormilize(cVariablesPE.getGradient(), cVariablesMLP[1].getGradient(), cVariablesPE.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cStatePE.CalcHiddenGradients(cStateToState.AsObject())) ReturnFalse; //--- Position Encoder if(!cStateMLP[1].CalcHiddenGradients(cStatePE.AsObject())) ReturnFalse; if(!cVariablesMLP[1].CalcHiddenGradients(cVariablesPE.AsObject())) ReturnFalse; //--- Variables Encoder if(!cVariablesMLP[0].CalcHiddenGradients(cVariablesMLP[1].AsObject())) ReturnFalse; if(!cVariablesRNN.CalcHiddenGradients(cVariablesMLP[0].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cVariablesRNN.AsObject())) ReturnFalse; if(!NeuronOCL.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getGradient(), cTranspose.getGradient(), 1, false)) ReturnFalse; //--- State Encoder if(!cStateMLP[0].CalcHiddenGradients(cStateMLP[1].AsObject())) ReturnFalse; if(!cStateRNN.CalcHiddenGradients(cStateMLP[0].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cStateRNN.AsObject())) ReturnFalse; if(!SumAndNormilize(cTranspose.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- State Encoder if(!cStateRNN.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cStateMLP[0].UpdateInputWeights(cStateRNN.AsObject())) ReturnFalse; if(!cStateMLP[1].UpdateInputWeights(cStateMLP[0].AsObject())) ReturnFalse; //--- Variables Encoder if(!cVariablesRNN.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; if(!cVariablesMLP[0].UpdateInputWeights(cVariablesRNN.AsObject())) ReturnFalse; if(!cVariablesMLP[1].UpdateInputWeights(cVariablesMLP[0].AsObject())) ReturnFalse; //--- Position Encoder if(!cStatePE.UpdateInputWeights(cStateMLP[1].AsObject())) ReturnFalse; if(!cVariablesPE.UpdateInputWeights(cVariablesMLP[1].AsObject())) ReturnFalse; //--- Context if(!cStateToState.UpdateInputWeights(cStatePE.AsObject())) ReturnFalse; if(!cStateToVariable.UpdateInputWeights(cStateToState.AsObject(), cVariablesPE.getOutput())) ReturnFalse; if(!cVariableToState.UpdateInputWeights(cVariablesPE.AsObject(), cStateToVariable.getOutput())) ReturnFalse; if(!cContextMLP[0].UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cContextMLP[1].UpdateInputWeights(cContextMLP[0].AsObject())) ReturnFalse; //--- Lane aware if(!cHighLevelInteraction.UpdateInputWeights(cContextMLP[1].AsObject())) ReturnFalse; if(!caMamba[0].UpdateInputWeights(cHighLevelInteraction.AsObject())) ReturnFalse; for(int i = 1; i < int(caMamba.Size()); i++) { if(!caMamba[i].UpdateInputWeights(caMamba[i - 1].AsObject())) ReturnFalse; } if(!cLaneAware.UpdateInputWeights(cVariablesPE.AsObject(), caMamba[caMamba.Size() - 1].getOutput())) ReturnFalse; //--- Forecast if(!caForecastMLP[0].UpdateInputWeights(cLaneAware.AsObject())) ReturnFalse; if(!caForecastMLP[1].UpdateInputWeights(caForecastMLP[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- State Encoder if(!cStateRNN.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cStateMLP[i].Save(file_handle)) ReturnFalse; //--- Variables Encoder if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cVariablesRNN.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cVariablesMLP[i].Save(file_handle)) ReturnFalse; //--- Context Encoder if(!cStatePE.Save(file_handle)) ReturnFalse; if(!cVariablesPE.Save(file_handle)) ReturnFalse; if(!cStateToState.Save(file_handle)) ReturnFalse; if(!cVariableToState.Save(file_handle)) ReturnFalse; if(!cStateToVariable.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cContextMLP[i].Save(file_handle)) ReturnFalse; //--- if(!cHighLevelInteraction.Save(file_handle)) ReturnFalse; int total = int(caMamba.Size()); if(FileWriteInteger(file_handle, total) < INT_VALUE) ReturnFalse; for(int i = 0; i < total; i++) if(!caMamba[i].Save(file_handle)) ReturnFalse; if(!cLaneAware.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!caForecastMLP[i].Save(file_handle)) ReturnFalse; if(!cTransposeOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTrajLLMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- State Encoder if(!LoadInsideLayer(file_handle, cStateRNN.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cStateMLP[i].AsObject())) ReturnFalse; //--- Variables Encoder if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariablesRNN.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cVariablesMLP[i].AsObject())) ReturnFalse; //--- Context Encoder if(!LoadInsideLayer(file_handle, cStatePE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariablesPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVariableToState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToVariable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cContext.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cContextMLP[i].AsObject())) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cHighLevelInteraction.AsObject())) ReturnFalse; int total = FileReadInteger(file_handle); if(total != int(caMamba.Size())) { ReturnFalse; } for(int i = 0; i < total; i++) if(!LoadInsideLayer(file_handle, caMamba[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLaneAware.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, caForecastMLP[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTrajLLMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- State Encoder cStateRNN.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cStateMLP[i].SetOpenCL(OpenCL); //--- Variables Encoder cTranspose.SetOpenCL(OpenCL); cVariablesRNN.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cVariablesMLP[i].SetOpenCL(OpenCL); //--- Context Encoder cStatePE.SetOpenCL(OpenCL); cVariablesPE.SetOpenCL(OpenCL); cStateToState.SetOpenCL(OpenCL); cVariableToState.SetOpenCL(OpenCL); cStateToVariable.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cContextMLP[i].SetOpenCL(OpenCL); //--- cHighLevelInteraction.SetOpenCL(OpenCL); int total = int(caMamba.Size()); for(int i = 0; i < total; i++) caMamba[i].SetOpenCL(OpenCL); cLaneAware.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) caForecastMLP[i].SetOpenCL(OpenCL); cTransposeOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronUniTraj : public CNeuronBaseOCL { protected: uint iVariables; float fDropout; //--- CBufferFloat cHistoryMask; CBufferFloat cFutureMask; CNeuronBaseOCL cData; CNeuronLearnabledPE cPE; CNeuronMVMHAttentionMLKV cEncoder; CNeuronBaseOCL cDForw; CNeuronBaseOCL cDBakw; CNeuronConvOCL cProjDForw; CNeuronConvOCL cProjDBakw; CNeuronBaseOCL cDataDForw; CNeuronBaseOCL cDataDBakw; CNeuronBaseOCL cConcatDataDForwBakw; CNeuronMambaBlockOCL cSSM[4]; CNeuronConvOCL cStat; CNeuronTransposeOCL cTranspStat; CVAE cVAE; CNeuronTransposeOCL cTranspVAE; CNeuronConvOCL cDecoder[2]; CNeuronTransposeOCL cTranspResult; //--- virtual bool Prepare(const CBufferFloat* history, const CBufferFloat* future); virtual bool PrepareGrad(CBufferFloat* history_gr, CBufferFloat* future_gr); virtual bool BTS(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { return feedForward(NeuronOCL, NULL); } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return calcInputGradients(NeuronOCL, NULL, NULL, None); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return updateInputWeights(NeuronOCL, NULL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; //--- public: CNeuronUniTraj(void) {}; ~CNeuronUniTraj(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronUniTrajOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * (units_count + forecast), optimization_type, batch)) ReturnFalse; //--- iVariables = window; fDropout = MathMax(MathMin(dropout, 1), 0); //--- if(!cHistoryMask.BufferInit(iVariables * units_count, 1) || !cHistoryMask.BufferCreate(OpenCL)) ReturnFalse; if(!cFutureMask.BufferInit(iVariables * forecast, 1) || !cFutureMask.BufferCreate(OpenCL)) ReturnFalse; if(!cData.Init(0, 0, OpenCL, 3 * iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cPE.Init(0, 1, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cEncoder.Init(0, 2, OpenCL, 3, window_key, heads, (heads + 1) / 2, iVariables, 1, 1, (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cDForw.Init(0, 3, OpenCL, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse;; if(!cDBakw.Init(0, 4, OpenCL, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; if(!cProjDForw.Init(0, 5, OpenCL, 1, 1, 3, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; cProjDForw.SetActivationFunction(SIGMOID); if(!cProjDBakw.Init(0, 6, OpenCL, 1, 1, 3, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; cProjDBakw.SetActivationFunction(SIGMOID); if(!cDataDForw.Init(0, 7, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cDataDBakw.Init(0, 8, OpenCL, cData.Neurons(), optimization, iBatch)) ReturnFalse; if(!cConcatDataDForwBakw.Init(0, 9, OpenCL, 2 * cData.Neurons(), optimization, iBatch)) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) { if(!cSSM[i].Init(0, 10 + i, OpenCL, 6 * iVariables, 12 * iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; } uint id = 10 + cSSM.Size(); if(!cStat.Init(0, id, OpenCL, 6, 6, 12, iVariables * (units_count + forecast), optimization, iBatch)) ReturnFalse; id++; if(!cTranspStat.Init(0, id, OpenCL, iVariables * (units_count + forecast), 12, optimization, iBatch)) ReturnFalse; id++; if(!cVAE.Init(0, id, OpenCL, cTranspStat.Neurons() / 2, optimization, iBatch)) ReturnFalse; id++; if(!cTranspVAE.Init(0, id, OpenCL, cVAE.Neurons() / iVariables, iVariables, optimization, iBatch)) ReturnFalse; id++; uint w = cTranspVAE.Neurons() / iVariables; if(!cDecoder[0].Init(0, id, OpenCL, w, w, 2 * (units_count + forecast), iVariables, optimization, iBatch)) ReturnFalse; cDecoder[0].SetActivationFunction(LReLU); id++; if(!cDecoder[1].Init(0, id, OpenCL, 2 * (units_count + forecast), 2 * (units_count + forecast), (units_count + forecast), iVariables, optimization, iBatch)) ReturnFalse; cDecoder[1].SetActivationFunction(TANH); id++; if(!cTranspResult.Init(0, id, OpenCL, iVariables, (units_count + forecast), optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cTranspResult.getOutput(), true) || !SetGradient(cTranspResult.getGradient(), true)) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)cDecoder[1].Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Prepare(const CBufferFloat * history, const CBufferFloat * future) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {MathMax(cHistoryMask.Total(), cFutureMask.Total()) / iVariables, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajPrepare, def_k_utp_h_mask, cHistoryMask.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_history, history.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_f_mask, cFutureMask.GetIndex()) setBuffer(def_k_UniTrajPrepare, def_k_utp_future, (!!future ? future.GetIndex() : cFutureMask.GetIndex())) setBuffer(def_k_UniTrajPrepare, def_k_utp_output, cData.getOutputIndex()) setArgument(def_k_UniTrajPrepare, def_k_utp_h_total, cHistoryMask.Total()) setArgument(def_k_UniTrajPrepare, def_k_utp_f_total, cFutureMask.Total()) kernelExecute(def_k_UniTrajPrepare, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::PrepareGrad(CBufferFloat * history_gr, CBufferFloat * future_gr) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {MathMax(cHistoryMask.Total(), cFutureMask.Total()) / iVariables, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_history_gr, history_gr.GetIndex()) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_future_gr, (!!future_gr ? future_gr.GetIndex() : cFutureMask.GetIndex())) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_output, cData.getOutputIndex()) setBuffer(def_k_UniTrajPrepareGrad, def_k_utpg_output_gr, cData.getGradientIndex()) setArgument(def_k_UniTrajPrepareGrad, def_k_utpg_h_total, cHistoryMask.Total()) setArgument(def_k_UniTrajPrepareGrad, def_k_utpg_f_total, cFutureMask.Total()) kernelExecute(def_k_UniTrajPrepareGrad, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::BTS(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {2, iVariables}; ResetLastError(); setBuffer(def_k_UniTrajBTS, def_k_utbts_concat_inp, cData.getOutputIndex()) setBuffer(def_k_UniTrajBTS, def_k_utbts_d_forw, cDForw.getOutputIndex()) setBuffer(def_k_UniTrajBTS, def_k_utbts_d_bakw, cDBakw.getOutputIndex()) setArgument(def_k_UniTrajBTS, def_k_utbts_total, int(cDForw.Neurons() / iVariables)) kernelExecute(def_k_UniTrajBTS, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!NeuronOCL) ReturnFalse; //--- Create History Mask int total = cHistoryMask.Total(); if(!cHistoryMask.BufferInit(total, 1)) ReturnFalse; if(bTrain) { for(int i = 0; i < int(total * fDropout); i++) cHistoryMask.Update(RND(total), 0); } if(!cHistoryMask.BufferWrite()) ReturnFalse; //--- Create Future Mask total = cFutureMask.Total(); if(!cFutureMask.BufferInit(total, (!SecondInput ? 0 : 1))) ReturnFalse; if(bTrain && !!SecondInput) { for(int i = 0; i < int(total * fDropout); i++) cFutureMask.Update(RND(total), 0); } if(!cFutureMask.BufferWrite()) ReturnFalse; //--- Prepare Data if(!Prepare(NeuronOCL.getOutput(), SecondInput)) ReturnFalse; //--- Encoder if(!cPE.FeedForward(cData.AsObject())) ReturnFalse; if(!cEncoder.FeedForward(cPE.AsObject())) ReturnFalse; //--- BTS if(!BTS()) ReturnFalse; if(!cProjDForw.FeedForward(cDForw.AsObject())) ReturnFalse; if(!cProjDBakw.FeedForward(cDBakw.AsObject())) ReturnFalse; if(!ElementMult(cEncoder.getOutput(), cProjDForw.getOutput(), cDataDForw.getOutput())) ReturnFalse; if(!ElementMult(cEncoder.getOutput(), cProjDBakw.getOutput(), cDataDBakw.getOutput())) ReturnFalse; if(!Concat(cDataDForw.getOutput(), cDataDBakw.getOutput(), cConcatDataDForwBakw.getOutput(), 3, 3, cData.Neurons() / 3)) ReturnFalse; //--- SSM if(!cSSM[0].FeedForward(cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 1; i < cSSM.Size(); i++) if(!cSSM[i].FeedForward(cSSM[i - 1].AsObject())) ReturnFalse; //--- VAE if(!cStat.FeedForward(cSSM[cSSM.Size() - 1].AsObject())) ReturnFalse; if(!cTranspStat.FeedForward(cStat.AsObject())) ReturnFalse; if(!cVAE.FeedForward(cTranspStat.AsObject())) ReturnFalse; //--- Decoder if(!cTranspVAE.FeedForward(cVAE.AsObject())) ReturnFalse; if(!cDecoder[0].FeedForward(cTranspVAE.AsObject())) ReturnFalse; if(!cDecoder[1].FeedForward(cDecoder[0].AsObject())) ReturnFalse; if(!cTranspResult.FeedForward(cDecoder[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { //--- Decoder if(!cDecoder[1].CalcHiddenGradients(cTranspResult.AsObject())) ReturnFalse; if(!cDecoder[0].CalcHiddenGradients(cDecoder[1].AsObject())) ReturnFalse; if(!cTranspVAE.CalcHiddenGradients(cDecoder[0].AsObject())) ReturnFalse; if(!cVAE.CalcHiddenGradients(cTranspVAE.AsObject())) ReturnFalse; //--- VAE if(!cTranspStat.CalcHiddenGradients(cVAE.AsObject())) ReturnFalse; if(!cStat.CalcHiddenGradients(cTranspStat.AsObject())) ReturnFalse; if(!cSSM[cSSM.Size() - 1].CalcHiddenGradients(cStat.AsObject())) ReturnFalse; //--- SSM for(int i = (int)cSSM.Size() - 2; i >= 0; i--) if(!cSSM[i].CalcHiddenGradients(cSSM[i + 1].AsObject())) ReturnFalse; if(!cConcatDataDForwBakw.CalcHiddenGradients(cSSM[0].AsObject())) ReturnFalse; //--- BTS if(!DeConcat(cDataDForw.getGradient(), cDataDBakw.getGradient(), cConcatDataDForwBakw.getGradient(), 3, 3, cData.Neurons() / 3)) ReturnFalse; if(!ElementMultGrad(cEncoder.getOutput(), cEncoder.getGradient(), cProjDForw.getOutput(), cProjDForw.getGradient(), cDataDForw.getGradient(), cEncoder.Activation(), cProjDForw.Activation())) ReturnFalse; if(!ElementMultGrad(cEncoder.getOutput(), cPE.getGradient(), cProjDBakw.getOutput(), cProjDBakw.getGradient(), cDataDBakw.getGradient(), cEncoder.Activation(), cProjDBakw.Activation())) ReturnFalse; if(!SumAndNormilize(cEncoder.getGradient(), cPE.getGradient(), cEncoder.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cDForw.CalcHiddenGradients(cProjDForw.AsObject())) ReturnFalse; if(!cDBakw.CalcHiddenGradients(cProjDBakw.AsObject())) ReturnFalse; //--- Encoder if(!cPE.CalcHiddenGradients(cEncoder.AsObject())) ReturnFalse; if(!cData.CalcHiddenGradients(cPE.AsObject())) ReturnFalse; //--- Prepare Data if(!PrepareGrad(NeuronOCL.getGradient(), SecondGradient)) ReturnFalse; //--- Deactivation Gradient if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; } if(SecondActivation != None && !!SecondInput && !!SecondGradient) { if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * second) { if(!NeuronOCL) ReturnFalse; //--- Encoder if(!cPE.UpdateInputWeights(cData.AsObject())) ReturnFalse; if(!cEncoder.UpdateInputWeights(cPE.AsObject())) ReturnFalse; //--- BTS if(!cProjDForw.UpdateInputWeights(cDForw.AsObject())) ReturnFalse; if(!cProjDBakw.UpdateInputWeights(cDBakw.AsObject())) ReturnFalse; //--- SSM if(!cSSM[0].UpdateInputWeights(cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 1; i < cSSM.Size(); i++) if(!cSSM[i].UpdateInputWeights(cSSM[i - 1].AsObject())) ReturnFalse; //--- VAE if(!cStat.UpdateInputWeights(cSSM[cSSM.Size() - 1].AsObject())) ReturnFalse; //--- Decoder if(!cDecoder[0].UpdateInputWeights(cTranspVAE.AsObject())) ReturnFalse; if(!cDecoder[1].UpdateInputWeights(cDecoder[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteFloat(file_handle, fDropout) < sizeof(float)) ReturnFalse; //--- if(!cHistoryMask.Save(file_handle)) ReturnFalse; if(!cFutureMask.Save(file_handle)) ReturnFalse; if(!cData.Save(file_handle)) ReturnFalse; if(!cPE.Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cDForw.Save(file_handle)) ReturnFalse; if(!cDBakw.Save(file_handle)) ReturnFalse; if(!cProjDForw.Save(file_handle)) ReturnFalse; if(!cProjDBakw.Save(file_handle)) ReturnFalse; if(!cDataDForw.Save(file_handle)) ReturnFalse; if(!cDataDBakw.Save(file_handle)) ReturnFalse; if(!cConcatDataDForwBakw.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) if(!cSSM[i].Save(file_handle)) ReturnFalse; if(!cStat.Save(file_handle)) ReturnFalse; if(!cTranspStat.Save(file_handle)) ReturnFalse; if(!cVAE.Save(file_handle)) ReturnFalse; if(!cTranspVAE.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cDecoder[i].Save(file_handle)) ReturnFalse; if(!cTranspResult.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronUniTraj::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; fDropout = FileReadFloat(file_handle); //--- if(!cHistoryMask.Load(file_handle)) ReturnFalse; if(!cFutureMask.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cData.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDataDForw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDataDBakw.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatDataDForwBakw.AsObject())) ReturnFalse; for(uint i = 0; i < cSSM.Size(); i++) if(!LoadInsideLayer(file_handle, cSSM[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStat.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspStat.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cVAE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspVAE.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cDecoder[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspResult.AsObject())) ReturnFalse; //--- if(!SetOutput(cTranspResult.getOutput(), true) || !SetGradient(cTranspResult.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronUniTraj::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cHistoryMask.BufferCreate(OpenCL); cFutureMask.BufferCreate(OpenCL); cData.SetOpenCL(OpenCL); cPE.SetOpenCL(OpenCL); cEncoder.SetOpenCL(OpenCL); cDForw.SetOpenCL(OpenCL); cDBakw.SetOpenCL(OpenCL); cProjDForw.SetOpenCL(OpenCL); cProjDBakw.SetOpenCL(OpenCL); cDataDForw.SetOpenCL(OpenCL); cDataDBakw.SetOpenCL(OpenCL); cConcatDataDForwBakw.SetOpenCL(OpenCL); for(uint i = 0; i < cSSM.Size(); i++) cSSM[i].SetOpenCL(OpenCL); cStat.SetOpenCL(OpenCL); cTranspStat.SetOpenCL(OpenCL); cVAE.SetOpenCL(OpenCL); cTranspVAE.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cDecoder[i].SetOpenCL(OpenCL); cTranspResult.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHiVTAAEncoder : public CNeuronMVMHAttentionMLKV { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHiVTAAEncoder(void) {}; ~CNeuronHiVTAAEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHiVTAAEncoder; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); iVariables = variables; //--- uint num_q = iWindowKey * iHeads * iUnits * iVariables; //Size of Q tensor uint num_kv = iWindowKey * iHeadsKV * iUnits * iVariables; //Size of KV tensor uint q_weights = (iWindow * iHeads + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = (iWindow * iHeadsKV + 1) * iWindowKey; //Size of weights' matrix of Q tenzor uint scores = iUnits * iUnits * iHeads * iVariables; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits * iVariables; //Size of multi-heads self-attention uint out = iWindow * iUnits * iVariables; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint gate = (2 * iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint self = (iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV tensor if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Tensors.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Concatenate temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(2 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Gate temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Self temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Out if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(q_weights)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < q_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; //--- Initilize K weights if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(kv_weights)) ReturnFalse; for(uint w = 0; w < kv_weights; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Gate Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(gate)) ReturnFalse; k = (float)(1 / sqrt(2 * iWindow + 1)); for(uint w = 0; w < gate; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Self temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(self)) ReturnFalse; k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < self; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? q_weights : iWindowKey * iHeads), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!K_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? kv_weights : iWindowKey * iHeadsKV), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!V_Weights.Add(temp)) ReturnFalse; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Gate Momentum temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? gate : 2 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize Self Momentum temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? self : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax(2 * num_kv, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *kv = NULL; for(uint i = 0; (i < iLayers && !IsStopped()); i++) { //--- Calculate Queries, Keys, Values CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(10 * i - 6)); CBufferFloat *q = QKV_Tensors.At(i * 2); if(IsStopped() || !ConvolutionForward(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), inputs, q, iWindow, iWindowKey * iHeads, None)) ReturnFalse; if((i % iLayersToOneKV) == 0) { uint i_kv = i / iLayersToOneKV; kv = KV_Tensors.At(i_kv * 2); CBufferFloat *k = K_Tensors.At(i_kv * 2); CBufferFloat *v = V_Tensors.At(i_kv * 2); if(IsStopped() || !ConvolutionForward(K_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, k, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !ConvolutionForward(V_Weights.At(i_kv * (optimization == SGD ? 2 : 3)), inputs, v, iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !Concat(k, v, kv, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; } //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors.At(i * 2); CBufferFloat *out = AO_Tensors.At(i * 2); if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors.At(i * 10); if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Concat out = FF_Tensors.At(i * 10 + 1); if(IsStopped() || !Concat(temp, inputs, out, iWindow, iWindow, iUnits)) ReturnFalse; //--- Gate if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), out, FF_Tensors.At(i * 10 + 2), 2 * iWindow, iWindow, SIGMOID)) ReturnFalse; //--- Self if(IsStopped() || !ConvolutionForward(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), inputs, FF_Tensors.At(i * 10 + 3), iWindow, iWindow, None)) ReturnFalse; //--- Out if(IsStopped() || !GateElementMult(FF_Tensors.At(i * 10 + 3), temp, FF_Tensors.At(i * 10 + 2), FF_Tensors.At(i * 10 + 4))) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors.At(KV_Tensors.Total() - 1); //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors.At((i / iLayersToOneKV) * 2 + 1); //--- Gate out if(IsStopped() || !GateElementMultGrad(FF_Tensors.At(i * 10 + 3), FF_Tensors.At(i * 10 + 8), FF_Tensors.At(i * 10), FF_Tensors.At(i * 10 + 5), FF_Tensors.At(i * 10 + 2), FF_Tensors.At(i * 10 + 7), out_grad, None, None, SIGMOID)) ReturnFalse; //--- Passing gradient through Self CBufferFloat *inp = NULL; CBufferFloat *inp_gr = NULL; if(i == 0) { inp = prevLayer.getOutput(); inp_gr = prevLayer.getGradient(); } else { inp = FF_Tensors.At(i * 10 - 6); inp_gr = FF_Tensors.At(i * 10 - 1); } if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(i * 10 + 8), inp, inp_gr, iWindow, iWindow, None)) ReturnFalse; //--- Passing gradient through Gate if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 10 + 7), FF_Tensors.At(i * 10 + 1), FF_Tensors.At(i * 10 + 6), 2 * iWindow, iWindow, None)) ReturnFalse; //--- Concat if(IsStopped() || !DeConcat(FF_Tensors.At(i * 10 + 5), GetPointer(Temp), FF_Tensors.At(i * 10 + 6), iWindow, iWindow, iUnits)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), FF_Tensors.At(i * 10 + 5), AO_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1), iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), kv_g, S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors.At(i * 2), QKV_Tensors.At(i * 2 + 1), KV_Tensors.At((i / iLayersToOneKV) * 2), GetPointer(Temp), S_Tensors.At(i * 2), AO_Tensors.At(i * 2 + 1))) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- if(IsStopped() || !ConvolutionInputGradients(QKV_Weights.At(i * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(i * 2 + 1), inp, GetPointer(Temp), iWindow, iWindowKey * iHeads, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { CBufferFloat *k_g = K_Tensors.At((i / iLayersToOneKV) * 2 + 1); CBufferFloat *v_g = V_Tensors.At((i / iLayersToOneKV) * 2 + 1); if(IsStopped() || !DeConcat(k_g, v_g, kv_g, iWindowKey * iHeadsKV * iVariables, iWindowKey * iHeadsKV * iVariables, iUnits)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(K_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), k_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(IsStopped() || !ConvolutionInputGradients(V_Weights.At(i / iLayersToOneKV * (optimization == SGD ? 2 : 3)), v_g, inp, GetPointer(Temp), iWindow, iWindowKey * iHeadsKV, None)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), inp_gr, inp_gr, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) out_grad = inp_gr; } //--- if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTAAEncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; CBufferFloat *inputs = NeuronOCL.getOutput(); for(uint l = 0; l < iLayers; l++) { if(IsStopped() || !ConvolutuionUpdateWeights(QKV_Weights.At(l * (optimization == SGD ? 2 : 3)), QKV_Tensors.At(l * 2 + 1), inputs, (optimization == SGD ? QKV_Weights.At(l * 2 + 1) : QKV_Weights.At(l * 3 + 1)), (optimization == SGD ? NULL : QKV_Weights.At(l * 3 + 2)), iWindow, iWindowKey * iHeads)) ReturnFalse; if(l % iLayersToOneKV == 0) { uint l_kv = l / iLayersToOneKV; if(IsStopped() || !ConvolutuionUpdateWeights(K_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), K_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? K_Weights.At(l_kv * 2 + 1) : K_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : K_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; if(IsStopped() || !ConvolutuionUpdateWeights(V_Weights.At(l_kv * (optimization == SGD ? 2 : 3)), V_Tensors.At(l_kv * 2 + 1), inputs, (optimization == SGD ? V_Weights.At(l_kv * 2 + 1) : V_Weights.At(l_kv * 3 + 1)), (optimization == SGD ? NULL : V_Weights.At(l_kv * 3 + 2)), iWindow, iWindowKey * iHeadsKV, 0, iHeadsKV)) ReturnFalse; } //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9)), FF_Tensors.At(l * 10 + 5), AO_Tensors.At(l * 2), (optimization == SGD ? FF_Weights.At(l * 6 + 3) : FF_Weights.At(l * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(l * 10 + 7), FF_Tensors.At(l * 10 + 1), (optimization == SGD ? FF_Weights.At(l * 6 + 4) : FF_Weights.At(l * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 7)), 2 * iWindow, iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(l * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(l * 10 + 8), inputs, (optimization == SGD ? FF_Weights.At(l * 6 + 5) : FF_Weights.At(l * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(l * 9 + 8)), iWindow, iWindow, 0, 1)) ReturnFalse; inputs = FF_Tensors.At(l * 10 + 4); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTransposeRCDOCL : public CNeuronTransposeOCL { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTransposeRCDOCL(void) {}; ~CNeuronTransposeRCDOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronTransposeRCDOCL; }///< Identificator of class.@return Type of class virtual uint GetDimension(void) const { return Neurons() / (iCount * iWindow); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * dimension, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iCount = count; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iCount, iWindow, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, Output.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeRCDOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iWindow, iCount, (Neurons() / (iCount * iWindow))}; setBuffer(def_k_TransposeRCD, def_k_tr_matrix_out, NeuronOCL.getGradientIndex()) setBuffer(def_k_TransposeRCD, def_k_tr_matrix_in, Gradient.GetIndex()) kernelExecute(def_k_TransposeRCD, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHiVTOCL : public CNeuronBaseOCL { protected: uint iHistory; uint iVariables; uint iForecast; uint iNumTraj; //--- CNeuronBaseOCL cDataTAD; CNeuronConvOCL cEmbeddingTAD; CNeuronTransposeRCDOCL cTransposeATD; CNeuronHiVTAAEncoder cAAEncoder; CNeuronTransposeRCDOCL cTransposeTAD; CNeuronLearnabledPE cPosEmbeddingTAD; CNeuronMVMHAttentionMLKV cTemporalEncoder; CNeuronLearnabledPE cPosLineEmbeddingTAD; CNeuronPatching cLineEmbeddibg; CNeuronMVCrossAttentionMLKV cALEncoder; CNeuronMLMHAttentionMLKV cGlobalEncoder; CNeuronTransposeOCL cTransposeADT; CNeuronConvOCL cDecoder[3]; // Agent * Traj * Forecast CNeuronConvOCL cProbProj; CNeuronSoftMaxOCL cProbability; // Agent * Traj CNeuronBaseOCL cForecast; CNeuronTransposeOCL cTransposeTA; //--- virtual bool Prepare(const CNeuronBaseOCL *history); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHiVTOCL(void) {}; ~CNeuronHiVTOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, uint num_traj, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHiVTOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint forecast, uint num_traj, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units_count < 2 || !CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * forecast, optimization_type, batch)) ReturnFalse; //--- iVariables = window; iHistory = units_count - 1; iForecast = forecast; iNumTraj = MathMax(num_traj, 1); //--- if(!cDataTAD.Init(0, 0, OpenCL, 2 * iVariables * iVariables * iHistory, optimization, iBatch)) ReturnFalse; if(!cEmbeddingTAD.Init(0, 1, OpenCL, 2 * iVariables, 2 * iVariables, window_key, iVariables * iHistory, 1, optimization, iBatch)) ReturnFalse; if(!cTransposeATD.Init(0, 2, OpenCL, iHistory, iVariables, window_key, optimization, iBatch)) ReturnFalse; if(!cAAEncoder.Init(0, 3, OpenCL, window_key, window_key, heads, (heads + 1) / 2, iVariables, 2, 1, iHistory, optimization, iBatch)) ReturnFalse; if(!cTransposeTAD.Init(0, 4, OpenCL, iVariables, iHistory, window_key, optimization, iBatch)) ReturnFalse; if(!cPosEmbeddingTAD.Init(0, 5, OpenCL, iVariables * iHistory * window_key, optimization, iBatch)) ReturnFalse; if(!cTemporalEncoder.Init(0, 6, OpenCL, window_key, window_key, heads, (heads + 1) / 2, iHistory, 2, 1, iVariables, optimization, iBatch)) ReturnFalse; if(!cLineEmbeddibg.Init(0, 7, OpenCL, 3, 1, 8, iHistory - 1, iVariables, optimization, iBatch)) ReturnFalse; if(!cPosLineEmbeddingTAD.Init(0, 8, OpenCL, cLineEmbeddibg.Neurons(), optimization, iBatch)) ReturnFalse; if(!cALEncoder.Init(0, 9, OpenCL, window_key, window_key, heads, 8, (heads + 1) / 2, iHistory, iHistory - 1, 2, 1, iVariables, iVariables, optimization, iBatch)) ReturnFalse; if(!cGlobalEncoder.Init(0, 10, OpenCL, window_key * iVariables, window_key * iVariables, heads, (heads + 1) / 2, iHistory, 4, 2, optimization, iBatch)) ReturnFalse; if(!cTransposeADT.Init(0, 11, OpenCL, iHistory, window_key * iVariables, optimization, iBatch)) ReturnFalse; if(!cDecoder[0].Init(0, 12, OpenCL, iHistory, iHistory, iForecast, window_key * iVariables, optimization, iBatch)) ReturnFalse; cDecoder[0].SetActivationFunction(SIGMOID); if(!cDecoder[1].Init(0, 13, OpenCL, iForecast * window_key, iForecast * window_key, iForecast * window_key, iVariables, optimization, iBatch)) ReturnFalse; cDecoder[1].SetActivationFunction(LReLU); if(!cDecoder[2].Init(0, 14, OpenCL, iForecast * window_key, iForecast * window_key, iForecast * iNumTraj, iVariables, optimization, iBatch)) ReturnFalse; cDecoder[2].SetActivationFunction(TANH); if(!cProbProj.Init(0, 15, OpenCL, iForecast * iNumTraj, iForecast * iNumTraj, iNumTraj, iVariables, optimization, iBatch)) ReturnFalse; if(!cProbability.Init(0, 16, OpenCL, iForecast * iNumTraj * iVariables, optimization, iBatch)) ReturnFalse; cProbability.SetHeads(iVariables); // Agent * Traj if(!cForecast.Init(0, 17, OpenCL, iForecast * iVariables, optimization, iBatch)) ReturnFalse; if(!cTransposeTA.Init(0, 18, OpenCL, iVariables, iForecast, optimization, iBatch)) ReturnFalse; //--- SetOutput(cTransposeTA.getOutput(), true); SetGradient(cTransposeTA.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Prepare(const CNeuronBaseOCL * history) { if(!history) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iHistory, iVariables}; setBuffer(def_k_HiVTPrepare, def_k_hivtp_data, history.getOutputIndex()) setBuffer(def_k_HiVTPrepare, def_k_hivtp_output, cDataTAD.getOutputIndex()) kernelExecute(def_k_HiVTPrepare, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!Prepare(NeuronOCL)) ReturnFalse; if(!cEmbeddingTAD.FeedForward(cDataTAD.AsObject())) ReturnFalse; if(!cTransposeATD.FeedForward(cEmbeddingTAD.AsObject())) ReturnFalse; if(!cAAEncoder.FeedForward(cTransposeATD.AsObject())) ReturnFalse; if(!cTransposeTAD.FeedForward(cAAEncoder.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.FeedForward(cTransposeTAD.AsObject())) ReturnFalse; if(!cTemporalEncoder.FeedForward(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cLineEmbeddibg.FeedForward(NeuronOCL)) ReturnFalse; if(!cPosLineEmbeddingTAD.FeedForward(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cALEncoder.FeedForward(cTemporalEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput())) ReturnFalse; if(!cGlobalEncoder.FeedForward(cALEncoder.AsObject())) ReturnFalse; if(!cTransposeADT.FeedForward(cGlobalEncoder.AsObject())) ReturnFalse; if(!cDecoder[0].FeedForward(cTransposeADT.AsObject())) ReturnFalse; if(!cDecoder[1].FeedForward(cDecoder[0].AsObject())) ReturnFalse; if(!cDecoder[2].FeedForward(cDecoder[1].AsObject())) ReturnFalse; if(!cProbProj.FeedForward(cDecoder[2].AsObject())) ReturnFalse; if(!cProbability.FeedForward(cProbProj.AsObject())) ReturnFalse; if(IsStopped() || !MatMul(cDecoder[2].getOutput(), cProbability.getOutput(), cForecast.getOutput(), iForecast, iNumTraj, 1, iVariables)) ReturnFalse; if(!cTransposeTA.FeedForward(cForecast.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cForecast.CalcHiddenGradients(cTransposeTA.AsObject())) ReturnFalse; if(IsStopped() || !MatMulGrad(cDecoder[2].getOutput(), cDecoder[2].getGradient(), cProbability.getOutput(), cProbability.getGradient(), cForecast.getGradient(), iForecast, iNumTraj, 1, iVariables)) ReturnFalse; if(!cProbProj.CalcHiddenGradients(cProbability.AsObject())) ReturnFalse; if(!cDecoder[1].CalcHiddenGradients(cDecoder[2].AsObject())) ReturnFalse; if(!cDecoder[0].CalcHiddenGradients(cDecoder[1].AsObject())) ReturnFalse; if(!cTransposeADT.CalcHiddenGradients(cDecoder[0].AsObject())) ReturnFalse; if(!cGlobalEncoder.CalcHiddenGradients(cTransposeADT.AsObject())) ReturnFalse; if(!cALEncoder.CalcHiddenGradients(cGlobalEncoder.AsObject())) ReturnFalse; if(!cTemporalEncoder.CalcHiddenGradients(cALEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput(), cPosLineEmbeddingTAD.getGradient(), (ENUM_ACTIVATION)cPosLineEmbeddingTAD.Activation())) ReturnFalse; if(!cLineEmbeddibg.CalcHiddenGradients(cPosLineEmbeddingTAD.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.CalcHiddenGradients(cTemporalEncoder.AsObject())) ReturnFalse; if(!cTransposeTAD.CalcHiddenGradients(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cAAEncoder.CalcHiddenGradients(cTransposeTAD.AsObject())) ReturnFalse; if(!cTransposeATD.CalcHiddenGradients(cAAEncoder.AsObject())) ReturnFalse; if(!cEmbeddingTAD.CalcHiddenGradients(cTransposeATD.AsObject())) ReturnFalse; if(!cDataTAD.CalcHiddenGradients(cEmbeddingTAD.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cEmbeddingTAD.UpdateInputWeights(cDataTAD.AsObject())) ReturnFalse; if(!cAAEncoder.UpdateInputWeights(cTransposeATD.AsObject())) ReturnFalse; if(!cPosEmbeddingTAD.UpdateInputWeights(cTransposeTAD.AsObject())) ReturnFalse; if(!cTemporalEncoder.UpdateInputWeights(cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!cLineEmbeddibg.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPosLineEmbeddingTAD.UpdateInputWeights(cLineEmbeddibg.AsObject())) ReturnFalse; if(!cALEncoder.UpdateInputWeights(cTemporalEncoder.AsObject(), cPosLineEmbeddingTAD.getOutput())) ReturnFalse; if(!cGlobalEncoder.UpdateInputWeights(cALEncoder.AsObject())) ReturnFalse; if(!cDecoder[0].UpdateInputWeights(cTransposeADT.AsObject())) ReturnFalse; if(!cDecoder[1].UpdateInputWeights(cDecoder[0].AsObject())) ReturnFalse; if(!cDecoder[2].UpdateInputWeights(cDecoder[1].AsObject())) ReturnFalse; if(!cProbProj.UpdateInputWeights(cDecoder[2].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHistory)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iForecast)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iNumTraj)) < INT_VALUE) ReturnFalse; //--- if(!cDataTAD.Save(file_handle)) ReturnFalse; if(!cEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cTransposeATD.Save(file_handle)) ReturnFalse; if(!cAAEncoder.Save(file_handle)) ReturnFalse; if(!cTransposeTAD.Save(file_handle)) ReturnFalse; if(!cPosEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cTemporalEncoder.Save(file_handle)) ReturnFalse; if(!cLineEmbeddibg.Save(file_handle)) ReturnFalse; if(!cPosLineEmbeddingTAD.Save(file_handle)) ReturnFalse; if(!cALEncoder.Save(file_handle)) ReturnFalse; if(!cGlobalEncoder.Save(file_handle)) ReturnFalse; if(!cTransposeADT.Save(file_handle)) ReturnFalse; if(!cDecoder[0].Save(file_handle)) ReturnFalse; if(!cDecoder[1].Save(file_handle)) ReturnFalse; if(!cDecoder[2].Save(file_handle)) ReturnFalse; if(!cProbProj.Save(file_handle)) ReturnFalse; if(!cProbability.Save(file_handle)) ReturnFalse; if(!cForecast.Save(file_handle)) ReturnFalse; if(!cTransposeTA.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiVTOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHistory = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iForecast = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iNumTraj = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cDataTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeATD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAAEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPosEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLineEmbeddibg.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPosLineEmbeddingTAD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cALEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGlobalEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeADT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDecoder[2].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProbProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProbability.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cForecast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeTA.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHiVTOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDataTAD.SetOpenCL(OpenCL); cEmbeddingTAD.SetOpenCL(OpenCL); cTransposeATD.SetOpenCL(OpenCL); cAAEncoder.SetOpenCL(OpenCL); cTransposeTAD.SetOpenCL(OpenCL); cPosEmbeddingTAD.SetOpenCL(OpenCL); cTemporalEncoder.SetOpenCL(OpenCL); cLineEmbeddibg.SetOpenCL(OpenCL); cPosLineEmbeddingTAD.SetOpenCL(OpenCL); cALEncoder.SetOpenCL(OpenCL); cGlobalEncoder.SetOpenCL(OpenCL); cTransposeADT.SetOpenCL(OpenCL); cDecoder[0].SetOpenCL(OpenCL); cDecoder[1].SetOpenCL(OpenCL); cDecoder[2].SetOpenCL(OpenCL); cProbProj.SetOpenCL(OpenCL); cProbability.SetOpenCL(OpenCL); cForecast.SetOpenCL(OpenCL); cTransposeTA.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPointNetOCL : public CNeuronBaseOCL { protected: CNeuronPointNetOCL *cTNet1; CNeuronBaseOCL *cTurned1; CNeuronConvOCL cPreNet[2]; CNeuronBatchNormOCL cPreNetNorm[2]; CNeuronPointNetOCL *cTNet2; CNeuronBaseOCL *cTurned2; CNeuronConvOCL cFeatureNet[3]; CNeuronBatchNormOCL cFeatureNetNorm[3]; CNeuronTransposeOCL cTranspose; CNeuronProofOCL cMaxPool; CNeuronBaseOCL cFinalMLP[2]; //--- virtual bool OrthoganalLoss(CNeuronBaseOCL *NeuronOCL, bool add = false); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPointNetOCL(void) {}; ~CNeuronPointNetOCL(void); //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronPointNetOCL; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronPointNetOCL::~CNeuronPointNetOCL(void) { DeleteObj(cTNet1); DeleteObj(cTNet2); DeleteObj(cTurned1); DeleteObj(cTurned2); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, output, optimization_type, batch)) ReturnFalse; //--- Init T-Nets if(use_tnets) { if(!cTNet1) { cTNet1 = new CNeuronPointNetOCL(); if(!cTNet1) ReturnFalse; } if(!cTNet1.Init(0, 0, OpenCL, window, units_count, window * window, false, optimization, iBatch)) ReturnFalse; if(!cTurned1) { cTurned1 = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } if(!cTurned1.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; if(!cTNet2) { cTNet2 = new CNeuronPointNetOCL(); if(!cTNet2) ReturnFalse; } if(!cTNet2.Init(0, 2, OpenCL, 64, units_count, 64 * 64, false, optimization, iBatch)) ReturnFalse; if(!cTurned2) { cTurned2 = new CNeuronBaseOCL(); if(!cTurned2) ReturnFalse; } if(!cTurned2.Init(0, 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } //--- Init PreNet if(!cPreNet[0].Init(0, 0, OpenCL, window, window, 64, units_count, optimization, iBatch)) ReturnFalse; cPreNet[0].SetActivationFunction(None); if(!cPreNetNorm[0].Init(0, 1, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cPreNetNorm[0].SetActivationFunction(LReLU); if(!cPreNet[1].Init(0, 2, OpenCL, 64, 64, 64, units_count, optimization, iBatch)) ReturnFalse; cPreNet[1].SetActivationFunction(None); if(!cPreNetNorm[1].Init(0, 3, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cPreNetNorm[1].SetActivationFunction(None); //--- Init Feature Net if(!cFeatureNet[0].Init(0, 4, OpenCL, 64, 64, 64, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[0].SetActivationFunction(None); if(!cFeatureNetNorm[0].Init(0, 5, OpenCL, 64 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNet[0].SetActivationFunction(LReLU); if(!cFeatureNet[1].Init(0, 6, OpenCL, 64, 64, 128, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[1].SetActivationFunction(None); if(!cFeatureNetNorm[1].Init(0, 7, OpenCL, 128 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[1].SetActivationFunction(LReLU); if(!cFeatureNet[2].Init(0, 8, OpenCL, 128, 128, 512, units_count, optimization, iBatch)) ReturnFalse; cFeatureNet[2].SetActivationFunction(None); if(!cFeatureNetNorm[2].Init(0, 9, OpenCL, 512 * units_count, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[2].SetActivationFunction(None); if(!cTranspose.Init(0, 10, OpenCL, units_count, 512, optimization, iBatch)) ReturnFalse; if(!cMaxPool.Init(512, 11, OpenCL, units_count, units_count, 512, optimization, iBatch)) ReturnFalse; //--- Init Final MLP if(!cFinalMLP[0].Init(256, 12, OpenCL, 512, optimization, iBatch)) ReturnFalse; cFinalMLP[0].SetActivationFunction(LReLU); if(!cFinalMLP[1].Init(output, 13, OpenCL, 256, optimization, iBatch)) ReturnFalse; cFinalMLP[1].SetActivationFunction(LReLU); //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- PreNet if(!cTNet1) { if(!cPreNet[0].FeedForward(NeuronOCL)) ReturnFalse; } else { if(!cTurned1) ReturnFalse; if(!cTNet1.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNet1.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNet1.getOutput(), cTurned1.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!cPreNet[0].FeedForward(cTurned1.AsObject())) ReturnFalse; } if(!cPreNetNorm[0].FeedForward(cPreNet[0].AsObject())) ReturnFalse; if(!cPreNet[1].FeedForward(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cPreNetNorm[1].FeedForward(cPreNet[1].AsObject())) ReturnFalse; //--- Feature Net if(!cTNet2) { if(!cFeatureNet[0].FeedForward(cPreNetNorm[1].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTNet2.FeedForward(cPreNetNorm[1].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet2.Neurons()); if(IsStopped() || !MatMul(cPreNetNorm[1].getOutput(), cTNet2.getOutput(), cTurned2.getOutput(), cPreNetNorm[1].Neurons() / window, window, window)) ReturnFalse; if(!cFeatureNet[0].FeedForward(cTurned2.AsObject())) ReturnFalse; } if(!cFeatureNetNorm[0].FeedForward(cFeatureNet[0].AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = 1; i < total; i++) { if(!cFeatureNet[i].FeedForward(cFeatureNetNorm[i - 1].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i].FeedForward(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cTranspose.FeedForward(cFeatureNetNorm[total - 1].AsObject())) ReturnFalse; if(!cMaxPool.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cFinalMLP[0].FeedForward(cMaxPool.AsObject())) ReturnFalse; if(!cFinalMLP[1].FeedForward(cFinalMLP[0].AsObject())) ReturnFalse; if(!CNeuronBaseOCL::feedForward(cFinalMLP[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::OrthoganalLoss(CNeuronBaseOCL * NeuronOCL, bool add = false) { if(!NeuronOCL) ReturnFalse; //--- uint dimension = (uint)MathSqrt(NeuronOCL.Neurons()); uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {dimension, dimension}; uint local_work_size[2] = {1, dimension}; int kernel = def_k_OrthoganalLoss; setBuffer(kernel, def_k_ol_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ol_grad, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_ol_add, (int)add) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronBaseOCL::calcInputGradients(cFinalMLP[1].AsObject())) ReturnFalse; if(!cFinalMLP[0].CalcHiddenGradients(cFinalMLP[1].AsObject())) ReturnFalse; //--- Feature Net if(!cMaxPool.CalcHiddenGradients(cFinalMLP[0].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cMaxPool.AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = total - 1; i > 0; i--) { if(!cFeatureNet[i].CalcHiddenGradients(cFeatureNetNorm[i].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i - 1].CalcHiddenGradients(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFeatureNet[0].CalcHiddenGradients(cFeatureNetNorm[0].AsObject())) ReturnFalse; if(!cTNet2) { if(!cPreNetNorm[1].CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTurned2.CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet2.Neurons()); if(IsStopped() || !MatMulGrad(cPreNetNorm[1].getOutput(), cPreNet[1].getGradient(), cTNet2.getOutput(), cTNet2.getGradient(), cTurned2.getGradient(), cPreNetNorm[1].Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNet2.AsObject(), true)) ReturnFalse; if(!cPreNetNorm[1].CalcHiddenGradients((CObject*)cTNet2)) ReturnFalse; if(!SumAndNormilize(cPreNetNorm[1].getGradient(), cPreNet[1].getGradient(), cPreNetNorm[1].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- PreNet if(!cPreNet[1].CalcHiddenGradients(cPreNetNorm[1].AsObject())) ReturnFalse; if(!cPreNetNorm[0].CalcHiddenGradients(cPreNet[1].AsObject())) ReturnFalse; if(!cPreNet[0].CalcHiddenGradients(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cTNet1) { if(!NeuronOCL.CalcHiddenGradients(cPreNet[0].AsObject())) ReturnFalse; } else { if(!cTurned1) ReturnFalse; if(!cTurned1.CalcHiddenGradients(cPreNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNet1.Neurons()); if(IsStopped() || !MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTNet1.getOutput(), cTNet1.getGradient(), cTurned1.getGradient(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNet1, true)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); NeuronOCL.SetGradient(cTurned1.getGradient(), false); cTurned1.SetGradient(temp, false); //--- if(!NeuronOCL.CalcHiddenGradients(cTNet1.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cTurned1.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- PreNet if(!cTNet1) { if(!cPreNet[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; } else { if(!cTNet1.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPreNet[0].UpdateInputWeights(cTurned1.AsObject())) ReturnFalse; } if(!cPreNetNorm[0].UpdateInputWeights(cPreNet[0].AsObject())) ReturnFalse; if(!cPreNet[1].UpdateInputWeights(cPreNetNorm[0].AsObject())) ReturnFalse; if(!cPreNetNorm[1].UpdateInputWeights(cPreNet[1].AsObject())) ReturnFalse; //--- Feature Net if(!cTNet2) { if(!cFeatureNet[0].UpdateInputWeights(cPreNetNorm[1].AsObject())) ReturnFalse; } else { if(!cTurned2) ReturnFalse; if(!cTNet2.UpdateInputWeights(cPreNetNorm[1].AsObject())) ReturnFalse; if(!cFeatureNet[0].UpdateInputWeights(cTurned2.AsObject())) ReturnFalse; } if(!cFeatureNetNorm[0].UpdateInputWeights(cFeatureNet[0].AsObject())) ReturnFalse; uint total = cFeatureNet.Size(); for(uint i = 1; i < total; i++) { if(!cFeatureNet[i].UpdateInputWeights(cFeatureNetNorm[i - 1].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i].UpdateInputWeights(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFinalMLP[0].UpdateInputWeights(cMaxPool.AsObject())) ReturnFalse; if(!cFinalMLP[1].UpdateInputWeights(cFinalMLP[0].AsObject())) ReturnFalse; if(!CNeuronBaseOCL::updateInputWeights(cFinalMLP[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!(!cTNet1 || !cTNet2)) { if(FileWriteInteger(file_handle, 1) < INT_VALUE) ReturnFalse; int units = int(cTurned1.Neurons() / MathSqrt(cTNet1.Neurons())); if(FileWriteInteger(file_handle, units) < INT_VALUE) ReturnFalse; if(!cTNet1.Save(file_handle)) ReturnFalse; if(!cTNet2.Save(file_handle)) ReturnFalse; } else if(FileWriteInteger(file_handle, 0) < INT_VALUE) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!cPreNet[i].Save(file_handle)) ReturnFalse; if(!cPreNetNorm[i].Save(file_handle)) ReturnFalse; } for(int i = 0; i < 3; i++) { if(!cFeatureNet[i].Save(file_handle)) ReturnFalse; if(!cFeatureNetNorm[i].Save(file_handle)) ReturnFalse; } if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cMaxPool.Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFinalMLP[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNetOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; bool use_tnets = bool(FileReadInteger(file_handle)); if(use_tnets) { int units = FileReadInteger(file_handle); if(!cTNet1) { cTNet1 = new CNeuronPointNetOCL(); if(!cTNet1) ReturnFalse; } if(!cTNet1.Init(0, 0, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNet1)) ReturnFalse; if(!cTurned1) { cTurned1 = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } int dimension = (int)MathSqrt(cTNet1.Neurons()); if(!cTurned1.Init(0, 1, OpenCL, dimension * units, optimization, iBatch)) ReturnFalse; //--- if(!cTNet2) { cTNet2 = new CNeuronPointNetOCL(); if(!cTNet2) ReturnFalse; } if(!cTNet2.Init(0, 2, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNet2)) ReturnFalse; if(!cTurned2) { cTurned2 = new CNeuronBaseOCL(); if(!cTurned2) ReturnFalse; } dimension = (int)MathSqrt(cTNet2.Neurons()); if(!cTurned2.Init(0, 3, OpenCL, dimension * units, optimization, iBatch)) ReturnFalse; } //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cPreNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPreNetNorm[i].AsObject())) ReturnFalse; } for(int i = 0; i < 3; i++) { if(!LoadInsideLayer(file_handle, cFeatureNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureNetNorm[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMaxPool.AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, cFinalMLP[i].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPointNetOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(!!cTNet1) cTNet1.SetOpenCL(OpenCL); if(!!cTurned1) cTurned1.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) { cPreNet[i].SetOpenCL(OpenCL); cPreNetNorm[i].SetOpenCL(OpenCL); } if(!!cTNet2) cTNet2.SetOpenCL(OpenCL); if(!!cTurned2) cTurned2.SetOpenCL(OpenCL); for(int i = 0; i < 3; i++) { cFeatureNet[i].SetOpenCL(OpenCL); cFeatureNetNorm[i].SetOpenCL(OpenCL); } cTranspose.SetOpenCL(OpenCL); cMaxPool.SetOpenCL(OpenCL); for(int i = 0; i < 2; i++) cFinalMLP[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPointNet2Local : public CNeuronConvOCL { protected: float fRadius; uint iUnits; //--- CBufferFloat cDistance; CNeuronConvOCL cFeatureNet[3]; CNeuronBatchNormOCL cFeatureNetNorm[3]; CNeuronBaseOCL cLocalMaxPool; CNeuronConvOCL cFinalMLP; //--- virtual bool CalcDistance(CNeuronBaseOCL *NeuronOCL); virtual bool LocalMaxPool(void); virtual bool LocalMaxPoolGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPointNet2Local(void) {}; ~CNeuronPointNet2Local(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint window_out, float radius, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronPointNet2LocalOCL; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint window_out, float radius, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, 128, 128, window_out, units_count, 1, optimization_type, batch)) ReturnFalse; //--- fRadius = MathMax(0.1f, radius); iUnits = units_count; //--- cDistance.BufferFree(); if(!cDistance.BufferInit(iUnits * iUnits, 0) || !cDistance.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cFeatureNet[0].Init(0, 0, OpenCL, window, window, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[0].Init(0, 1, OpenCL, 64 * iUnits, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[0].SetActivationFunction(LReLU); if(!cFeatureNet[1].Init(0, 2, OpenCL, 64, 64, 128, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[1].Init(0, 3, OpenCL, 128 * iUnits, iBatch, optimization)) ReturnFalse; cFeatureNetNorm[1].SetActivationFunction(LReLU); if(!cFeatureNet[2].Init(0, 4, OpenCL, 128, 128, 256, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeatureNetNorm[2].Init(0, 5, OpenCL, 256 * iUnits, iBatch, optimization)) cFeatureNetNorm[2].SetActivationFunction(None); //--- if(!cLocalMaxPool.Init(0, 6, OpenCL, cFeatureNetNorm[2].Neurons(), optimization, iBatch)) ReturnFalse; if(!cFinalMLP.Init(0, 7, OpenCL, 256, 256, 128, iUnits, 1, optimization, iBatch)) ReturnFalse; cFinalMLP.SetActivationFunction(LReLU); //--- SetActivationFunction(None); return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::CalcDistance(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iUnits}; uint local_work_size[2] = {1, iUnits}; int kernel = def_k_CalcDistance; setBuffer(kernel, def_k_cd_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_cd_distance, cDistance.GetIndex()) setArgument(kernel, def_k_cd_dimension, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::LocalMaxPool(void) { //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_FeedForwardLocalMax; setBuffer(kernel, def_k_fflm_matrix_i, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getOutputIndex()) setBuffer(kernel, def_k_fflm_distance, cDistance.GetIndex()) setBuffer(kernel, def_k_fflm_matrix_o, cLocalMaxPool.getOutputIndex()) setArgument(kernel, def_k_fflm_radius, fRadius) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::LocalMaxPoolGrad(void) { //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_CalcInputGradientLocalMax; setBuffer(kernel, def_k_cglm_matrix_i, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getOutputIndex()) setBuffer(kernel, def_k_cglm_matrix_ig, cFeatureNetNorm[cFeatureNetNorm.Size() - 1].getGradientIndex()) setBuffer(kernel, def_k_cglm_distance, cDistance.GetIndex()) setBuffer(kernel, def_k_cglm_matrix_o, cLocalMaxPool.getOutputIndex()) setBuffer(kernel, def_k_cglm_matrix_g, cLocalMaxPool.getGradientIndex()) setArgument(kernel, def_k_cglm_radius, fRadius) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!CalcDistance(NeuronOCL)) ReturnFalse; CNeuronBaseOCL *temp = NeuronOCL; uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].FeedForward(temp)) ReturnFalse; if(!cFeatureNetNorm[i].FeedForward(cFeatureNet[i].AsObject())) ReturnFalse; temp = cFeatureNetNorm[i].AsObject(); } if(!LocalMaxPool()) ReturnFalse; if(!cFinalMLP.FeedForward(cLocalMaxPool.AsObject())) ReturnFalse; if(!CNeuronConvOCL::feedForward(cFinalMLP.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cFinalMLP.AsObject())) ReturnFalse; if(!cLocalMaxPool.CalcHiddenGradients(cFinalMLP.AsObject())) ReturnFalse; if(!LocalMaxPoolGrad()) ReturnFalse; int total = (int)cFeatureNet.Size() - 1; for(int i = total; i > 0; i--) { if(!cFeatureNet[i].FeedForward(cFeatureNetNorm[i].AsObject())) ReturnFalse; if(!cFeatureNetNorm[i - 1].FeedForward(cFeatureNet[i].AsObject())) ReturnFalse; } if(!cFeatureNet[0].FeedForward(cFeatureNetNorm[0].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFeatureNet[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *temp = NeuronOCL; uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].UpdateInputWeights(temp)) ReturnFalse; if(!cFeatureNetNorm[i].UpdateInputWeights(cFeatureNet[i].AsObject())) ReturnFalse; temp = cFeatureNetNorm[i].AsObject(); } if(!cFinalMLP.UpdateInputWeights(cLocalMaxPool.AsObject())) ReturnFalse; if(!CNeuronConvOCL::updateInputWeights(cFinalMLP.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteFloat(file_handle, fRadius) < sizeof(float)) ReturnFalse; if(FileWriteInteger(file_handle, iUnits) < INT_VALUE) ReturnFalse; //--- Save objects uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!cFeatureNet[i].Save(file_handle)) ReturnFalse; if(!cFeatureNetNorm[i].Save(file_handle)) ReturnFalse; } if(!cFinalMLP.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2Local::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- Load constants fRadius = FileReadFloat(file_handle); iUnits = FileReadInteger(file_handle); //--- Load objects uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { if(!LoadInsideLayer(file_handle, cFeatureNet[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureNetNorm[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cFinalMLP.AsObject())) ReturnFalse; //--- cDistance.BufferFree(); if(!cDistance.BufferInit(iUnits * iUnits, 0) || !cDistance.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cLocalMaxPool.Init(0, 6, OpenCL, cFeatureNetNorm[2].Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPointNet2Local::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); uint total = cFeatureNet.Size(); for(uint i = 0; i < total; i++) { cFeatureNet[i].SetOpenCL(OpenCL); cFeatureNetNorm[i].SetOpenCL(OpenCL); } cFinalMLP.SetOpenCL(OpenCL); cDistance.BufferCreate(OpenCL); cLocalMaxPool.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPointNet2OCL : public CNeuronPointNetOCL { protected: CNeuronPointNetOCL *cTNetG; CNeuronBaseOCL *cTurnedG; //--- CNeuronPointNet2Local caLocalPointNet[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPointNet2OCL(void) {}; ~CNeuronPointNet2OCL(void) ; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronPointNet2OCL; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronPointNet2OCL::~CNeuronPointNet2OCL(void) { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPointNetOCL::Init(numOutputs, myIndex, open_cl, 64, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; //--- Init T-Nets if(use_tnets) { if(!cTNetG) { cTNetG = new CNeuronPointNetOCL(); if(!cTNetG) ReturnFalse; } if(!cTNetG.Init(0, 0, OpenCL, window, units_count, window * window, false, optimization, iBatch)) ReturnFalse; if(!cTurnedG) { cTurnedG = new CNeuronBaseOCL(); if(!cTurned1) ReturnFalse; } if(!cTurnedG.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; } else { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //--- if(!caLocalPointNet[0].Init(0, 0, OpenCL, window, units_count, 64, 0.1f, optimization, iBatch)) ReturnFalse; if(!caLocalPointNet[1].Init(0, 0, OpenCL, 64, units_count, 64, 0.3f, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet if(!cTNetG) { if(!caLocalPointNet[0].FeedForward(NeuronOCL)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[0].FeedForward(cTurnedG.AsObject())) ReturnFalse; } if(!caLocalPointNet[1].FeedForward(caLocalPointNet[0].AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(caLocalPointNet[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronPointNetOCL::calcInputGradients(caLocalPointNet[1].AsObject())) ReturnFalse; //--- Local Net if(!caLocalPointNet[0].CalcHiddenGradients(caLocalPointNet[1].AsObject())) ReturnFalse; if(!cTNetG) { if(!NeuronOCL.CalcHiddenGradients(caLocalPointNet[0].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[0].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); NeuronOCL.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!NeuronOCL.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cTurnedG.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet if(!cTNetG) { if(!caLocalPointNet[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caLocalPointNet[0].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } if(!caLocalPointNet[1].UpdateInputWeights(caLocalPointNet[0].AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(caLocalPointNet[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPointNet2OCL::SetOpenCL(COpenCLMy * obj) { CNeuronPointNetOCL::SetOpenCL(obj); if(!!cTNetG) { cTNetG.SetOpenCL(OpenCL); cTurnedG.SetOpenCL(OpenCL); } caLocalPointNet[0].SetOpenCL(OpenCL); caLocalPointNet[1].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Save(const int file_handle) { if(!CNeuronPointNetOCL::Save(file_handle)) ReturnFalse; if(!caLocalPointNet[0].Save(file_handle)) ReturnFalse; if(!caLocalPointNet[1].Save(file_handle)) ReturnFalse; bool use_tnet = bool(!!cTNetG); if(FileWriteInteger(file_handle, int(use_tnet)) < INT_VALUE) ReturnFalse; if(use_tnet) { if(!cTNetG.Save(file_handle)) ReturnFalse; if(!cTurnedG.Save(file_handle)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointNet2OCL::Load(const int file_handle) { if(!CNeuronPointNetOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPointNet[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPointNet[1].AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; bool use_tnet = bool(FileReadInteger(file_handle)); if(use_tnet) { if(!cTNetG) { cTNetG = new CNeuronPointNetOCL(); if(!cTNetG) ReturnFalse; } if(!cTNetG.Init(0, 0, OpenCL, 1, 1, 1, false, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTNetG)) ReturnFalse; if(!cTurnedG) { cTurnedG = new CNeuronBaseOCL(); if(!cTurnedG) ReturnFalse; } if(!cTurnedG.Init(0, 1, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTurnedG)) ReturnFalse; } else { DeleteObj(cTNetG); DeleteObj(cTurnedG); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPointFormer : public CNeuronPointNet2OCL { protected: CNeuronMLMHSparseAttention caLocalAttention[2]; CNeuronMLCrossAttentionMLKV caLocalGlobalAttention[2]; CNeuronMLMHAttentionMLKV caGlobalAttention[2]; CNeuronLearnabledPE caLocalPE[2]; CNeuronLearnabledPE caGlobalPE[2]; CNeuronBaseOCL cConcatenate; CNeuronConvOCL cScale; //--- CBufferFloat *cbTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPointFormer(void) {}; ~CNeuronPointFormer(void) { DeleteObj(cbTemp); } //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronPointFormer; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Init(0, i * 5, OpenCL, 64, 16, 4, units_count, 2, optimization, iBatch)) ReturnFalse; if(!caLocalGlobalAttention[i].Init(0, i * 5 + 1, OpenCL, 64, 16, 4, 64, 2, units_count, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caGlobalAttention[i].Init(0, i * 5 + 2, OpenCL, 64, 16, 4, 2, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caLocalPE[i].Init(0, i * 5 + 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; if(!caGlobalPE[i].Init(0, i * 5 + 4, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } caLocalAttention[0].Sparse(0.1f); caLocalAttention[1].Sparse(0.3f); //--- if(!cConcatenate.Init(0, 10, OpenCL, 128 * units_count, optimization, iBatch)) ReturnFalse; if(!cScale.Init(0, 11, OpenCL, 128, 128, 64, units_count, 1, optimization, iBatch)) ReturnFalse; //--- DeleteObj(cbTemp); cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].FeedForward(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(inputs)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[i].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].FeedForward(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].FeedForward(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].FeedForward(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!Concat(caGlobalAttention[0].getOutput(), caGlobalAttention[1].getOutput(), cConcatenate.getOutput(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; if(!cScale.FeedForward(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(cScale.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPointNetOCL::calcInputGradients(cScale.AsObject())) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cScale.AsObject())) ReturnFalse; if(!DeConcat(caGlobalAttention[0].getGradient(), caGlobalAttention[1].getGradient(), cConcatenate.getGradient(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; //--- CNeuronBaseOCL *inputs = caGlobalAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { //--- Global Attention if(!caLocalGlobalAttention[i].CalcHiddenGradients(caGlobalAttention[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalPE[i].CalcHiddenGradients(caLocalGlobalAttention[i].AsObject(), caGlobalPE[i].getOutput(), caGlobalPE[i].getGradient(), (ENUM_ACTIVATION)caGlobalPE[i].Activation())) ReturnFalse; //--- Position Encoder if(!caLocalAttention[i].CalcHiddenGradients(caLocalPE[i].AsObject())) ReturnFalse; if(!caLocalPointNet[i].CalcHiddenGradients(caGlobalPE[i].AsObject())) ReturnFalse; //--- Local Attention CBufferFloat *temp = caLocalPointNet[i].getGradient(); caLocalPointNet[i].SetGradient(cbTemp, false); cbTemp = temp; if(!caLocalPointNet[i].CalcHiddenGradients(caLocalAttention[i].AsObject())) ReturnFalse; if(!SumAndNormilize(caLocalPointNet[i].getGradient(), cbTemp, caLocalPointNet[i].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) { temp = inputs.getGradient(); inputs.SetGradient(cbTemp, false); cbTemp = temp; } //--- Local Net if(!cTNetG || i > 0) { if(!inputs.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(inputs.getOutput(), inputs.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), inputs.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = inputs.getGradient(); inputs.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!inputs.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormilize(inputs.getGradient(), cTurnedG.getGradient(), inputs.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) { if(!SumAndNormilize(inputs.getGradient(), cbTemp, inputs.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; inputs = caGlobalAttention[i - 1].AsObject(); } else inputs = NeuronOCL; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].UpdateInputWeights(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(inputs)) ReturnFalse; if(!caLocalPointNet[i].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].UpdateInputWeights(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].UpdateInputWeights(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].UpdateInputWeights(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!cScale.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(cScale.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalPE[i].Save(file_handle)) ReturnFalse; if(!caGlobalPE[i].Save(file_handle)) ReturnFalse; } if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPointFormer::Load(const int file_handle) { DeleteObj(cbTemp); //--- if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, caLocalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPE[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalPE[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; //--- cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPointFormer::SetOpenCL(COpenCLMy * obj) { CNeuronPointNet2OCL::SetOpenCL(obj); //--- for(int i = 0; i < 2; i++) { caLocalAttention[i].SetOpenCL(OpenCL); caLocalGlobalAttention[i].SetOpenCL(OpenCL); caGlobalAttention[i].SetOpenCL(OpenCL); caLocalPE[i].SetOpenCL(OpenCL); caGlobalPE[i].SetOpenCL(OpenCL); } cConcatenate.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSceneSpecific : public CNeuronMLCrossAttentionMLKV { protected: CNeuronBaseOCL cOne; CNeuronBaseOCL cSceneSpecificKnowledge; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return feedForward(NeuronOCL); } //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return updateInputWeights(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSceneSpecific(void) {}; ~CNeuronSceneSpecific(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSceneSpecific; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, 8, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(8 * units_count_kv, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cSceneSpecificKnowledge.Init(0, 1, OpenCL, 8 * units_count_kv, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::feedForward(CNeuronBaseOCL * NeuronOCL) { if(bTrain && !cSceneSpecificKnowledge.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(NeuronOCL, cSceneSpecificKnowledge.getOutput(), cSceneSpecificKnowledge.getGradient(), (ENUM_ACTIVATION)cSceneSpecificKnowledge.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(NeuronOCL, cSceneSpecificKnowledge.getOutput())) ReturnFalse; if(!cSceneSpecificKnowledge.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cSceneSpecificKnowledge.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSceneSpecific::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSceneSpecificKnowledge.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSceneSpecific::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cSceneSpecificKnowledge.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMLMHSceneConditionAttention : public CNeuronMLMHAttentionMLKV { protected: CLayer cSceneAgnostic; CLayer cSceneSpecific; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMLMHSceneConditionAttention(void) {}; ~CNeuronMLMHSceneConditionAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronMLMHSceneConditionAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = fmax(window, 1); iWindowKey = fmax(window_key, 1); iUnits = fmax(units_count, 1); iHeads = fmax(heads, 1); iLayers = fmax(layers, 1); iHeadsKV = fmax(heads_kv, 1); iLayersToOneKV = fmax(layers_to_one_kv, 1); //--- uint num_q = iWindowKey * iHeads * iUnits; //Size of Q tensor uint num_kv = 2 * iWindowKey * iHeadsKV * iUnits; //Size of KV tensor uint q_weights = (iWindow * iHeads) * iWindowKey; //Size of weights' matrix of Q tenzor uint kv_weights = 2 * (iWindow * iHeadsKV) * iWindowKey; //Size of weights' matrix of KV tenzor uint scores = iUnits * iUnits * iHeads; //Size of Score tensor uint mh_out = iWindowKey * iHeads * iUnits; //Size of multi-heads self-attention uint out = iWindow * iUnits; //Size of out tensore uint w0 = (iWindowKey * iHeads + 1) * iWindow; //Size W0 tensor uint ff_1 = 4 * (iWindow + 1) * iWindow; //Size of weights' matrix 1-st feed forward layer uint ff_2 = (4 * iWindow + 1) * iWindow; //Size of weights' matrix 2-nd feed forward layer //--- CNeuronBaseOCL *base = NULL; CNeuronSceneSpecific *ss = NULL; //--- for(uint i = 0; i < iLayers; i++) { CBufferFloat *temp = NULL; for(int d = 0; d < 2; d++) { //--- Initilize Q tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_q, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Tensors.Add(temp)) ReturnFalse; //--- Initilize Q weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(q_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!QKV_Weights.Add(temp)) ReturnFalse; if(i % iLayersToOneKV == 0) { //--- Initilize KV tensor temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(num_kv, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Tensors.Add(temp)) ReturnFalse; //--- Initilize KV weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(kv_weights, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!KV_Weights.Add(temp)) ReturnFalse; } //--- Initialize scores temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(scores, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!S_Tensors.Add(temp)) ReturnFalse; //--- Initialize multi-heads attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(mh_out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!AO_Tensors.Add(temp)) ReturnFalse; //--- Initialize attention out temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 1 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(4 * out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; //--- Initialize Feed Forward 2 if(i == iLayers - 1) { if(!FF_Tensors.Add(d == 0 ? Output : Gradient)) ReturnFalse; continue; } temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit(out, 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Tensors.Add(temp)) ReturnFalse; } if(i % iLayersToOneKV == 0) { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init((q_weights + kv_weights), cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init((q_weights + kv_weights), cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, (q_weights + kv_weights), optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } else { //--- Initilize Scene-Specific layers ss = new CNeuronSceneSpecific(); if(!ss) ReturnFalse; if(!ss.Init(q_weights, cSceneSpecific.Total(), OpenCL, iWindow, iWindowKey, 4, 2, iUnits, 20, 2, 2, optimization, iBatch)) ReturnFalse; if(!cSceneSpecific.Add(ss)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneSpecific.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; base.SetActivationFunction(TANH); if(!cSceneSpecific.Add(base)) ReturnFalse; //--- Initilize Scene-Agnostic layers base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(q_weights, cSceneAgnostic.Total(), OpenCL, 1, optimization, iBatch)) ReturnFalse; temp = base.getOutput(); if(!temp.BufferInit(1, 1) || !temp.BufferWrite()) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, cSceneAgnostic.Total(), OpenCL, q_weights, optimization, iBatch)) ReturnFalse; if(!cSceneAgnostic.Add(base)) ReturnFalse; } //--- Initilize Weights0 temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(w0)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(uint w = 0; w < w0; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_1)) ReturnFalse; for(uint w = 0; w < ff_1; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.Reserve(ff_2)) ReturnFalse; k = (float)(1 / sqrt(4 * iWindow + 1)); for(uint w = 0; w < ff_2; w++) { if(!temp.Add(GenerateWeight() * 2 * k - k)) ReturnFalse; } if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- for(int d = 0; d < (optimization == SGD ? 1 : 2); d++) { temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? w0 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; //--- Initilize FF Weights temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_1 : 4 * iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; temp = new CBufferFloat(); if(CheckPointer(temp) == POINTER_INVALID) ReturnFalse; if(!temp.BufferInit((d == 0 || optimization == ADAM ? ff_2 : iWindow), 0)) ReturnFalse; if(!temp.BufferCreate(OpenCL)) ReturnFalse; if(!FF_Weights.Add(temp)) ReturnFalse; } } //--- if(!Temp.BufferInit(MathMax((num_q + num_kv)*iWindow, out), 0)) ReturnFalse; if(!Temp.BufferCreate(OpenCL)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.FeedForward(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.FeedForward(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(bTrain && !sa.FeedForward(cSceneAgnostic[i * 2])) ReturnFalse; CBufferFloat *inputs = (i == 0 ? NeuronOCL.getOutput() : FF_Tensors.At(6 * i - 4)); //--- weights q_weights = QKV_Weights[i * 2]; q = QKV_Tensors[i * 2]; if(i % iLayersToOneKV == 0) { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), GetPointer(Temp))) ReturnFalse; kv_weights = KV_Weights[(i / iLayersToOneKV) * 2]; kv = KV_Tensors[(i / iLayersToOneKV) * 2]; if(IsStopped() || !DeConcat(q_weights, kv_weights, GetPointer(Temp), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(IsStopped() || !MatMul(inputs, kv_weights, kv, iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; } else { if(IsStopped() || !ElementMult(ss.getOutput(), sa.getOutput(), q_weights)) ReturnFalse; } if(IsStopped() || !MatMul(inputs, q_weights, q, iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Score calculation and Multi-heads attention calculation CBufferFloat *temp = S_Tensors[i * 2]; CBufferFloat *out = AO_Tensors[i * 2]; if(IsStopped() || !AttentionOut(q, kv, temp, out)) ReturnFalse; //--- Attention out calculation temp = FF_Tensors[i * 6]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9)], out, temp, iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Sum and normilize attention if(IsStopped() || !SumAndNormilize(temp, inputs, temp, iWindow, true)) ReturnFalse; //--- Feed Forward inputs = temp; temp = FF_Tensors[i * 6 + 1]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], inputs, temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; out = FF_Tensors[i * 6 + 2]; if(IsStopped() || !ConvolutionForward(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], temp, out, 4 * iWindow, iWindow, activation)) ReturnFalse; //--- Sum and normilize out if(IsStopped() || !SumAndNormilize(out, inputs, out, iWindow, true)) ReturnFalse; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(CheckPointer(prevLayer) == POINTER_INVALID) ReturnFalse; //--- CBufferFloat *out_grad = Gradient; CBufferFloat *kv_g = KV_Tensors[KV_Tensors.Total() - 1]; CNeuronBaseOCL *ss = NULL, *sa = NULL; //--- for(int i = int(iLayers - 1); (i >= 0 && !IsStopped()); i--) { if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) kv_g = KV_Tensors[(i / iLayersToOneKV) * 2 + 1]; //--- Passing gradient through feed forward layers if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 2], out_grad, FF_Tensors[i * 6 + 1], FF_Tensors[i * 6 + 4], 4 * iWindow, iWindow, None)) ReturnFalse; CBufferFloat *temp = FF_Tensors[i * 6 + 3]; if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9) + 1], FF_Tensors[i * 6 + 4], FF_Tensors[i * 6], temp, iWindow, 4 * iWindow, LReLU)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false)) ReturnFalse; out_grad = temp; //--- Split gradient to multi-heads if(IsStopped() || !ConvolutionInputGradients(FF_Weights[i * (optimization == SGD ? 6 : 9)], out_grad, AO_Tensors[i * 2], AO_Tensors[i * 2 + 1], iWindowKey * iHeads, iWindow, None)) ReturnFalse; //--- Passing gradient to query, key and value sa = cSceneAgnostic[i * 2 + 1]; ss = cSceneSpecific[i * 2 + 1]; if(i == int(iLayers - 1) || (i + 1) % iLayersToOneKV == 0) { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[(i / iLayersToOneKV) * 2], kv_g, S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; } else { if(IsStopped() || !AttentionInsideGradients(QKV_Tensors[i * 2], QKV_Tensors[i * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2], GetPointer(Temp), S_Tensors[i * 2], AO_Tensors[i * 2 + 1])) ReturnFalse; if(IsStopped() || !SumAndNormilize(kv_g, GetPointer(Temp), kv_g, iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } //--- CBufferFloat *inp = NULL; if(i == 0) { inp = prevLayer.getOutput(); temp = prevLayer.getGradient(); } else { temp = FF_Tensors.At(i * 6 - 1); inp = FF_Tensors.At(i * 6 - 4); } if(IsStopped() || !MatMulGrad(inp, temp, QKV_Weights[i * 2], QKV_Weights[i * 2 + 1], QKV_Tensors[i * 2 + 1], iUnits, iWindow, iHeads * iWindowKey, 1)) ReturnFalse; //--- Sum and normilize gradients if(IsStopped() || !SumAndNormilize(out_grad, temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((i % iLayersToOneKV) == 0) { if(IsStopped() || !MatMulGrad(inp, GetPointer(Temp), KV_Weights[i / iLayersToOneKV * 2], KV_Weights[i / iLayersToOneKV * 2 + 1], KV_Tensors[i / iLayersToOneKV * 2 + 1], iUnits, iWindow, 2 * iHeadsKV * iWindowKey, 1)) ReturnFalse; if(IsStopped() || !SumAndNormilize(GetPointer(Temp), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(QKV_Weights[i * 2 + 1], KV_Weights[i / iLayersToOneKV * 2 + 1], ss.getGradient(), iHeads, 2 * iHeadsKV, iWindow * iWindowKey)) ReturnFalse; if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), ss.getGradient(), ss.Activation(), sa.Activation())) ReturnFalse; } else { if(!ElementMultGrad(ss.getOutput(), ss.getGradient(), sa.getOutput(), sa.getGradient(), QKV_Weights[i * 2 + 1], ss.Activation(), sa.Activation())) ReturnFalse; } if(i > 0) out_grad = temp; } //--- CBufferFloat *inp_grad = prevLayer.getGradient(); if(!prevLayer.SetGradient(GetPointer(Temp), false)) ReturnFalse; for(int i = int(iLayers - 2); (i >= 0 && !IsStopped()); i -= 2) { ss = cSceneSpecific[i]; if(IsStopped() || !ss.CalcHiddenGradients(cSceneSpecific[i + 1])) ReturnFalse; if(IsStopped() || !prevLayer.CalcHiddenGradients(ss, NULL)) ReturnFalse; if(IsStopped() || !SumAndNormilize(prevLayer.getGradient(), inp_grad, inp_grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!prevLayer.SetGradient(inp_grad, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *ss = NULL, *sa = NULL; CBufferFloat *q_weights = NULL, *kv_weights = NULL, *q = NULL, *kv = NULL; //--- for(uint i = 0; i < iLayers; i++) { //--- Scene-Specific ss = cSceneSpecific[i * 2]; if(!ss.UpdateInputWeights(NeuronOCL)) ReturnFalse; ss = cSceneSpecific[i * 2 + 1]; if(!ss.UpdateInputWeights(cSceneSpecific[i * 2])) ReturnFalse; //--- Scene-Agnostic sa = cSceneAgnostic[i * 2 + 1]; if(!sa.UpdateInputWeights(cSceneAgnostic[i * 2])) ReturnFalse; //--- Attention out if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9)), FF_Tensors.At(i * 6 + 3), AO_Tensors.At(i * 2), (optimization == SGD ? FF_Weights.At(i * 6 + 3) : FF_Weights.At(i * 9 + 3)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 6)), iWindowKey * iHeads, iWindow, 0, 1)) ReturnFalse; //--- Feed Forward if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 1), FF_Tensors.At(i * 6 + 4), FF_Tensors.At(i * 6), (optimization == SGD ? FF_Weights.At(i * 6 + 4) : FF_Weights.At(i * 9 + 4)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 7)), iWindow, 4 * iWindow, 0, 1)) ReturnFalse; //--- if(IsStopped() || !ConvolutuionUpdateWeights(FF_Weights.At(i * (optimization == SGD ? 6 : 9) + 2), FF_Tensors.At(i * 6 + 5), FF_Tensors.At(i * 6 + 1), (optimization == SGD ? FF_Weights.At(i * 6 + 5) : FF_Weights.At(i * 9 + 5)), (optimization == SGD ? NULL : FF_Weights.At(i * 9 + 8)), 4 * iWindow, iWindow, 0, 1)) ReturnFalse; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Save(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cSceneSpecific.Save(file_handle)) ReturnFalse; if(!cSceneAgnostic.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMLMHSceneConditionAttention::Load(const int file_handle) { if(!CNeuronMLMHAttentionMLKV::Load(file_handle)) ReturnFalse; if(!cSceneSpecific.Load(file_handle)) ReturnFalse; if(!cSceneAgnostic.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMLMHSceneConditionAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMLMHAttentionMLKV::SetOpenCL(obj); cSceneSpecific.SetOpenCL(OpenCL); cSceneAgnostic.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHyperDet : public CNeuronPointNet2OCL { protected: CNeuronMLMHSparseAttention caLocalAttention[2]; CNeuronMLCrossAttentionMLKV caLocalGlobalAttention[2]; CNeuronMLMHSceneConditionAttention caGlobalAttention[2]; CNeuronLearnabledPE caLocalPE[2]; CNeuronLearnabledPE caGlobalPE[2]; CNeuronBaseOCL cConcatenate; CNeuronConvOCL cScale; //--- CBufferFloat *cbTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHyperDet(void) {}; ~CNeuronHyperDet(void) { DeleteObj(cbTemp); } //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronHyperDet; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Init(0, i * 5, OpenCL, 64, 16, 4, units_count, 2, optimization, iBatch)) ReturnFalse; if(!caLocalGlobalAttention[i].Init(0, i * 5 + 1, OpenCL, 64, 16, 4, 64, 2, units_count, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caGlobalAttention[i].Init(0, i * 5 + 2, OpenCL, 64, 16, 4, 2, units_count, 2, 2, optimization, iBatch)) ReturnFalse; if(!caLocalPE[i].Init(0, i * 5 + 3, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; if(!caGlobalPE[i].Init(0, i * 5 + 4, OpenCL, 64 * units_count, optimization, iBatch)) ReturnFalse; } caLocalAttention[0].Sparse(0.1f); caLocalAttention[1].Sparse(0.3f); //--- if(!cConcatenate.Init(0, 10, OpenCL, 128 * units_count, optimization, iBatch)) ReturnFalse; if(!cScale.Init(0, 11, OpenCL, 128, 128, 64, units_count, 1, optimization, iBatch)) ReturnFalse; //--- DeleteObj(cbTemp); cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].FeedForward(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(inputs)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[i].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].FeedForward(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].FeedForward(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].FeedForward(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].FeedForward(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!Concat(caGlobalAttention[0].getOutput(), caGlobalAttention[1].getOutput(), cConcatenate.getOutput(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; if(!cScale.FeedForward(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::feedForward(cScale.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronPointNetOCL::calcInputGradients(cScale.AsObject())) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cScale.AsObject())) ReturnFalse; if(!DeConcat(caGlobalAttention[0].getGradient(), caGlobalAttention[1].getGradient(), cConcatenate.getGradient(), 64, 64, cConcatenate.Neurons() / 128)) ReturnFalse; //--- CNeuronBaseOCL *inputs = caGlobalAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { //--- Global Attention if(!caLocalGlobalAttention[i].CalcHiddenGradients(caGlobalAttention[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalPE[i].CalcHiddenGradients(caLocalGlobalAttention[i].AsObject(), caGlobalPE[i].getOutput(), caGlobalPE[i].getGradient(), (ENUM_ACTIVATION)caGlobalPE[i].Activation())) ReturnFalse; //--- Position Encoder if(!caLocalAttention[i].CalcHiddenGradients(caLocalPE[i].AsObject())) ReturnFalse; if(!caLocalPointNet[i].CalcHiddenGradients(caGlobalPE[i].AsObject())) ReturnFalse; //--- Local Attention CBufferFloat *temp = caLocalPointNet[i].getGradient(); caLocalPointNet[i].SetGradient(cbTemp, false); cbTemp = temp; if(!caLocalPointNet[i].CalcHiddenGradients(caLocalAttention[i].AsObject())) ReturnFalse; if(!SumAndNormilize(caLocalPointNet[i].getGradient(), cbTemp, caLocalPointNet[i].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; if(i > 0) { temp = inputs.getGradient(); inputs.SetGradient(cbTemp, false); cbTemp = temp; } //--- Local Net if(!cTNetG || i > 0) { if(!inputs.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[i].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(inputs.getOutput(), inputs.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), inputs.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = inputs.getGradient(); inputs.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!inputs.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormilize(inputs.getGradient(), cTurnedG.getGradient(), inputs.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0) if(!SumAndNormilize(inputs.getGradient(), cbTemp, inputs.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; inputs = NeuronOCL; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- LocalNet CNeuronBaseOCL *inputs = NeuronOCL; for(int i = 0; i < 2; i++) { if(!cTNetG || i > 0) { if(!caLocalPointNet[i].UpdateInputWeights(inputs)) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(inputs)) ReturnFalse; if(!caLocalPointNet[i].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Local Attention if(!caLocalAttention[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Position Encoder if(!caLocalPE[i].UpdateInputWeights(caLocalAttention[i].AsObject())) ReturnFalse; if(!caGlobalPE[i].UpdateInputWeights(caLocalPointNet[i].AsObject())) ReturnFalse; //--- Local to Global Attention if(!caLocalGlobalAttention[i].UpdateInputWeights(caLocalPE[i].AsObject(), caGlobalPE[i].getOutput())) ReturnFalse; //--- Global Attention if(!caGlobalAttention[i].UpdateInputWeights(caLocalGlobalAttention[i].AsObject())) ReturnFalse; inputs = caGlobalAttention[i].AsObject(); } //--- if(!cScale.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; if(!CNeuronPointNetOCL::updateInputWeights(cScale.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!caLocalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caGlobalAttention[i].Save(file_handle)) ReturnFalse; if(!caLocalPE[i].Save(file_handle)) ReturnFalse; if(!caGlobalPE[i].Save(file_handle)) ReturnFalse; } if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperDet::Load(const int file_handle) { DeleteObj(cbTemp); //--- if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, caLocalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caLocalPE[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caGlobalPE[i].AsObject())) ReturnFalse; } if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; //--- cbTemp = new CBufferFloat(); if(!cbTemp || !cbTemp.BufferInit(caGlobalAttention[0].Neurons(), 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHyperDet::SetOpenCL(COpenCLMy * obj) { CNeuronPointNet2OCL::SetOpenCL(obj); //--- for(int i = 0; i < 2; i++) { caLocalAttention[i].SetOpenCL(OpenCL); caLocalGlobalAttention[i].SetOpenCL(OpenCL); caGlobalAttention[i].SetOpenCL(OpenCL); caLocalPE[i].SetOpenCL(OpenCL); caGlobalPE[i].SetOpenCL(OpenCL); } cConcatenate.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSEFormer : public CNeuronPointNet2OCL { protected: uint iUnits; uint iPoints; //--- CLayer cQuery; CLayer cKey; CLayer cValue; CLayer cKeyValue; CArrayInt cScores; CLayer cMHAttentionOut; CLayer cAttentionOut; CLayer cResidual; CLayer cFeedForward; CLayer cCenterPoints; CLayer cFinalAttention; CNeuronMLCrossAttentionMLKV SEOut; CBufferFloat cbTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool AttentionOut(CBufferFloat *q, CBufferFloat *kv, int scores, CBufferFloat *out); virtual bool AttentionInsideGradients(CBufferFloat *q, CBufferFloat *q_g, CBufferFloat *kv, CBufferFloat *kv_g, int scores, CBufferFloat *gradient); //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; //--- public: CNeuronSEFormer(void) {}; ~CNeuronSEFormer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint output, bool use_tnets, uint center_points, uint center_window, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSEFormer; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint output, bool use_tnets, uint center_points, uint center_window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPointNet2OCL::Init(numOutputs, myIndex, open_cl, window, units_count, output, use_tnets, optimization_type, batch)) ReturnFalse; //--- iUnits = units_count; iPoints = MathMax(center_points, 9); //--- cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cCenterPoints.SetOpenCL(OpenCL); cFinalAttention.SetOpenCL(OpenCL); //--- Init center points CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iPoints * center_window * 2, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; if(!cCenterPoints.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base.Init(0, 1, OpenCL, iPoints * center_window * 2, optimization, iBatch)) ReturnFalse; if(!cCenterPoints.Add(base)) ReturnFalse; //--- Inside layers for(int i = 0; i < 2; i++) { //--- Interpolation CNeuronMVCrossAttentionMLKV *cross = new CNeuronMVCrossAttentionMLKV(); if(!cross || !cross.Init(0, i * 12 + 2, OpenCL, center_window, 32, 4, 64, 2, iPoints, iUnits, 2, 2, 2, 1, optimization, iBatch)) ReturnFalse; if(!cCenterPoints.Add(cross)) ReturnFalse; //--- Query CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 3, OpenCL, 64, 64, 64, iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 4, OpenCL, center_window, center_window, 32, iPoints, 2, optimization, iBatch)) ReturnFalse; if(!cKey.Add(conv)) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 5, OpenCL, center_window, center_window, 32, 1, iPoints * 2, optimization, iBatch)) ReturnFalse; if(!cValue.Add(conv)) ReturnFalse; //--- Key-Value base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 6, OpenCL, iPoints * 2 * 32 * 2, optimization, iBatch)) ReturnFalse; if(!cKeyValue.Add(base)) ReturnFalse; //--- Score int s = int(iUnits * iPoints * 4); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- MH Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 7, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!cMHAttentionOut.Add(base)) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 8, OpenCL, 64, 64, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 9, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 10, OpenCL, 64, 64, 256, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, i * 12 + 11, OpenCL, 256, 64, 64, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, i * 12 + 12, OpenCL, iUnits * 64, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient(), true)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Final Attention CNeuronMLMHSceneConditionAttention *att = new CNeuronMLMHSceneConditionAttention(); if(!att || !att.Init(0, i * 12 + 13, OpenCL, 64, 16, 4, 2, iUnits, 2, 1, optimization, iBatch)) ReturnFalse; if(!cFinalAttention.Add(att)) ReturnFalse; } //--- if(!SEOut.Init(0, 26, OpenCL, 64, 64, 4, 16, 4, iUnits, iUnits, 4, 1, optimization, iBatch)) ReturnFalse; //--- uint buf_size = MathMax(MathMax(64, window) * iUnits, center_window * iPoints); if(!cbTemp.BufferInit(buf_size, 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::AttentionOut(CBufferFloat * q, CBufferFloat * kv, int scores, CBufferFloat * out) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits/*Q units*/, iPoints, 4}; uint local_work_size[3] = {1, iPoints, 1}; ResetLastError(); setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_q, q.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_score, scores) setBuffer(def_k_MH2AttentionOut, def_k_mh2ao_out, out.GetIndex()) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_dimension, 16) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_heads_kv, (2 * 2)) setArgument(def_k_MH2AttentionOut, def_k_mh2ao_mask, 0) kernelExecuteLoc(def_k_MH2AttentionOut, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::AttentionInsideGradients(CBufferFloat * q, CBufferFloat * qg, CBufferFloat * kv, CBufferFloat * kvg, int scores, CBufferFloat * outg ) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, 16, 4}; ResetLastError(); setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_q, q.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_qg, qg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kv, kv.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kvg, kvg.GetIndex()) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_score, scores) setBuffer(def_k_MH2AttentionInsideGradients, def_k_mh2aig_outg, outg.GetIndex()) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_kunits, (int)iPoints) setArgument(def_k_MH2AttentionInsideGradients, def_k_mh2aig_heads_kv, (2 * 2)) kernelExecute(def_k_MH2AttentionInsideGradients, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- CNeuronBaseOCL *neuron = NULL, *q = NULL, *k = NULL, *v = NULL, *kv = NULL; //--- Init Points if(bTrain) { neuron = cCenterPoints[1]; if(!neuron || !neuron.FeedForward(cCenterPoints[0])) ReturnFalse; } //--- Inside Layers for(int l = 0; l < 2; l++) { //--- Segmentation Inputs if(l > 0 || !cTNetG) { if(!caLocalPointNet[l].FeedForward((l == 0 ? NeuronOCL : GetPointer(caLocalPointNet[l - 1])))) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.FeedForward(NeuronOCL)) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMul(NeuronOCL.getOutput(), cTNetG.getOutput(), cTurnedG.getOutput(), NeuronOCL.Neurons() / window, window, window)) ReturnFalse; if(!caLocalPointNet[0].FeedForward(cTurnedG.AsObject())) ReturnFalse; } //--- Interpolate center points neuron = cCenterPoints[l + 2]; if(!neuron || !neuron.FeedForward(cCenterPoints[l + 1], caLocalPointNet[l].getOutput())) ReturnFalse; //--- Structure-Embedding Attention q = cQuery[l]; k = cKey[l]; v = cValue[l]; kv = cKeyValue[l]; //--- Query if(!q || !q.FeedForward(GetPointer(caLocalPointNet[l]))) ReturnFalse; //--- Key if(!k || !k.FeedForward(cCenterPoints[l + 2])) ReturnFalse; //--- Value if(!v || !v.FeedForward(cCenterPoints[l + 2])) ReturnFalse; if(!kv || !Concat(k.getOutput(), v.getOutput(), kv.getOutput(), 32 * 2, 32 * 2, iPoints)) ReturnFalse; //--- Multi-Head Attention neuron = cMHAttentionOut[l]; if(!neuron || !AttentionOut(q.getOutput(), kv.getOutput(), cScores[l], neuron.getOutput())) ReturnFalse; //--- Scale neuron = cAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[l])) ReturnFalse; //--- Residual q = cResidual[l * 2]; if(!q || !SumAndNormilize(caLocalPointNet[l].getOutput(), neuron.getOutput(), q.getOutput(), 64, true, 0, 0, 0, 1)) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.FeedForward(q)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cFeedForward[l * 2])) ReturnFalse; //--- Residual k = cResidual[l * 2 + 1]; if(!k || !SumAndNormilize(q.getOutput(), neuron.getOutput(), k.getOutput(), 64, true, 0, 0, 0, 1)) ReturnFalse; //--- Final Attention neuron = cFinalAttention[l]; if(!neuron || !neuron.FeedForward(k)) ReturnFalse; } //--- Cross scale attention if(!SEOut.FeedForward(cFinalAttention[0], neuron.getOutput())) ReturnFalse; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::feedForward(SEOut.AsObject())) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *neuron = NULL, *q = NULL, *k = NULL, *v = NULL, *kv = NULL; CBufferFloat *buf = NULL; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::calcInputGradients(SEOut.AsObject())) ReturnFalse; //--- Cross scale attention neuron = cFinalAttention[0]; q = cFinalAttention[1]; if(!neuron.CalcHiddenGradients(SEOut.AsObject(), q.getOutput(), q.getGradient(), (ENUM_ACTIVATION)q.Activation())) ReturnFalse; //--- Inside Layers for(int l = 1; l >= 0; l--) { //--- Final Attention neuron = cResidual[l * 2 + 1]; if(!neuron || !neuron.CalcHiddenGradients(cFinalAttention[l])) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; neuron = cResidual[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual q = cResidual[l * 2 + 1]; k = neuron; neuron = cAttentionOut[l]; if(!neuron || !SumAndNormilize(q.getGradient(), k.getGradient(), neuron.getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Scale neuron = cMHAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cAttentionOut[l])) ReturnFalse; //--- MH Attention q = cQuery[l]; kv = cKeyValue[l]; k = cKey[l]; v = cValue[l]; if(!AttentionInsideGradients(q.getOutput(), q.getGradient(), kv.getOutput(), kv.getGradient(), cScores[l], neuron.getGradient())) ReturnFalse; if(!DeConcat(k.getGradient(), v.getGradient(), kv.getGradient(), 32 * 2, 32 * 2, iPoints)) ReturnFalse; if(l == 0) { buf = caLocalPointNet[l].getGradient(); if(!caLocalPointNet[l].SetGradient(GetPointer(cbTemp), false)) ReturnFalse; } if(!caLocalPointNet[l].CalcHiddenGradients(q, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormilize(buf, GetPointer(cbTemp), buf, 64, false, 0, 0, 0, 1)) ReturnFalse; if(!caLocalPointNet[l].SetGradient(buf, false)) ReturnFalse; } neuron = cAttentionOut[l]; //--- Residual if(!SumAndNormilize(caLocalPointNet[l].getGradient(), neuron.getGradient(), caLocalPointNet[l].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Interpolate Center points neuron = cCenterPoints[l + 2]; if(!neuron) ReturnFalse; buf = neuron.getGradient(); if(!neuron.SetGradient(GetPointer(cbTemp), false)) ReturnFalse; if(!neuron.CalcHiddenGradients(k, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormilize(buf, GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormilize(GetPointer(cbTemp), GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 0.5f)) ReturnFalse; } if(!neuron.CalcHiddenGradients(v, NULL)) ReturnFalse; if(!SumAndNormilize(buf, GetPointer(cbTemp), buf, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!neuron.SetGradient(buf, false)) ReturnFalse; neuron = cCenterPoints[l + 1]; if(!neuron.CalcHiddenGradients(cCenterPoints[l + 2], caLocalPointNet[l].getOutput(), GetPointer(cbTemp), (ENUM_ACTIVATION)caLocalPointNet[l].Activation())) ReturnFalse; if(!SumAndNormilize(caLocalPointNet[l].getGradient(), GetPointer(cbTemp), caLocalPointNet[l].getGradient(), 64, false, 0, 0, 0, 1)) ReturnFalse; //--- Local Net neuron = (l > 0 ? GetPointer(caLocalPointNet[l - 1]) : NeuronOCL); if(l > 0 || !cTNetG) { if(!neuron.CalcHiddenGradients(caLocalPointNet[l].AsObject())) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTurnedG.CalcHiddenGradients(caLocalPointNet[l].AsObject())) ReturnFalse; int window = (int)MathSqrt(cTNetG.Neurons()); if(IsStopped() || !MatMulGrad(neuron.getOutput(), neuron.getGradient(), cTNetG.getOutput(), cTNetG.getGradient(), cTurnedG.getGradient(), neuron.Neurons() / window, window, window)) ReturnFalse; if(!OrthoganalLoss(cTNetG, true)) ReturnFalse; //--- CBufferFloat *temp = neuron.getGradient(); neuron.SetGradient(cTurnedG.getGradient(), false); cTurnedG.SetGradient(temp, false); //--- if(!neuron.CalcHiddenGradients(cTNetG.AsObject())) ReturnFalse; if(!SumAndNormilize(neuron.getGradient(), cTurnedG.getGradient(), neuron.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- CNeuronBaseOCL *q = NULL, *k = NULL, *v = NULL, *kv = NULL; //--- Init Points CNeuronBaseOCL *neuron = cCenterPoints[1]; if(!neuron || !neuron.UpdateInputWeights(cCenterPoints[0])) ReturnFalse; //--- Inside Layers for(int l = 0; l < 2; l++) { //--- Segmentation Inputs if(l > 0 || !cTNetG) { if(!caLocalPointNet[l].UpdateInputWeights((l == 0 ? NeuronOCL : GetPointer(caLocalPointNet[l - 1])))) ReturnFalse; } else { if(!cTurnedG) ReturnFalse; if(!cTNetG.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caLocalPointNet[0].UpdateInputWeights(cTurnedG.AsObject())) ReturnFalse; } //--- Interpolate center points neuron = cCenterPoints[l + 2]; if(!neuron || !neuron.UpdateInputWeights(cCenterPoints[l + 1], caLocalPointNet[l].getOutput())) ReturnFalse; //--- Structure-Embedding Attention q = cQuery[l]; k = cKey[l]; v = cValue[l]; //--- Query if(!q || !q.UpdateInputWeights(GetPointer(caLocalPointNet[l]))) ReturnFalse; //--- Key if(!k || !k.UpdateInputWeights(cCenterPoints[l + 2])) ReturnFalse; //--- Value if(!v || !v.UpdateInputWeights(cCenterPoints[l + 2])) ReturnFalse; //--- Scale neuron = cAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[l])) ReturnFalse; //--- Feed Forward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.UpdateInputWeights(cResidual[l * 2])) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; //--- Final Attention neuron = cFinalAttention[l]; if(!neuron || !neuron.UpdateInputWeights(cResidual[l * 2 + 1])) ReturnFalse; } //--- Cross scale attention if(!SEOut.UpdateInputWeights(cFinalAttention[0], neuron.getOutput())) ReturnFalse; //--- Global Point Cloud Embedding if(!CNeuronPointNetOCL::updateInputWeights(SEOut.AsObject())) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Save(const int file_handle) { if(!CNeuronPointNet2OCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPoints) < INT_VALUE) ReturnFalse; //--- Save objects if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cKeyValue.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; if(!cCenterPoints.Save(file_handle)) ReturnFalse; if(!cFinalAttention.Save(file_handle)) ReturnFalse; if(!SEOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSEFormer::Load(const int file_handle) { if(!CNeuronPointNet2OCL::Load(file_handle)) ReturnFalse; //--- Save constants iUnits = (uint)FileReadInteger(file_handle); iPoints = (uint)FileReadInteger(file_handle); //--- Save objects if(!cQuery.Load(file_handle)) ReturnFalse; if(!cKey.Load(file_handle)) ReturnFalse; if(!cValue.Load(file_handle)) ReturnFalse; if(!cKeyValue.Load(file_handle)) ReturnFalse; if(!cMHAttentionOut.Load(file_handle)) ReturnFalse; if(!cAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; if(!cCenterPoints.Load(file_handle)) ReturnFalse; if(!cFinalAttention.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, SEOut.AsObject())) ReturnFalse; //--- Score int s = int(iUnits * iPoints * 4); for(int i = 0; i < 2; i++) { OpenCL.BufferFree(cScores[i]); if(!cScores.Update(i, OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE)) || cScores[i] < 0) ReturnFalse; } //--- CNeuronBaseOCL *ff = NULL, *residual = NULL; for(int i = 1; i < cFeedForward.Total(); i += 2) { ff = cFeedForward[i]; residual = cResidual[i]; if(!ff || !residual || !residual.SetGradient(ff.getGradient())) ReturnFalse; } //--- int buf_size = MathMax(((CNeuronBaseOCL*)cCenterPoints[2]).Neurons(), caLocalPointNet[0].Neurons()); cbTemp.BufferFree(); if(!cbTemp.BufferInit(buf_size, 0) || !cbTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSEFormer::SetOpenCL(COpenCLMy * obj) { bool create_score = false; if(!!OpenCL && obj != OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); create_score = true; } //--- CNeuronPointNet2OCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cCenterPoints.SetOpenCL(OpenCL); cFinalAttention.SetOpenCL(OpenCL); SEOut.SetOpenCL(OpenCL); //--- Score int s = int(iUnits * iPoints * 4); for(int i = 0; i < 2 && create_score; i++) cScores.Update(i, OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE)); //--- cbTemp.BufferFree(); cbTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSPFormer : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; uint iHeads; uint iSPWindow; uint iSPUnits; uint iSPHeads; uint iWindowKey; uint iLayers; uint iLayersSP; //--- CLayer cSuperPoints; CLayer cQuery; CLayer cSPKeyValue; CLayer cMask; CArrayInt cScores; CLayer cMHCrossAttentionOut; CLayer cCrossAttentionOut; CLayer cResidual; CLayer cQKeyValue; CLayer cMHSelfAttentionOut; CLayer cSelfAttentionOut; CLayer cFeedForward; CBufferFloat cTempSP; CBufferFloat cTempQ; CBufferFloat cTempSelfKV; CBufferFloat cTempCrossKV; //--- virtual bool CreateBuffers(void); virtual bool AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *kv, const int scores, CNeuronBaseOCL *out, CNeuronBaseOCL *mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f); virtual bool AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *kv, const int scores, CNeuronBaseOCL *out, CNeuronBaseOCL *mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSPFormer(void) {}; ~CNeuronSPFormer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSPFormer; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; //--- Init SuperPoints for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, r + 2, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } } //--- CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 6, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 7, OpenCL, iSPWindow, iSPWindow, 2 * iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKeyValue.Add(conv)) ReturnFalse; } //--- Mask conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 8, OpenCL, iSPWindow, iSPWindow, iUnits * iHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cMask.Add(conv)) ReturnFalse; transp = new CNeuronTransposeOCL(); if(!transp) ReturnFalse; if(!transp.Init(0, l * 14 + 9, OpenCL, iSPUnits, iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMask.Add(transp)) ReturnFalse; //--- MH Cross Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 10, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; //--- Cross Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 11, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 12, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 13, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; //--- Key-Value if(l % iLayersSP == 0) { conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 14, OpenCL, iWindow, iWindow, 2 * iWindowKey * iSPHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKeyValue.Add(conv)) ReturnFalse; } //--- MH Attention out base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 15, OpenCL, iWindow * iUnits * iHeads, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; //--- Attention out conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 16, OpenCL, iWindow * iHeads, iWindow * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 17, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; //--- FeedForward conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 18, OpenCL, iWindow, iWindow, iWindow * 4, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv) ReturnFalse; if(!conv.Init(0, l * 14 + 19, OpenCL, iWindow * 4, iWindow * 4, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(0, l * 14 + 20, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(l == (iLayers - 1)) { if(!SetGradient(conv.getGradient())) ReturnFalse; if(!SetOutput(base.getOutput())) ReturnFalse; } } //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionOut(CNeuronBaseOCL * q, CNeuronBaseOCL * kv, const int scores, CNeuronBaseOCL * out, CNeuronBaseOCL * mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f ) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHMaskAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_mask_at_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_at_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_at_score, scores) setBuffer(kernel, def_k_mask_at_out, out.getOutputIndex()) setBuffer(kernel, def_k_mask_at_mask, (!mask ? scores : mask.getOutputIndex())) setArgument(kernel, def_k_mask_at_dimension, dimension) setArgument(kernel, def_k_mask_at_heads_kv, heads_kv) setArgument(kernel, def_k_mask_at_mask_level, (!mask ? 0 : mask_level)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::AttentionInsideGradients(CNeuronBaseOCL * q, CNeuronBaseOCL * kv, const int scores, CNeuronBaseOCL * out, CNeuronBaseOCL * mask, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const float mask_level = 0.5f) { if(!OpenCL || !q || !kv || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHMaskAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_mask_atg_q, q.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_kv, kv.getOutputIndex()) setBuffer(kernel, def_k_mask_atg_kv_g, kv.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_scores, scores) setBuffer(kernel, def_k_mask_atg_gradient, out.getGradientIndex()) setBuffer(kernel, def_k_mask_atg_mask, (!mask ? scores : mask.getOutputIndex())) setBuffer(kernel, def_k_mask_atg_mask_g, (!mask ? scores : mask.getGradientIndex())) setArgument(kernel, def_k_mask_atg_kunits, units_kv) setArgument(kernel, def_k_mask_atg_heads_kv, heads_kv) setArgument(kernel, def_k_mask_atg_mask_level, (!mask ? 0 : mask_level)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.FeedForward(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.FeedForward(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.FeedForward(superpoints)) ReturnFalse; neuron = cMask[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cMask[l * 2])) ReturnFalse; if(!AttentionOut(q, kv_cross, cScores[l * 2], cMHCrossAttentionOut[l], neuron, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3]; if(!inputs || !SumAndNormilize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.FeedForward(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.FeedForward(inputs)) ReturnFalse; } if(!AttentionOut(q, kv_self, cScores[l * 2 + 1], cMHSelfAttentionOut[l], NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 1]; if(!inputs || !SumAndNormilize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.FeedForward(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.FeedForward(cFeedForward[l * 2])) ReturnFalse; q = inputs; inputs = cResidual[l * 3 + 2]; if(!inputs || !SumAndNormilize(q.getOutput(), neuron.getOutput(), inputs.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *superpoints = cSuperPoints[cSuperPoints.Total() - 1]; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = cSPKeyValue[cSPKeyValue.Total() - 1], *kv_self = cQKeyValue[cQKeyValue.Total() - 1]; //--- if(!cTempSP.Fill(0) || !cTempSelfKV.Fill(0) || !cTempCrossKV.Fill(0)) ReturnFalse; for(int l = int(iLayers - 1); l >= 0; l--) { //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; neuron = cResidual[l * 3 + 1]; if(!neuron || !neuron.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; if(!SumAndNormilize(((CNeuronBaseOCL*)cResidual[l * 3 + 2]).getGradient(), neuron.getGradient(), ((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Self-Attention neuron = cMHSelfAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cSelfAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 3]; if(((l + 1) % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !cTempSelfKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_self, cScores[l * 2 + 1], neuron, NULL, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormilize(kv_self.getGradient(), GetPointer(cTempSelfKV), kv_self.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormilize(kv_self.getGradient(), GetPointer(cTempSelfKV), GetPointer(cTempSelfKV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } inputs = cResidual[l * 3]; if(!inputs || !inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if((l % iLayersSP) == 0) { CBufferFloat *temp = inputs.getGradient(); if(!inputs.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!inputs.CalcHiddenGradients(kv_self, NULL)) ReturnFalse; if(!SumAndNormilize(temp, GetPointer(cTempQ), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!inputs.SetGradient(temp, false)) ReturnFalse; } if(!SumAndNormilize(((CNeuronBaseOCL*)cSelfAttentionOut[l]).getGradient(), inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross Attention neuron = cMHCrossAttentionOut[l]; if(!neuron || !neuron.CalcHiddenGradients(cCrossAttentionOut[l])) ReturnFalse; q = cQuery[l * 2 + 2]; if(((l + 1) % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !cTempCrossKV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(q, kv_cross, cScores[l * 2], neuron, cMask[l * 2 + 1], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; inputs = (l == 0 ? cQuery[1] : cResidual[l * 3 - 1]); if(!inputs.CalcHiddenGradients(q, NULL)) ReturnFalse; if(!SumAndNormilize(inputs.getGradient(), ((CNeuronBaseOCL*)cCrossAttentionOut[l]).getGradient(), inputs.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if((l % iLayersSP) == 0) { if(!superpoints.CalcHiddenGradients(kv_cross, NULL)) ReturnFalse; if(!SumAndNormilize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.CalcHiddenGradients(cMask[l * 2 + 1]) || !DeActivation(neuron.getOutput(), neuron.getGradient(), neuron.getGradient(), neuron.Activation())) ReturnFalse; if(!superpoints.CalcHiddenGradients(neuron, NULL)) ReturnFalse; if(l == 0) { if(!SumAndNormilize(superpoints.getGradient(), GetPointer(cTempSP), superpoints.getGradient(), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormilize(superpoints.getGradient(), GetPointer(cTempSP), GetPointer(cTempSP), iSPWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- for(int l = cSuperPoints.Total() - 2; l >= 0; l--) { superpoints = cSuperPoints[l]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[l + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(superpoints, NULL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *superpoints = NeuronOCL; CNeuronBaseOCL *neuron = NULL, *inputs = NULL, *q = NULL, *kv_cross = NULL, *kv_self = NULL; //--- Superpoints for(int l = 0; l < cSuperPoints.Total(); l++) { neuron = cSuperPoints[l]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; superpoints = neuron; } //--- Query neuron = cQuery[1]; if(!neuron || !neuron.UpdateInputWeights(cQuery[0])) ReturnFalse; //--- inputs = neuron; for(uint l = 0; l < iLayers; l++) { //--- Cross Attentionn q = cQuery[l * 2 + 2]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_cross = cSPKeyValue[l / iLayersSP]; if(!kv_cross || !kv_cross.UpdateInputWeights(superpoints)) ReturnFalse; } neuron = cMask[l * 2]; if(!neuron || !neuron.UpdateInputWeights(superpoints)) ReturnFalse; neuron = cCrossAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Attention q = cQuery[l * 2 + 3]; if(!q || !q.UpdateInputWeights(inputs)) ReturnFalse; if((l % iLayersSP) == 0) { kv_self = cQKeyValue[l / iLayersSP]; if(!kv_self || !kv_self.UpdateInputWeights(inputs)) ReturnFalse; } neuron = cSelfAttentionOut[l]; if(!neuron || !neuron.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- FeedForward neuron = cFeedForward[l * 2]; if(!neuron || !neuron.UpdateInputWeights(inputs)) ReturnFalse; neuron = cFeedForward[l * 2 + 1]; if(!neuron || !neuron.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSPFormer::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL);; cQuery.SetOpenCL(OpenCL); cSPKeyValue.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cQKeyValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::CreateBuffers(void) { for(uint l = 0; l < iLayers; l++) { //--- Cross Attention int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); vector temp = vector::Zeros(s); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; if(!OpenCL.BufferFromVector(s, temp, CL_MEM_READ_WRITE)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iSPWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempSelfKV.BufferInit(2 * iWindowKey * iUnits * iHeads, 0) || !cTempSelfKV.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossKV.BufferInit(2 * iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossKV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cSPKeyValue.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cQKeyValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSPFormer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cSPKeyValue.Load(file_handle)) ReturnFalse; if(!cMask.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cQKeyValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(uint i = 0; i < iLayers; i++) { neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; } //--- if(Output != neuron.getOutput()) if(!SetOutput(neuron.getOutput())) ReturnFalse; //--- if(!CreateBuffers()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMAFT : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; uint iHeads; uint iSPWindow; uint iSPUnits; uint iSPHeads; uint iWindowKey; uint iLayers; uint iLayersSP; //--- CLayer cSuperPoints; CLayer cQuery; CLayer cQPosition; CLayer cQKey; CLayer cQValue; CLayer cMHSelfAttentionOut; CLayer cSelfAttentionOut; CLayer cSPKey; CLayer cSPValue; CArrayInt cScores; CArrayInt cPositionBias; CLayer cMHCrossAttentionOut; CLayer cCrossAttentionOut; CLayer cResidual; CLayer cFeedForward; CBufferFloat cTempSP; CBufferFloat cTempQ; CBufferFloat cTempCrossK; CBufferFloat cTempCrossV; //--- virtual bool CreateBuffers(void); virtual bool CalcPositionBias(CBufferFloat *pos_q, CBufferFloat *pos_k, const int pos_bias, const int units, const int units_kv, const int dimension); virtual bool AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v, const int scores, CNeuronBaseOCL *out, const int pos_bias, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const bool use_pos_bias); virtual bool AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v, const int scores, CNeuronBaseOCL *out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMAFT(void) {}; ~CNeuronMAFT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMAFT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- Init Querys CNeuronBaseOCL *base = new CNeuronBaseOCL(); if(!base) ReturnFalse; if(!base.Init(iWindow * iUnits, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *buf = base.getOutput(); if(!buf || !buf.BufferInit(1, 1) || !buf.BufferWrite()) ReturnFalse; buf = base.getWeights(); if(!buf || !buf.BufferInit(buf.Total(), 0) || !buf.BufferWrite()) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 1, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(base)) ReturnFalse; CNeuronLearnabledPE *pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 2, OpenCL, base.Neurons(), optimization, iBatch)) ReturnFalse; if(!cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 3, OpenCL, pe.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetOutput(pe.GetPE())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 4; for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual) ReturnFalse; if(!residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; } layer_id++; } CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(conv)) ReturnFalse; layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch)) ReturnFalse; if(!cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch)) ReturnFalse; if(!cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMAFT::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Self-Attention //--- Score int s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross Attention s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionOut(CNeuronBaseOCL * q, CNeuronBaseOCL * k, CNeuronBaseOCL * v, const int scores, CNeuronBaseOCL * out, const int pos_bias, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension, const bool use_pos_bias) { if(!OpenCL || !q || !k || !v || !out || scores < 0 || (use_pos_bias && pos_bias < 0)) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, (use_pos_bias ? pos_bias : scores)) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, int(use_pos_bias)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::AttentionInsideGradients(CNeuronBaseOCL * q, CNeuronBaseOCL * k, CNeuronBaseOCL * v, const int scores, CNeuronBaseOCL * out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::CalcPositionBias(CBufferFloat * pos_q, CBufferFloat * pos_k, const int pos_bias, const int units, const int units_kv, const int dimension) { if(!OpenCL || !pos_q || !pos_k || pos_bias < 0) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {units/*Q units*/, units_kv}; int kernel = def_k_CalcPositionBias; //--- ResetLastError(); setBuffer(kernel, def_k_cpb_data1, pos_q.GetIndex()) setBuffer(kernel, def_k_cpb_data2, pos_k.GetIndex()) setBuffer(kernel, def_k_cpb_result, pos_bias) setArgument(kernel, def_k_cpb_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.FeedForward(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 2], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormilize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; value = cQPosition[l * 2]; if(!value || !SumAndNormilize(inputs.getOutput(), value.getOutput(), inputs.getOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Calc Position bias if(!CalcPositionBias(value.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 2 + 1], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormilize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormilize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormilize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormilize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cCrossAttentionOut[l]; if(!value || !SumAndNormilize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 4]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2 + 1], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormilize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormilize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormilize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 2 + 3]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; if(l == 0) base = cQuery[2]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), residual.getGradient(), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), GetPointer(cTempQ), GetPointer(cTempQ), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), GetPointer(cTempQ), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormilize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- Qeury query = cQuery[1]; if(!query || !query.CalcHiddenGradients(cQuery[2])) ReturnFalse; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[2]).AddPEGradient(base.getGradient()))) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormilize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } //--- From Value superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormilize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *inputs = NULL; for(int i = 0; i < 2; i++) { inputs = cQuery[i + 1]; if(!inputs || !inputs.UpdateInputWeights(cQuery[i])) ReturnFalse; } CNeuronBaseOCL *query = NULL, *key = NULL, *value = NULL, *base = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Self-Atention query = cQuery[l * 2 + 3]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; base = cSelfAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Cross-Attention query = cQuery[l * 2 + 4]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iSPHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayers) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLayersSP) < INT_VALUE) ReturnFalse; //--- Save Objects if(!cSuperPoints.Save(file_handle)) ReturnFalse; if(!cQuery.Save(file_handle)) ReturnFalse; if(!cQPosition.Save(file_handle)) ReturnFalse; if(!cQKey.Save(file_handle)) ReturnFalse; if(!cQValue.Save(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Save(file_handle)) ReturnFalse; if(!cSPKey.Save(file_handle)) ReturnFalse; if(!cSPValue.Save(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMAFT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iSPWindow = (uint)FileReadInteger(file_handle); iSPUnits = (uint)FileReadInteger(file_handle); iSPHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iLayers = (uint)FileReadInteger(file_handle); iLayersSP = (uint)FileReadInteger(file_handle); //--- Load Objects if(!cSuperPoints.Load(file_handle)) ReturnFalse; if(!cQuery.Load(file_handle)) ReturnFalse; if(!cQPosition.Load(file_handle)) ReturnFalse; if(!cQKey.Load(file_handle)) ReturnFalse; if(!cQValue.Load(file_handle)) ReturnFalse; if(!cMHSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSelfAttentionOut.Load(file_handle)) ReturnFalse; if(!cSPKey.Load(file_handle)) ReturnFalse; if(!cSPValue.Load(file_handle)) ReturnFalse; if(!cMHCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cCrossAttentionOut.Load(file_handle)) ReturnFalse; if(!cResidual.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- CBufferFloat *grad = ((CNeuronBaseOCL*)cFeedForward[cFeedForward.Total() - 1]).getGradient(); if(Gradient != grad) if(!SetGradient(grad)) ReturnFalse; for(uint i = 0; i < iLayers; i++) { CNeuronBaseOCL *neuron = cResidual[i * 3 + 2]; if(!neuron) ReturnFalse; grad = ((CNeuronBaseOCL*)cFeedForward[i * 2 + 1]).getGradient(); if(neuron.getGradient() != grad) if(!neuron.SetGradient(grad)) ReturnFalse; neuron = cQPosition[i * 2 + 1]; if(!neuron || !(((CNeuronBaseOCL*)cQPosition[(i + 1) * 2]).SetGradient(neuron.getGradient())) ) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[cQPosition.Total() - 1]).SetGradientIndex(getGradientIndex()))) ReturnFalse; if(Type() == defNeuronMAFT) { if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[2]).GetPE()))) ReturnFalse; } //--- if(!CreateBuffers()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGRES : public CNeuronMAFT { protected: CLayer cReference; CLayer cRefKey; CLayer cRefValue; CLayer cMHRefAttentionOut; CLayer cRefAttentionOut; //--- virtual bool CreateBuffers(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronGRES(void) {}; ~CNeuronGRES(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint ref_size, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronGRES; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DiversityLoss(CNeuronBaseOCL * neuron, const int units, const int dimension, const bool add = false) { if(!OpenCL || !neuron) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {units/*Q units*/, (uint)MathMin(units, OpenCL.GetMaxLocalSize(1))}; uint local_work_size[] = {1, global_work_size[1] }; int kernel = def_k_DiversityLoss; //--- ResetLastError(); setBuffer(kernel, def_k_dl_data, neuron.getOutputIndex()) setBuffer(kernel, def_k_dl_grad, neuron.getGradientIndex()) setArgument(kernel, def_k_dl_dimension, dimension) setArgument(kernel, def_k_dl_activation, neuron.Activation()) setArgument(kernel, def_k_dl_add, (int)add) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_sp, uint units_sp, uint heads_sp, uint ref_size, uint layers, uint layers_to_sp, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; iHeads = heads; iSPUnits = units_sp; iSPWindow = window_sp; iSPHeads = heads_sp; iWindowKey = window_key; iLayers = MathMax(layers, 1); iLayersSP = MathMax(layers_to_sp, 1); //--- CNeuronBaseOCL *base = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronConvOCL *conv = NULL; CNeuronLearnabledPE *pe = NULL; //--- Init Querys cQuery.Clear(); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 0, OpenCL, iSPUnits, iSPWindow, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iSPUnits, iSPUnits, iUnits, 1, iSPWindow, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 2, OpenCL, iSPWindow, iUnits, optimization, iBatch) || !cQuery.Add(transp)) ReturnFalse; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 3, OpenCL, iSPWindow, iSPWindow, iWindow, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, 4, OpenCL, iWindow * iUnits, optimization, iBatch) || !cQuery.Add(pe)) ReturnFalse; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, 5, OpenCL, pe.Neurons(), optimization, iBatch) || !base.SetOutput(pe.GetPE()) || !cQPosition.Add(base)) ReturnFalse; //--- Init SuperPoints int layer_id = 6; cSuperPoints.Clear(); for(int r = 0; r < 4; r++) { if(iSPUnits % 2 == 0) { iSPUnits /= 2; CResidualConv *residual = new CResidualConv(); if(!residual || !residual.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPUnits, optimization, iBatch) || !cSuperPoints.Add(residual)) ReturnFalse; } else { iSPUnits--; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 2 * iSPWindow, iSPWindow, iSPWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); } layer_id++; } conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iSPWindow, iSPWindow, iWindow, iSPUnits, 1, optimization, iBatch) || !cSuperPoints.Add(conv)) ReturnFalse; conv.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch) || !cSuperPoints.Add(pe)) ReturnFalse; layer_id++; //--- Reference cReference.Clear(); base = new CNeuronBaseOCL(); if(!base || !base.Init(iWindow * iUnits, layer_id, OpenCL, ref_size, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cReference.Add(base)) ReturnFalse; base.SetActivationFunction(SIGMOID); layer_id++; pe = new CNeuronLearnabledPE(); if(!pe || !pe.Init(0, layer_id, OpenCL, base.Neurons(), optimization, iBatch) || !cReference.Add(pe)) ReturnFalse; layer_id++; //--- Inside layers cQKey.Clear(); cQValue.Clear(); cSPKey.Clear(); cSPValue.Clear(); cSelfAttentionOut.Clear(); cCrossAttentionOut.Clear(); cMHCrossAttentionOut.Clear(); cMHSelfAttentionOut.Clear(); cMHRefAttentionOut.Clear(); cRefAttentionOut.Clear(); cRefKey.Clear(); cRefValue.Clear(); cResidual.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; if(l % iLayersSP == 0) { //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iSPHeads, iSPUnits, 1, optimization, iBatch) || !cSPValue.Add(conv)) ReturnFalse; layer_id++; } //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHCrossAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cCrossAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch) || !cResidual.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHSelfAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Self-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cSelfAttentionOut.Add(conv)) ReturnFalse; layer_id++; //--- Reference Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cQuery.Add(conv)) ReturnFalse; layer_id++; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefKey.Add(conv)) ReturnFalse; layer_id++; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cRefValue.Add(conv)) ReturnFalse; layer_id++; //--- Multy-Heads Attention Out base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHRefAttentionOut.Add(base)) ReturnFalse; layer_id++; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cRefAttentionOut.Add(conv)) ReturnFalse; layer_id++; if(!conv.SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[cSelfAttentionOut.Total() - 1]).getGradient(), true)) ReturnFalse; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(LReLU); if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; if(!cFeedForward.Add(conv)) ReturnFalse; layer_id++; //--- Residual base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, iWindow * iUnits, optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cResidual.Add(base)) ReturnFalse; layer_id++; //--- Delta position conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer_id, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; conv.SetActivationFunction(SIGMOID); if(!cQPosition.Add(conv)) ReturnFalse; layer_id++; base = new CNeuronBaseOCL(); if(!base || !base.Init(0, layer_id, OpenCL, conv.Neurons(), optimization, iBatch)) ReturnFalse; if(!base.SetGradient(conv.getGradient())) ReturnFalse; if(!cQPosition.Add(base)) ReturnFalse; layer_id++; } //--- base = cResidual[iLayers * 3 - 1]; if(!SetGradient(base.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::CreateBuffers(void) { cScores.Clear(); cPositionBias.Clear(); for(uint l = 0; l < iLayers; l++) { //--- Cross Attention //--- Score int s = int(iUnits * iSPUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Position Bias s = int(iUnits * iSPUnits); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cPositionBias.Add(s)) ReturnFalse; //--- Self-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Reference Cross-Attention //--- Score s = int(iUnits * iUnits * iHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; } //--- if(!cTempSP.BufferInit(iWindow * iSPUnits, 0) || !cTempSP.BufferCreate(OpenCL)) ReturnFalse; if(!cTempQ.BufferInit(iWindow * iUnits, 0) || !cTempQ.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossK.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossK.BufferCreate(OpenCL)) ReturnFalse; if(!cTempCrossV.BufferInit(iWindowKey * iSPUnits * iSPHeads, 0) || !cTempCrossV.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).FeedForward(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).FeedForward(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; if(!SecondInput) ReturnFalse; if(reference.getOutput() != SecondInput) if(!reference.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).FeedForward(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Calc Position bias cross = cQPosition[l * 2]; if(!cross || !CalcPositionBias(cross.getOutput(), ((CNeuronLearnabledPE*)superpoints).GetPE(), cPositionBias[l], iUnits, iSPUnits, iWindow)) ReturnFalse; //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.FeedForward(superpoints)) ReturnFalse; if(!value || !value.FeedForward(cSuperPoints[total_sp - 2])) ReturnFalse; } if(!AttentionOut(query, key, value, cScores[l * 3], cMHCrossAttentionOut[l], cPositionBias[l], iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey, true)) ReturnFalse; base = cCrossAttentionOut[l]; if(!base || !base.FeedForward(cMHCrossAttentionOut[l])) ReturnFalse; value = cResidual[l * 3]; if(!value || !SumAndNormilize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormilize(cross.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.FeedForward(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.FeedForward(inputs)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 1], cMHSelfAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.FeedForward(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.FeedForward(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.FeedForward(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.FeedForward(reference)) ReturnFalse; if(!AttentionOut(query, key, value, cScores[l * 3 + 2], cMHRefAttentionOut[l], -1, iUnits, iHeads, iUnits, iHeads, iWindowKey, false)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.FeedForward(cMHRefAttentionOut[l])) ReturnFalse; value = cResidual[l * 3 + 1]; if(!value || !SumAndNormilize(cross.getOutput(), self.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 1) || !SumAndNormilize(inputs.getOutput(), value.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.FeedForward(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.FeedForward(cFeedForward[l * 2])) ReturnFalse; value = cResidual[l * 3 + 2]; if(!value || !SumAndNormilize(inputs.getOutput(), base.getOutput(), value.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; inputs = value; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.FeedForward(inputs)) ReturnFalse; value = cQPosition[(l + 1) * 2]; query = cQPosition[l * 2]; if(!value || !SumAndNormilize(query.getOutput(), base.getOutput(), value.getOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; } //--- value = cQPosition[iLayers * 2]; if(!value || !SumAndNormilize(inputs.getOutput(), value.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGRES::SetOpenCL(COpenCLMy * obj) { if(OpenCL == obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); for(int i = 0; i < cPositionBias.Total(); i++) OpenCL.BufferFree(cPositionBias[i]); } } //--- cScores.Clear(); cPositionBias.Clear(); //--- CNeuronBaseOCL::SetOpenCL(obj); cSuperPoints.SetOpenCL(OpenCL); cQuery.SetOpenCL(OpenCL); cQPosition.SetOpenCL(OpenCL); cQKey.SetOpenCL(OpenCL); cQValue.SetOpenCL(OpenCL); cMHSelfAttentionOut.SetOpenCL(OpenCL); cSelfAttentionOut.SetOpenCL(OpenCL); cSPKey.SetOpenCL(OpenCL); cSPValue.SetOpenCL(OpenCL); cMHCrossAttentionOut.SetOpenCL(OpenCL); cCrossAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cReference.SetOpenCL(OpenCL); cRefKey.SetOpenCL(OpenCL); cRefValue.SetOpenCL(OpenCL); cMHRefAttentionOut.SetOpenCL(OpenCL); cRefAttentionOut.SetOpenCL(OpenCL); //--- (((CNeuronBaseOCL*)cQPosition[iLayers * 2]).SetGradientIndex(getGradientIndex())); //--- Buffers CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *residual = GetPointer(this), *query = NULL, *key = NULL, *value = NULL, *key_sp = NULL, *value_sp = NULL, *base = NULL; //--- Inside layers for(int l = (int)iLayers - 1; l >= 0; l--) { //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2 + 1])) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base || !base.CalcHiddenGradients(cFeedForward[l * 2])) ReturnFalse; //--- Residual value = cSelfAttentionOut[l]; if(!value || !SumAndNormilize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Reference Cross-Attention base = cMHRefAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cRefAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 7]; key = cRefKey[l]; value = cRefValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 3 + 2], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; value = cCrossAttentionOut[l]; if(!SumAndNormilize(base.getGradient(), residual.getGradient(), value.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = value; //--- Self-Attention base = cMHSelfAttentionOut[l]; if(!base || !base.CalcHiddenGradients(cSelfAttentionOut[l], NULL)) ReturnFalse; query = cQuery[l * 3 + 6]; key = cQKey[l]; value = cQValue[l]; if(!AttentionInsideGradients(query, key, value, cScores[l * 2 + 1], base, iUnits, iHeads, iUnits, iHeads, iWindowKey)) ReturnFalse; base = cResidual[l * 3 + 1]; if(!base.CalcHiddenGradients(query, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(key, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(value, NULL)) ReturnFalse; if(!SumAndNormilize(base.getGradient(), residual.getGradient(), residual.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Qeury position base = cQPosition[l * 2]; value = cQPosition[(l + 1) * 2]; if(!base || !SumAndNormilize(value.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- Cross-Attention base = cMHCrossAttentionOut[l]; if(!base || !base.CalcHiddenGradients(residual, NULL)) ReturnFalse; query = cQuery[l * 3 + 5]; if(((l + 1) % iLayersSP) == 0 || (l + 1) == iLayers) { key_sp = cSPKey[l / iLayersSP]; value_sp = cSPValue[l / iLayersSP]; if(!key_sp || !value_sp || !cTempCrossK.Fill(0) || !cTempCrossV.Fill(0)) ReturnFalse; } if(!AttentionInsideGradients(query, key_sp, value_sp, cScores[l * 2], base, iUnits, iHeads, iSPUnits, iSPHeads, iWindowKey)) ReturnFalse; if(iLayersSP > 1) { if((l % iLayersSP) == 0) { if(!SumAndNormilize(key_sp.getGradient(), GetPointer(cTempCrossK), key_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(value_sp.getGradient(), GetPointer(cTempCrossV), value_sp.getGradient(), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!SumAndNormilize(key_sp.getGradient(), GetPointer(cTempCrossK), GetPointer(cTempCrossK), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(value_sp.getGradient(), GetPointer(cTempCrossV), GetPointer(cTempCrossV), iWindowKey, false, 0, 0, 0, 1)) ReturnFalse; } } if(l == 0) base = cQuery[4]; else base = cResidual[l * 3 - 1]; if(!base || !base.CalcHiddenGradients(query, NULL)) ReturnFalse; //--- Residual if(!SumAndNormilize(base.getGradient(), residual.getGradient(), base.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; residual = base; } //--- Qeury query = cQuery[3]; if(!query || !query.CalcHiddenGradients(cQuery[4])) ReturnFalse; base = cQPosition[0]; if(!DeActivation(base.getOutput(), base.getGradient(), base.getGradient(), SIGMOID) || !(((CNeuronLearnabledPE*)cQuery[4]).AddPEGradient(base.getGradient()))) ReturnFalse; if(!DiversityLoss(query, iUnits, iWindow, true)) ReturnFalse; for(int i = 2; i >= 0; i--) { query = cQuery[i]; if(!query || !query.CalcHiddenGradients(cQuery[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(query, NULL)) ReturnFalse; CBufferFloat *inputs_gr = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(query.getGradient(), false)) ReturnFalse; //--- Superpoints //--- From Key int total_sp = cSuperPoints.Total(); CNeuronBaseOCL *superpoints = cSuperPoints[total_sp - 1]; if(!superpoints || !superpoints.CalcHiddenGradients(cSPKey[0])) ReturnFalse; if(cSPKey.Total() > 1) { CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 1; i < cSPKey.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPKey[i]) || !SumAndNormilize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; } superpoints = cSuperPoints[total_sp - 2]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[total_sp - 1])) ReturnFalse; //--- From Value CBufferFloat *grad = superpoints.getGradient(); if(!superpoints.SetGradient(GetPointer(cTempSP), false)) ReturnFalse; for(int i = 0; i < cSPValue.Total(); i++) { if(!superpoints.CalcHiddenGradients(cSPValue[i]) || !SumAndNormilize(superpoints.getGradient(), grad, grad, iWindow, false, 0, 0, 0, 1)) ReturnFalse; } if(!superpoints.SetGradient(grad, false)) ReturnFalse; if(!DiversityLoss(superpoints, iSPUnits, iSPWindow, true)) ReturnFalse; //--- for(int i = total_sp - 3; i >= 0; i--) { superpoints = cSuperPoints[i]; if(!superpoints || !superpoints.CalcHiddenGradients(cSuperPoints[i + 1])) ReturnFalse; } //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cSuperPoints[0])) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(inputs_gr, false)) ReturnFalse; //--- Reference base = cReference[0]; if(base.getGradient() != SecondGradient) { if(!base.SetGradient(SecondGradient)) ReturnFalse; base.SetActivationFunction(SecondActivation); } base = cReference[2]; if(!base || !base.CalcHiddenGradients(cRefKey[0])) ReturnFalse; inputs_gr = base.getGradient(); if(!base.SetGradient(GetPointer(cTempQ), false)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[0])) ReturnFalse; if(!SumAndNormilize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; for(uint i = 1; i < iLayers; i++) { if(!base.CalcHiddenGradients(cRefKey[i])) ReturnFalse; if(!SumAndNormilize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!base.CalcHiddenGradients(cRefValue[i])) ReturnFalse; if(!SumAndNormilize(base.getGradient(), inputs_gr, inputs_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!base.SetGradient(inputs_gr, false)) ReturnFalse; base = cReference[1]; if(!base.CalcHiddenGradients(cReference[2])) ReturnFalse; if(!DiversityLoss(base, iUnits, iWindow, true)) ReturnFalse; base = cReference[0]; if(!base.CalcHiddenGradients(cReference[1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- Superpoints CNeuronBaseOCL *superpoints = NeuronOCL; int total_sp = cSuperPoints.Total(); for(int i = 0; i < total_sp; i++) { if(!cSuperPoints[i] || !((CNeuronBaseOCL*)cSuperPoints[i]).UpdateInputWeights(superpoints)) ReturnFalse; superpoints = cSuperPoints[i]; } //--- Query CNeuronBaseOCL *query = NeuronOCL; for(int i = 0; i < 5; i++) { if(!cQuery[i] || !((CNeuronBaseOCL*)cQuery[i]).UpdateInputWeights(query)) ReturnFalse; query = cQuery[i]; } //--- Reference CNeuronBaseOCL *reference = cReference[0]; for(int i = 1; i < cReference.Total(); i++) { if(!cReference[i] || !((CNeuronBaseOCL*)cReference[i]).UpdateInputWeights(reference)) ReturnFalse; reference = cReference[i]; } CNeuronBaseOCL *inputs = query, *key = NULL, *value = NULL, *base = NULL, *cross = NULL, *self = NULL; //--- Inside layers for(uint l = 0; l < iLayers; l++) { //--- Cross-Attention query = cQuery[l * 3 + 5]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cSPKey[l / iLayersSP]; value = cSPValue[l / iLayersSP]; if(l % iLayersSP == 0) { if(!key || !key.UpdateInputWeights(superpoints)) ReturnFalse; if(!value || !value.UpdateInputWeights(cSuperPoints[total_sp - 2])) ReturnFalse; } base = cCrossAttentionOut[l]; if(!base || !base.UpdateInputWeights(cMHCrossAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3]; //--- Self-Atention query = cQuery[l * 3 + 6]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cQKey[l]; if(!key || !key.UpdateInputWeights(inputs)) ReturnFalse; value = cQValue[l]; if(!value || !value.UpdateInputWeights(inputs)) ReturnFalse; self = cSelfAttentionOut[l]; if(!self || !self.UpdateInputWeights(cMHSelfAttentionOut[l])) ReturnFalse; //--- Reference Cross-Attention query = cQuery[l * 3 + 7]; if(!query || !query.UpdateInputWeights(inputs)) ReturnFalse; key = cRefKey[l]; if(!key || !key.UpdateInputWeights(reference)) ReturnFalse; value = cRefValue[l]; if(!value || !value.UpdateInputWeights(reference)) ReturnFalse; cross = cRefAttentionOut[l]; if(!cross || !cross.UpdateInputWeights(cMHRefAttentionOut[l])) ReturnFalse; inputs = cResidual[l * 3 + 1]; //--- Feed Forward base = cFeedForward[l * 2]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; base = cFeedForward[l * 2 + 1]; if(!base || !base.UpdateInputWeights(cFeedForward[l * 2])) ReturnFalse; inputs = cResidual[l * 3 + 2]; //--- Delta Query position base = cQPosition[l * 2 + 1]; if(!base || !base.UpdateInputWeights(inputs)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Save(const int file_handle) { if(!CNeuronMAFT::Save(file_handle)) ReturnFalse; //--- if(!cReference.Save(file_handle)) ReturnFalse; if(!cRefKey.Save(file_handle)) ReturnFalse; if(!cRefValue.Save(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Save(file_handle)) ReturnFalse; if(!cRefAttentionOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGRES::Load(const int file_handle) { if(!CNeuronMAFT::Load(file_handle)) ReturnFalse; //--- if(!cReference.Load(file_handle)) ReturnFalse; if(!cRefKey.Load(file_handle)) ReturnFalse; if(!cRefValue.Load(file_handle)) ReturnFalse; if(!cMHRefAttentionOut.Load(file_handle)) ReturnFalse; if(!cRefAttentionOut.Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < iLayers; i++) { if(!(((CNeuronBaseOCL*)cRefAttentionOut[i]).SetGradient(((CNeuronBaseOCL*)cSelfAttentionOut[i]).getGradient(), true))) ReturnFalse; } if(!(((CNeuronBaseOCL*)cQPosition[0]).SetOutput(((CNeuronLearnabledPE*)cQuery[4]).GetPE()))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGEGWA : public CNeuronUShapeAttention { protected: CNeuronBaseOCL cResidual; CNeuronMLCrossAttentionMLKV cCrossAttention; CBufferFloat cTemp; bool bAddNeckGradient; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronGEGWA(void) : bAddNeckGradient(false) {}; ~CNeuronGEGWA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_kv, uint heads_kv, uint units_count_kv, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronGEGWA; }///< Identificator of class.@return Type of class //--- methods for working with files virtual bool Save(int const file_handle); ///< Save method @param[in] file_handle handle of file @return logical result of operation virtual bool Load(int const file_handle); ///< Load method @param[in] file_handle handle of file @return logical result of operation //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual CNeuronBaseOCL* GetInsideLayer(const int layer) const; virtual void AddNeckGradient(const bool flag) { bAddNeckGradient = flag; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_kv, uint heads_kv, uint units_count_kv, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAttention[0].Init(0, 0, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 1, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronGEGWA *temp = new CNeuronGEGWA(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, (units_count + 1) / 2, window_kv, heads_kv, units_count_kv, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronMLCrossAttentionMLKV *temp = new CNeuronMLCrossAttentionMLKV(); if(!temp) ReturnFalse; if(!temp.Init(0, 2, OpenCL, window, window_key, heads, window_kv, heads_kv, (units_count + 1) / 2, units_count_kv, layers, 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 4, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(!cResidual.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cCrossAttention.Init(0, 6, OpenCL, window, window_key, heads, window_kv, heads_kv, units_count, units_count_kv, layers, 1, optimization, iBatch)) ReturnFalse; if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cAttention[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, false)) ReturnFalse; if(!cCrossAttention.FeedForward(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::calcInputGradients(CNeuronBaseOCL * prevLayer, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; if(!cResidual.CalcHiddenGradients(cCrossAttention.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(bAddNeckGradient) { CBufferFloat *temp = cNeck.getGradient(); if(!cNeck.SetGradient(cMergeSplit[0].getGradient(), false)) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormilize(cNeck.getGradient(), temp, temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cNeck.SetGradient(temp, false)) ReturnFalse; } else if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject(), SecondInput, GetPointer(cTemp), SecondActivation)) ReturnFalse; if(!SumAndNormilize(SecondGradient, GetPointer(cTemp), SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), GetPointer(cTemp), cMergeSplit[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), GetPointer(cTemp), prevLayer.getGradient(), 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cAttention[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject(), SecondInput)) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; if(!cCrossAttention.FeedForward(cResidual.AsObject(), SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Save(const int file_handle) { if(!CNeuronUShapeAttention::Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGEGWA::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronGEGWA: cNeck = new CNeuronGEGWA(); if(!cNeck) ReturnFalse; break; case defNeuronMLCrossAttentionMLKV: cNeck = new CNeuronMLCrossAttentionMLKV(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- cTemp.BufferFree(); if(!cTemp.BufferInit(MathMax(cCrossAttention.GetSecondBufferSize(), cAttention[0].Neurons()), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; //--- if(!cResidual.Init(0, 0, OpenCL, cCrossAttention.Neurons(), optimization, iBatch)) ReturnFalse; if(Gradient != cCrossAttention.getGradient()) { if(!SetGradient(cCrossAttention.getGradient(), true)) ReturnFalse; } if(cResidual.getGradient() != cMergeSplit[1].getGradient()) { if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; } if(Output != cCrossAttention.getOutput()) { if(!SetOutput(cCrossAttention.getOutput(), true)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGEGWA::SetOpenCL(COpenCLMy * obj) { CNeuronUShapeAttention::SetOpenCL(obj); cResidual.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL* CNeuronGEGWA::GetInsideLayer(const int layer) const { if(layer < 0) return NULL; //--- if(layer == 0) return cNeck; //--- if(!cNeck || cNeck.Type() != Type()) return NULL; //--- CNeuronGEGWA* temp = cNeck; return temp.GetInsideLayer(layer - 1); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLPC : public CNeuronMLCrossAttentionMLKV { protected: CNeuronBaseOCL cOne; CNeuronBaseOCL cPrimitives; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return feedForward(NeuronOCL); } //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return updateInputWeights(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronLPC(void) {}; ~CNeuronLPC(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronLPC; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint units_count_kv, uint layers, uint layers_to_one_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMLCrossAttentionMLKV::Init(numOutputs, myIndex, open_cl, window, window_key, heads, window, heads_kv, units_count, units_count_kv, layers, layers_to_one_kv, optimization_type, batch)) ReturnFalse; if(!cOne.Init(window * units_count, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = cOne.getOutput(); if(!out.BufferInit(1, 1) || !out.BufferWrite()) ReturnFalse; if(!cPrimitives.Init(0, 1, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(bTrain && !cPrimitives.FeedForward(cOne.AsObject())) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::feedForward(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMLCrossAttentionMLKV::calcInputGradients(cPrimitives.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!DiversityLoss(cPrimitives.AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMLCrossAttentionMLKV::updateInputWeights(cPrimitives.AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!cPrimitives.UpdateInputWeights(cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Save(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; if(!cPrimitives.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLPC::Load(const int file_handle) { if(!CNeuronMLCrossAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPrimitives.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLPC::SetOpenCL(COpenCLMy * obj) { CNeuronMLCrossAttentionMLKV::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cPrimitives.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronOCM : public CNeuronBaseOCL { protected: uint iPrimWindow; uint iPrimUnits; uint iPrimHeads; uint iContWindow; uint iContUnits; uint iContHeads; uint iWindowKey; //--- CLayer cQuery; CLayer cKey; CLayer cValue; CLayer cMHAttentionOut; CLayer cAttentionOut; CArrayInt cScores; CLayer cResidual; CLayer cFeedForward; //--- virtual bool CreateBuffers(void); virtual bool AttentionOut(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v, const int scores, CNeuronBaseOCL *out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension); virtual bool AttentionInsideGradients(CNeuronBaseOCL *q, CNeuronBaseOCL *k, CNeuronBaseOCL *v, const int scores, CNeuronBaseOCL *out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } public: CNeuronOCM(void) {}; ~CNeuronOCM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint prim_window, uint window_key, uint prim_units, uint prim_heads, uint cont_window, uint cont_units, uint cont_heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronOCM; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool feedForward(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context); virtual bool calcInputGradients(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context); virtual bool updateInputWeights(CNeuronBaseOCL *Primitives, CNeuronBaseOCL *Context); //--- virtual uint GetPrimitiveWindow(void) const { return iPrimWindow; } virtual uint GetContextWindow(void) const { return iContWindow; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionOut(CNeuronBaseOCL * q, CNeuronBaseOCL * k, CNeuronBaseOCL * v, const int scores, CNeuronBaseOCL * out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, units_kv, heads}; uint local_work_size[3] = {1, units_kv, 1}; int kernel = def_k_MHPosBiasAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_pbao_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbao_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbao_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbao_score, scores) setBuffer(kernel, def_k_pbao_pos_bias, scores) setBuffer(kernel, def_k_pbao_out, out.getOutputIndex()) setArgument(kernel, def_k_pbao_dimension, dimension) setArgument(kernel, def_k_pbao_heads_kv, heads_kv) setArgument(kernel, def_k_pbao_use_pos_bias, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::AttentionInsideGradients(CNeuronBaseOCL * q, CNeuronBaseOCL * k, CNeuronBaseOCL * v, const int scores, CNeuronBaseOCL * out, const int units, const int heads, const int units_kv, const int heads_kv, const int dimension) { if(!OpenCL || !q || !k || !v || !out || scores < 0) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {units/*Q units*/, dimension, heads}; int kernel = def_k_MHPosBiasAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_pbaog_q, q.getOutputIndex()) setBuffer(kernel, def_k_pbaog_q_g, q.getGradientIndex()) setBuffer(kernel, def_k_pbaog_k, k.getOutputIndex()) setBuffer(kernel, def_k_pbaog_k_g, k.getGradientIndex()) setBuffer(kernel, def_k_pbaog_v, v.getOutputIndex()) setBuffer(kernel, def_k_pbaog_v_g, v.getGradientIndex()) setBuffer(kernel, def_k_pbaog_scores, scores) setBuffer(kernel, def_k_pbaog_gradient, out.getGradientIndex()) setArgument(kernel, def_k_pbaog_kunits, units_kv) setArgument(kernel, def_k_pbaog_heads_kv, heads_kv) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint prim_window, uint window_key, uint prim_units, uint prim_heads, uint cont_window, uint cont_units, uint cont_heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, cont_window * cont_units, optimization_type, batch)) ReturnFalse; //--- iPrimWindow = prim_window; iPrimUnits = prim_units; iPrimHeads = prim_heads; iContWindow = cont_window; iContUnits = cont_units; iContHeads = cont_heads; iWindowKey = window_key; //--- cQuery.Clear(); cKey.Clear(); cValue.Clear(); cMHAttentionOut.Clear(); cAttentionOut.Clear(); cResidual.Clear(); cFeedForward.Clear(); //--- CNeuronBaseOCL *neuron = NULL; CNeuronConvOCL *conv = NULL; //--- Primitives Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 0, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 2, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 4, OpenCL, iPrimHeads * iWindowKey, iPrimHeads * iWindowKey, iPrimWindow, iPrimUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 6, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 7, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 8, OpenCL, iPrimWindow, iPrimWindow, iPrimHeads * iWindowKey, iPrimUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Cross-Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 10, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Query conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 12, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cQuery.Add(conv) ) ReturnFalse; //--- Key conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 13, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cKey.Add(conv) ) ReturnFalse; //--- Value conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 14, OpenCL, iContWindow, iContWindow, iContHeads * iWindowKey, iContUnits, 1, optimization, iBatch) || !cValue.Add(conv) ) ReturnFalse; //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Attention Out conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 16, OpenCL, iContHeads * iWindowKey, iContHeads * iWindowKey, iContWindow, iContUnits, 1, optimization, iBatch) || !cAttentionOut.Add(conv) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, conv.Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 18, OpenCL, iContWindow, iContWindow, 4 * iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 19, OpenCL, 4 * iContWindow, 4 * iContWindow, iContWindow, iContUnits, 1, optimization, iBatch) || !cFeedForward.Add(conv) ) ReturnFalse; //--- if(!SetGradient(conv.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::feedForward(CNeuronBaseOCL * Primitives, CNeuronBaseOCL * Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Primitives) || !k.FeedForward(Primitives) || !v.FeedForward(Primitives) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[0]) ) ReturnFalse; v = cResidual[0]; if(!v || !SumAndNormilize(Primitives.getOutput(), neuron.getOutput(), v.getOutput(), iPrimWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(Context) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.FeedForward(cMHAttentionOut[1]) ) ReturnFalse; v = cResidual[1]; if(!v || !SumAndNormilize(Context.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v) ReturnFalse; if(!q.FeedForward(neuron) || !k.FeedForward(neuron) || !v.FeedForward(neuron) ) ReturnFalse; if(!AttentionOut(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey)) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.FeedForward(cMHAttentionOut[2]) ) ReturnFalse; v = cResidual[2]; if(!v || !SumAndNormilize(q.getOutput(), neuron.getOutput(), v.getOutput(), iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; neuron = v; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.FeedForward(neuron) || !k.FeedForward(q) || !SumAndNormilize(neuron.getOutput(), k.getOutput(), Output, iContWindow, true, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::calcInputGradients(CNeuronBaseOCL * Primitives, CNeuronBaseOCL * Context) { if(!Primitives || !Context) ReturnFalse; //--- CNeuronBaseOCL *neuron = cResidual[2], *q = cFeedForward[1], *k = cAttentionOut[2], *v = NULL, *residual = GetPointer(this); //--- Feed Forward if(!neuron || !q || !k || !q.CalcHiddenGradients(cFeedForward[1]) || !neuron.CalcHiddenGradients(q, NULL) || !SumAndNormilize(neuron.getGradient(), residual.getGradient(), k.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = k; //--- Context Self-Attention neuron = cMHAttentionOut[2]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[2], cMHAttentionOut[2], iContUnits, iContHeads, iContUnits, iContHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[1]; if(!neuron || !neuron.CalcHiddenGradients(q, NULL) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !SumAndNormilize(residual.getGradient(), neuron.getGradient(), q.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = q; if(!neuron.CalcHiddenGradients(k, NULL) || !SumAndNormilize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(v, NULL) || !SumAndNormilize(residual.getGradient(), neuron.getGradient(), residual.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- Cross-Attention neuron = cMHAttentionOut[1]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[1], cMHAttentionOut[1], iContUnits, iContHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; neuron = cResidual[0]; if(!Context.CalcHiddenGradients(q, NULL) || !DeActivation(Context.getOutput(), residual.getGradient(), residual.getGradient(), Context.Activation()) || !SumAndNormilize(Context.getGradient(), residual.getGradient(), Context.getGradient(), iContWindow, false, 0, 0, 0, 1) ) ReturnFalse; residual = cAttentionOut[0]; if(!neuron.CalcHiddenGradients(k, NULL)) ReturnFalse; CBufferFloat *temp = neuron.getGradient(); if(!neuron.SetGradient(residual.getGradient(), false)) ReturnFalse; if(!neuron.CalcHiddenGradients(v, NULL) || !SumAndNormilize(temp, neuron.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; //--- Primitives Self-Attention neuron = cMHAttentionOut[0]; if(!neuron || !neuron.CalcHiddenGradients(residual, NULL) ) ReturnFalse; q = cQuery[0]; k = cKey[0]; v = cValue[0]; if(!q || !k || !v || !AttentionInsideGradients(q, k, v, cScores[0], cMHAttentionOut[0], iPrimUnits, iPrimHeads, iPrimUnits, iPrimHeads, iWindowKey) ) ReturnFalse; if(!Primitives.CalcHiddenGradients(q, NULL) || !DeActivation(Primitives.getOutput(), residual.getGradient(), residual.getGradient(), Primitives.Activation()) || !SumAndNormilize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(k, NULL) || !SumAndNormilize(Primitives.getGradient(), residual.getGradient(), residual.getGradient(), iPrimWindow, false, 0, 0, 0, 1) || !Primitives.CalcHiddenGradients(v, NULL) || !SumAndNormilize(Primitives.getGradient(), residual.getGradient(), Primitives.getGradient(), iPrimWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::updateInputWeights(CNeuronBaseOCL * Primitives, CNeuronBaseOCL * Context) { CNeuronBaseOCL *neuron = NULL, *q = cQuery[0], *k = cKey[0], *v = cValue[0]; //--- Primitives Self-Attention if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Primitives) || !k.UpdateInputWeights(Primitives) || !v.UpdateInputWeights(Primitives) ) ReturnFalse; neuron = cAttentionOut[0]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[0]) ) ReturnFalse; neuron = cResidual[0]; //--- Cross-Attention q = cQuery[1]; k = cKey[1]; v = cValue[1]; if(!q || !k || !v) ReturnFalse; if(!q.UpdateInputWeights(Context) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; neuron = cAttentionOut[1]; if(!neuron || !neuron.UpdateInputWeights(cMHAttentionOut[1]) ) ReturnFalse; neuron = cResidual[1]; //--- Context Self-Attention q = cQuery[2]; k = cKey[2]; v = cValue[2]; if(!q || !k || !v || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(neuron) || !v.UpdateInputWeights(neuron) ) ReturnFalse; q = cAttentionOut[1]; if(!q || !q.UpdateInputWeights(cMHAttentionOut[2]) ) ReturnFalse; neuron = cResidual[2]; //--- Feed Forward q = cFeedForward[0]; k = cFeedForward[1]; if(!q || !k || !q.UpdateInputWeights(neuron) || !k.UpdateInputWeights(q) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::CreateBuffers(void) { //--- Primitives Self-Attention int s = int(iPrimUnits * iPrimUnits * iPrimHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Cross-Attention s = int(iContUnits * iPrimUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- Content Self-Attention s = int(iContUnits * iContUnits * iContHeads); s = OpenCL.AddBuffer(sizeof(float) * s, CL_MEM_READ_WRITE); if(s < 0 || !cScores.Add(s)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOCM::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { for(int i = 0; i < cScores.Total(); i++) OpenCL.BufferFree(cScores[i]); } cScores.Clear(); CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); CreateBuffers(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save objects if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cAttentionOut.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, (int)iPrimWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iPrimHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iContHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOCM::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load objects if(!cQuery.Load(file_handle)) ReturnFalse; if(!cKey.Load(file_handle)) ReturnFalse; if(!cValue.Load(file_handle)) ReturnFalse; if(!cAttentionOut.Load(file_handle)) ReturnFalse; if(!cFeedForward.Load(file_handle)) ReturnFalse; //--- Load constants iPrimWindow = (uint)FileReadInteger(file_handle); iPrimUnits = (uint)FileReadInteger(file_handle); iPrimHeads = (uint)FileReadInteger(file_handle); iContWindow = (uint)FileReadInteger(file_handle); iContUnits = (uint)FileReadInteger(file_handle); iContHeads = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); //--- CNeuronBaseOCL *neuron = NULL; cMHAttentionOut.Clear(); cResidual.Clear(); //--- Primitives Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, iPrimHeads * iWindowKey * iPrimUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, iPrimWindow * iPrimUnits, optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Cross-Attention //--- Multi-Heads Cross-Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 9, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 11, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- Context Self-Attention //--- Multi-Heads Attention Out neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 15, OpenCL, iContHeads * iWindowKey * iContUnits, optimization, iBatch) || !cMHAttentionOut.Add(neuron) ) ReturnFalse; //--- Residual neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 17, OpenCL, Neurons(), optimization, iBatch) || !cResidual.Add(neuron) ) ReturnFalse; //--- neuron = cFeedForward[1]; if(!SetGradient(neuron.getGradient())) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRefMask : public CNeuronBaseOCL { protected: CNeuronGEGWA cGEGWA; CLayer cContentEncoder; CLayer cBackGround; CNeuronLPC cLPC; CLayer cDecoder; CNeuronOCM cOCM; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronRefMask(void) {}; ~CNeuronRefMask(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint content_size, uint content_units, uint primitive_units, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRefMask; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint content_size, uint content_units, uint primitive_units, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * content_units, optimization_type, batch)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.Init(0, 0, OpenCL, window, window_key, heads, units_count, window, heads, (content_units + 3), 2, layers, optimization, iBatch)) ReturnFalse; cGEGWA.AddNeckGradient(true); //--- Content Encoder cContentEncoder.Clear(); cContentEncoder.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * content_units, 1, OpenCL, content_size, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 2, OpenCL, window * content_units, optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 3, OpenCL, window * (content_units + 3), optimization, iBatch) || !cContentEncoder.Add(neuron) ) ReturnFalse; //--- Background cBackGround.Clear(); cBackGround.SetOpenCL(OpenCL); neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * 3, 4, OpenCL, content_size, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, 5, OpenCL, window * 3, optimization, iBatch) || !cBackGround.Add(neuron) ) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.Init(0, 6, OpenCL, window, window_key, heads, heads, primitive_units, content_units, 2, 1, optimization, iBatch)) ReturnFalse; //--- Decoder cDecoder.Clear(); cDecoder.SetOpenCL(OpenCL); CNeuronOCM *ocm = new CNeuronOCM(); if(!ocm || !ocm.Init(0, 7, OpenCL, window, window_key, units_count, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; for(uint i = 0; i < layers; i++) { neuron = cGEGWA.GetInsideLayer(i); ocm = new CNeuronOCM(); if(!ocm || !neuron || !ocm.Init(0, i + 8, OpenCL, window, window_key, neuron.Neurons() / window, heads, window, primitive_units, heads, optimization, iBatch) || !cDecoder.Add(ocm) ) ReturnFalse; } //--- Object Cluster Module if(!cOCM.Init(0, layers + 8, OpenCL, window, window_key, primitive_units, heads, window, content_units, heads, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput(), true) || !SetGradient(cOCM.getGradient(), true) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!SecondInput) ReturnFalse; //--- Context Encoder CNeuronBaseOCL *context = cContentEncoder[0]; if(context.getOutput() != SecondInput) { if(!context.SetOutput(SecondInput, true)) ReturnFalse; } int content_total = cContentEncoder.Total(); for(int i = 1; i < content_total - 1; i++) { context = cContentEncoder[i]; if(!context || !context.FeedForward(cContentEncoder[i - 1]) ) ReturnFalse; } //--- Background Encoder CNeuronBaseOCL *background = NULL; if(bTrain) { for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.FeedForward(cBackGround[i - 1]) ) ReturnFalse; } } else { background = cBackGround[cBackGround.Total() - 1]; if(!background) ReturnFalse; } CNeuronBaseOCL *neuron = cContentEncoder[content_total - 1]; if(!neuron || !Concat(context.getOutput(), background.getOutput(), neuron.getOutput(), context.Neurons(), background.Neurons(), 1)) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention if(!cGEGWA.FeedForward(NeuronOCL, neuron.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.FeedForward(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.feedForward(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.feedForward(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.feedForward(decoder, context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *neuron = cContentEncoder[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) { if(!neuron.SetGradient(SecondGradient)) ReturnFalse; neuron.SetActivationFunction(SecondActivation); } //--- Object Cluster Module CNeuronBaseOCL *context = cContentEncoder[cContentEncoder.Total() - 2]; if(!cOCM.calcInputGradients(cDecoder[cDecoder.Total() - 1], context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = NULL; for(int i = cDecoder.Total() - 1; i > 0; i--) { decoder = cDecoder[i]; if(!decoder.calcInputGradients(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } decoder = cDecoder[0]; if(!decoder.calcInputGradients(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; //--- Linguistic Primitive Construction CBufferFloat *context_grad = context.getGradient(); if(!context.SetGradient(PrevOutput, false)) ReturnFalse; if(!cLPC.FeedForward(context) || !SumAndNormilize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) ) ReturnFalse; //--- Geometry-Enhaced Group-Word Attention neuron = cContentEncoder[cContentEncoder.Total() - 1]; if(!neuron || !NeuronOCL.CalcHiddenGradients((CObject*)GetPointer(cGEGWA), neuron.getOutput(), neuron.getGradient(), (ENUM_ACTIVATION)neuron.Activation())) ReturnFalse; if(!DiversityLoss(neuron, cOCM.GetContextWindow(), neuron.Neurons() / cOCM.GetContextWindow(), true)) ReturnFalse; CNeuronBaseOCL *background = cBackGround[cBackGround.Total() - 1]; if(!background || !DeConcat(context.getGradient(), background.getGradient(), neuron.getGradient(), context.Neurons(), background.Neurons(), 1) || !DeActivation(context.getOutput(), context.getGradient(), context.getGradient(), context.Activation()) || !SumAndNormilize(context_grad, context.getGradient(), context_grad, 1, false, 0, 0, 0, 1) || !context.SetGradient(context_grad, false) ) ReturnFalse; //--- Context Encoder for(int i = cContentEncoder.Total() - 3; i >= 0; i--) { context = cContentEncoder[i]; if(!context || !context.CalcHiddenGradients(cContentEncoder[i + 1]) ) ReturnFalse; } //--- Background if(!DeActivation(background.getOutput(), background.getGradient(), background.getGradient(), background.Activation())) ReturnFalse; for(int i = cBackGround.Total() - 2; i > 0; i--) { background = cBackGround[i]; if(!background || !background.CalcHiddenGradients(cBackGround[i + 1]) ) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- Context Encoder CNeuronBaseOCL *context = NULL; for(int i = 1; i < cContentEncoder.Total() - 1; i++) { context = cContentEncoder[i]; if(!context || !context.UpdateInputWeights(cContentEncoder[i - 1], NULL) ) ReturnFalse; } //--- Background CNeuronBaseOCL *background = NULL; for(int i = 1; i < cBackGround.Total(); i++) { background = cBackGround[i]; if(!background || !background.FeedForward(cBackGround[i - 1]) ) ReturnFalse; } //--- Geometry-Enhaced Group-Word Attention background = cContentEncoder[cContentEncoder.Total() - 1]; if(!cGEGWA.UpdateInputWeights(NeuronOCL, background.getOutput())) ReturnFalse; //--- Linguistic Primitive Construction if(!cLPC.UpdateInputWeights(context)) ReturnFalse; //--- Decoder CNeuronOCM *decoder = cDecoder[0]; if(!decoder.updateInputWeights(GetPointer(cGEGWA), GetPointer(cLPC))) ReturnFalse; for(int i = 1; i < cDecoder.Total(); i++) { decoder = cDecoder[i]; if(!decoder.updateInputWeights(cGEGWA.GetInsideLayer(i - 1), cDecoder[i - 1])) ReturnFalse; } //--- Object Cluster Module if(!cOCM.updateInputWeights(decoder, context)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRefMask::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cGEGWA.SetOpenCL(OpenCL); cContentEncoder.SetOpenCL(OpenCL); cBackGround.SetOpenCL(OpenCL); cLPC.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); cOCM.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cGEGWA.Save(file_handle)) ReturnFalse; if(!cContentEncoder.Save(file_handle)) ReturnFalse; if(!cBackGround.Save(file_handle)) ReturnFalse; if(!cLPC.Save(file_handle)) ReturnFalse; if(!cDecoder.Save(file_handle)) ReturnFalse; if(!cOCM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRefMask::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGEGWA.AsObject())) ReturnFalse; if(!cContentEncoder.Load(file_handle)) ReturnFalse; if(!cBackGround.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLPC.AsObject())) ReturnFalse; if(!cDecoder.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOCM.AsObject())) ReturnFalse; //--- if(!SetOutput(cOCM.getOutput()) || !SetGradient(cOCM.getGradient()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRelativeSelfAttention : public CNeuronBaseOCL { protected: uint iWindow; uint iWindowKey; uint iHeads; uint iUnits; int iScore; //--- CNeuronConvOCL cQuery; CNeuronConvOCL cKey; CNeuronConvOCL cValue; CNeuronTransposeOCL cTranspose; CNeuronBaseOCL cDistance; CLayer cBKey; CLayer cBValue; CLayer cGlobalContentBias; CLayer cGlobalPositionalBias; CLayer cMHAttentionPooling; CLayer cScale; CBufferFloat cTemp; //--- virtual bool AttentionOut(void); virtual bool AttentionGradient(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronRelativeSelfAttention(void) : iScore(-1) {}; ~CNeuronRelativeSelfAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRelativeSelfAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetWindow(void) const { return iWindow; } virtual uint GetUnits(void) const { return iUnits; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(GELU); idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(GELU); idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnits, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cMHAttentionPooling.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iUnits, optimization, iBatch) || !cScale.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 2 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(LReLU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 2 * iWindow, 2 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MHRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::AttentionGradient(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, iWindowKey, iHeads}; int kernel = def_k_MHRelativeAttentionInsideGradients; //--- ResetLastError(); setBuffer(kernel, def_k_ratg_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_ratg_q_g, cQuery.getGradientIndex()) setBuffer(kernel, def_k_ratg_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_ratg_k_g, cKey.getGradientIndex()) setBuffer(kernel, def_k_ratg_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_ratg_v_g, cValue.getGradientIndex()) setBuffer(kernel, def_k_ratg_scores, iScore) setBuffer(kernel, def_k_ratg_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bk_g, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_bv_g, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gc_g, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_ratg_gp_g, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getGradientIndex()) setBuffer(kernel, def_k_ratg_gradient, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradientIndex()) setArgument(kernel, def_k_ratg_kunits, int(cKey.Neurons() / (iHeads * iWindowKey))) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(NeuronOCL) || !cValue.FeedForward(NeuronOCL) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; if(!MatMul(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cScale[0]).getOutput(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = cScale.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).CalcHiddenGradients(cScale[i + 1])) ReturnFalse; if(!MatMulGrad(((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[cMHAttentionPooling.Total() - 1]).getGradient(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutput(), ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getGradient(), ((CNeuronBaseOCL*)cScale[0]).getGradient(), 1, iHeads, iWindowKey, iUnits) ) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).CalcHiddenGradients(cMHAttentionPooling[i + 1])) ReturnFalse; if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormilize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnits, iWindow, iUnits, 1) ) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cTranspose.getGradient(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1) || !NeuronOCL.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(NeuronOCL) || !cValue.UpdateInputWeights(NeuronOCL) ) ReturnFalse; for(int i = cBKey.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = cBValue.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = cGlobalContentBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; for(int i = cMHAttentionPooling.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; for(int i = cScale.Total() - 1; i > 0; i--) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeSelfAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau) || !cKey.WeightsUpdate(Source.cKey.AsObject(), tau) || !cValue.WeightsUpdate(Source.cValue.AsObject(), tau) ) ReturnFalse; if(!cBKey.WeightsUpdate(Source.cBKey.AsObject(), tau)) ReturnFalse; if(!cBValue.WeightsUpdate(Source.cBValue.AsObject(), tau)) ReturnFalse; if(!cGlobalContentBias.WeightsUpdate(Source.cGlobalContentBias.AsObject(), tau)) ReturnFalse; if(!cGlobalPositionalBias.WeightsUpdate(Source.cGlobalPositionalBias.AsObject(), tau)) ReturnFalse; if(!cMHAttentionPooling.WeightsUpdate(Source.cMHAttentionPooling.AsObject(), tau)) ReturnFalse; if(!cScale.WeightsUpdate(Source.cScale.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRelativeSelfAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnits * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(iUnits * iUnits, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cBKey.Save(file_handle)) ReturnFalse; if(!cBValue.Save(file_handle)) ReturnFalse; if(!cGlobalContentBias.Save(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Save(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeSelfAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iWindowKey = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; SetOpenCL(OpenCL); //--- if(!cBKey.Load(file_handle)) ReturnFalse; if(!cBValue.Load(file_handle)) ReturnFalse; if(!cGlobalContentBias.Load(file_handle)) ReturnFalse; if(!cGlobalPositionalBias.Load(file_handle)) ReturnFalse; if(!cMHAttentionPooling.Load(file_handle)) ReturnFalse; if(!cScale.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cScale[cScale.Total() - 1]; if(!SetGradient(neuron.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRMAT : public CNeuronBaseOCL { protected: CLayer cLayers; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronRMAT(void) {}; ~CNeuronRMAT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRMAT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override; //--- virtual uint GetWindow(void) const { CNeuronRelativeSelfAttention* neuron = cLayers[0]; return (!neuron ? 0 : neuron.GetWindow()); } virtual uint GetUnits(void) const { CNeuronRelativeSelfAttention* neuron = cLayers[0]; return (!neuron ? 0 : neuron.GetUnits()); } virtual void TrainMode(bool flag) override { bTrain = flag; CNeuronBaseOCL *neuron = NULL; for(int i = 0; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!!neuron) neuron.TrainMode(bTrain); } } //--- virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true) { CNeuronBaseOCL *last = cLayers[-1]; if(!last || !last.SetGradient(buffer, delete_prev) || !CNeuronBaseOCL::SetGradient(last.getGradient(), delete_prev)) ReturnFalse; return true; } virtual bool SetOutput(CBufferFloat *buffer, bool delete_prev = true) { CNeuronBaseOCL *last = cLayers[-1]; if(!last || !last.SetOutput(buffer, delete_prev) || !CNeuronBaseOCL::SetOutput(last.getOutput(), delete_prev)) ReturnFalse; return true; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CResidualConv(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, window, units_count, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *neuron = cLayers[0]; if(!neuron || !neuron.FeedForward(NeuronOCL)) ReturnFalse; for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron || !neuron.FeedForward(cLayers[i - 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(neuron.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1])) ReturnFalse; } neuron = cLayers[0]; if(!neuron.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRMAT *Source = source; if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle) || !cLayers.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRMAT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle) || !cLayers.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cLayers[-1]; SetOutput(neuron.getOutput(), true); SetGradient(neuron.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRMAT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cLayers.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHAttentionPooling : public CNeuronBaseOCL { protected: uint iWindow; uint iHeads; uint iUnits; CLayer cNeurons; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMHAttentionPooling(void) {}; ~CNeuronMHAttentionPooling(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMHAttentionPooling; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindow = window; iUnits = units_count; iHeads = heads; //--- cNeurons.Clear(); cNeurons.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow * iHeads, iWindow * iHeads, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iHeads, iUnits, 1, optimization, iBatch) || !cNeurons.Add(conv) ) ReturnFalse; idx++; conv.SetActivationFunction(None); CNeuronSoftMaxOCL *softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, idx, OpenCL, iHeads * iUnits, optimization, iBatch) || !cNeurons.Add(softmax) ) ReturnFalse; softmax.SetHeads(iUnits); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.FeedForward(prev) ) ReturnFalse; prev = current; } //--- if(!MatMul(current.getOutput(), NeuronOCL.getOutput(), Output, 1, iHeads, iWindow, iUnits)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *current = cNeurons[cNeurons.Total() - 1]; if(!MatMulGrad(current.getOutput(), current.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, 1, iHeads, iWindow, iUnits) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation()) ) ReturnFalse; for(int i = cNeurons.Total() - 2; i > 0; i--) { current = cNeurons[i]; if(!current.CalcHiddenGradients(cNeurons[i + 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *current = NULL; CObject *prev = NeuronOCL; for(int i = 0; i < cNeurons.Total(); i++) { current = cNeurons[i]; if(!current || !current.UpdateInputWeights(prev) ) ReturnFalse; prev = current;; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHAttentionPooling* Source = source; //--- if(!cNeurons.WeightsUpdate(Source.cNeurons.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cNeurons.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHAttentionPooling::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iHeads = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); cNeurons.SetOpenCL(OpenCL); //--- if(!cNeurons.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHAttentionPooling::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cNeurons.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMotifs : public CNeuronBaseOCL { protected: CNeuronConvOCL cMotifs; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMotifs(void) {}; ~CNeuronMotifs(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMotifs; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMotifs::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL::SetActivationFunction(value); cMotifs.SetActivationFunction(activation); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { uint inputs = (units_count * step + (window - step)) * dimension; uint motifs = units_count * dimension; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, inputs + motifs, optimization_type, batch)) ReturnFalse; if(!cMotifs.Init(0, 0, OpenCL, dimension * window, dimension * step, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.Activation() != activation) SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); if(!cMotifs.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cMotifs.getOutput(), Output, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cMotifs.getGradient(), Gradient, NeuronOCL.Neurons(), cMotifs.Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cMotifs.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cMotifs.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifs::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMotifs.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMotifs::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cMotifs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleAttention : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; //--- CNeuronBaseOCL cWideInputs; CNeuronRelativeSelfAttention cAttentions[4]; CNeuronBaseOCL cConcatAttentions; CNeuronMHAttentionPooling cPooling; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultiScaleAttention(void) {}; ~CNeuronMultiScaleAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultiScaleAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iUnits = units_count; //--- uint units1 = (iUnits + 1) / 2; uint units2 = (iUnits + 2) / 3; uint units3 = (iUnits + 3) / 4; uint wide = MathMax(MathMax(iUnits, units1 * 2), MathMax(units2 * 3, units3 * 4)); //--- int idx = 0; if(!cWideInputs.Init(0, idx, OpenCL, wide * iWindow, optimization, iBatch)) ReturnFalse; CBufferFloat *temp = cWideInputs.getOutput(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- idx++; if(!cAttentions[0].Init(0, idx, OpenCL, iWindow, window_key, iUnits, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[1].Init(0, idx, OpenCL, 2 * iWindow, window_key, units1, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[2].Init(0, idx, OpenCL, 3 * iWindow, window_key, units2, heads, optimization, iBatch)) ReturnFalse; idx++; if(!cAttentions[3].Init(0, idx, OpenCL, 4 * iWindow, window_key, units3, heads, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cConcatAttentions.Init(0, idx, OpenCL, 4 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; //--- idx++; if(!cPooling.Init(0, idx, OpenCL, iWindow, iUnits, 4, optimization, iBatch)) ReturnFalse; //--- SetActivationFunction(None); if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Attention if(!cAttentions[0].FeedForward(NeuronOCL)) ReturnFalse; //--- Copy inputs if(!Concat(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cWideInputs.getOutput(), iWindow, 0, iUnits)) ReturnFalse; if(cWideInputs.Activation() != NeuronOCL.Activation()) cWideInputs.SetActivationFunction((ENUM_ACTIVATION)NeuronOCL.Activation()); //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].FeedForward(cWideInputs.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!Concat(cAttentions[0].getOutput(), cAttentions[1].getOutput(), cAttentions[2].getOutput(), cAttentions[3].getOutput(), cConcatAttentions.getOutput(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention pooling if(!cPooling.FeedForward(cConcatAttentions.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Attention pooling if(!cConcatAttentions.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- Concatenate Multi-Scale Attentions if(!DeConcat(cAttentions[0].getGradient(), cAttentions[1].getGradient(), cAttentions[2].getGradient(), cAttentions[3].getGradient(), cConcatAttentions.getGradient(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- Attention if(!NeuronOCL.CalcHiddenGradients(cAttentions[0].AsObject())) ReturnFalse; //--- Multi scale attentions for(int i = 1; i < 4; i++) { if(!cAttentions[i].FeedForward(cWideInputs.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cWideInputs.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) ) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- Attention if(!cAttentions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- Multi scale attentions for(int i = 1; i < 4; i++) if(!cAttentions[i].UpdateInputWeights(cWideInputs.AsObject())) ReturnFalse; //--- Attention pooling if(!cPooling.UpdateInputWeights(cConcatAttentions.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cWideInputs.SetOpenCL(OpenCL); for(int i = 0; i < 4; i++) cAttentions[i].SetOpenCL(OpenCL); cConcatAttentions.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; //--- if(!cWideInputs.Save(file_handle)) ReturnFalse; for(int i = 0; i < 4; i++) if(!cAttentions[i].Save(file_handle)) ReturnFalse; if(!cConcatAttentions.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- iWindow = (uint)FileReadInteger(file_handle); iUnits = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cWideInputs.AsObject())) ReturnFalse; for(int i = 0; i < 4; i++) if(!LoadInsideLayer(file_handle, cAttentions[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatAttentions.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput()) || !SetGradient(cPooling.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMolformer : public CNeuronRMAT { public: CNeuronMolformer(void) {}; ~CNeuronMolformer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint motif_window, uint motif_step, ENUM_OPTIMIZATION optimization_type, uint batch); //Molformer //--- virtual int Type(void) override const { return defNeuronMolformer; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMolformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint motif_window, uint motif_step, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- int idx = 0; CNeuronMotifs *motif = new CNeuronMotifs(); uint motif_units = units_count - MathMax(motif_window - motif_step, 0); motif_units = (motif_units + motif_step - 1) / motif_step; if(!motif || !motif.Init(0, idx, OpenCL, window, motif_window, motif_step, motif_units, optimization, iBatch) || !cLayers.Add(motif) ) ReturnFalse; //--- idx++; CNeuronMultiScaleAttention *msat = NULL; CResidualConv *ff = NULL; uint units_total = units_count + motif_units; for(uint i = 0; i < layers; i++) { //--- Attention msat = new CNeuronMultiScaleAttention(); if(!msat || !msat.Init(0, idx, OpenCL, window, window_key, units_total, heads, optimization, iBatch) || !cLayers.Add(msat) ) ReturnFalse; idx++; //--- FeedForward ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units_total, optimization, iBatch) || !cLayers.Add(ff) ) ReturnFalse; idx++; } //--- Out CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, units_total, window, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, units_total, units_total, units_count, window, 1, optimization, iBatch) || !cLayers.Add(conv) ) ReturnFalse; idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cLayers.Add(transp) ) ReturnFalse; //--- if(!SetOutput(transp.getOutput()) || !SetGradient(transp.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRelativeCrossAttention : public CNeuronRelativeSelfAttention { protected: uint iUnitsKV; //--- CLayer cKVProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronRelativeCrossAttention(void) {}; ~CNeuronRelativeCrossAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRelativeCrossAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetUnitsKV(void) const { return iUnitsKV; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = window_key; iUnits = units_count; iUnitsKV = units_kv; iHeads = heads; //--- int idx = 0; if(!cQuery.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnits, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cKey.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cValue.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch)) ReturnFalse; idx++; if(!cTranspose.Init(0, idx, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; idx++; if(!cDistance.Init(0, idx, OpenCL, iUnits * iUnitsKV, optimization, iBatch)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBKey.Add(conv)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iUnits, iUnits, iWindow, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; idx++; conv.SetActivationFunction(TANH); conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindow, iWindow, iWindowKey * iHeads, iUnitsKV, 1, optimization, iBatch) || !cBValue.Add(conv)) ReturnFalse; //--- idx++; CNeuronBaseOCL *neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; idx++; CBufferFloat *buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalContentBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(iWindowKey * iHeads * iUnits, idx, OpenCL, 1, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; idx++; buffer = neuron.getOutput(); buffer.BufferInit(1, 1); if(!buffer.BufferWrite()) ReturnFalse; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cGlobalPositionalBias.Add(neuron)) ReturnFalse; //--- idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, iWindowKey * iHeads * iUnits, optimization, iBatch) || !cMHAttentionPooling.Add(neuron) ) ReturnFalse; idx++; CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling(); if(!pooling || !pooling.Init(0, idx, OpenCL, iWindowKey, iUnits, iHeads, optimization, iBatch) || !cMHAttentionPooling.Add(pooling) ) ReturnFalse; //--- idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, iWindowKey, iWindowKey, 4 * iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(GELU); idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, 1, optimization, iBatch) || !cScale.Add(conv) ) ReturnFalse; conv.SetActivationFunction(None); //--- if(!SetGradient(conv.getGradient(), true)) ReturnFalse; //--- Key-Value inputs projection cKVProjection.Clear(); cKVProjection.SetOpenCL(OpenCL); idx++; neuron = new CNeuronBaseOCL; if(!neuron || !neuron.Init(0, idx, OpenCL, window_kv * iUnitsKV, optimization, iBatch) || !cKVProjection.Add(neuron) ) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, window_kv, window_kv, iWindow, iUnitsKV, 1, optimization, iBatch) || !cKVProjection.Add(conv) ) ReturnFalse; //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRelativeCrossAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) OpenCL.BufferFree(iScore); cTemp.BufferFree(); //--- CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cValue.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cBKey.SetOpenCL(OpenCL); cBValue.SetOpenCL(OpenCL); cGlobalContentBias.SetOpenCL(OpenCL); cGlobalPositionalBias.SetOpenCL(OpenCL); cMHAttentionPooling.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cKVProjection.SetOpenCL(OpenCL); //--- iScore = OpenCL.AddBuffer(sizeof(float) * (iUnits * iUnitsKV * iHeads), CL_MEM_READ_WRITE); cTemp.BufferInit(MathMax(iUnits, iWindow) * iUnitsKV, 0); cTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron || !SecondInput) ReturnFalse; if(neuron.getOutput() != SecondInput) if(!neuron.SetOutput(SecondInput, true)) ReturnFalse; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.FeedForward(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.FeedForward(NeuronOCL) || !cKey.FeedForward(neuron) || !cValue.FeedForward(neuron) ) ReturnFalse; //--- if(!cTranspose.FeedForward(NeuronOCL) || !MatMul(neuron.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!((CNeuronBaseOCL*)cBKey[0]).FeedForward(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).FeedForward(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).FeedForward(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).FeedForward(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).FeedForward(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).FeedForward(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).FeedForward(cMHAttentionPooling[i - 1])) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cScale[0]).FeedForward(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).FeedForward(cScale[i - 1])) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), ((CNeuronBaseOCL*)cScale[cScale.Total() - 1]).getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; CNeuronBaseOCL *neuron = cKVProjection[0]; if(!neuron) ReturnFalse; if(neuron.getGradient() != SecondGradient) if(!neuron.SetGradient(SecondGradient)) ReturnFalse; if(neuron.Activation() != SecondActivation) neuron.SetActivationFunction(SecondActivation); //--- if(!DiversityLoss(AsObject(), iUnits, iWindow, true)) ReturnFalse; //--- CNeuronBaseOCL* next = cScale[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cScale.Total() - 2; i >= 0; i--) { curr = cScale[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } for(int i = cMHAttentionPooling.Total() - 1; i >= 0; i--) { curr = cMHAttentionPooling[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!AttentionGradient()) ReturnFalse; for(int i = cGlobalContentBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).CalcHiddenGradients(cGlobalContentBias[i + 1])) ReturnFalse; for(int i = cGlobalPositionalBias.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).CalcHiddenGradients(cGlobalPositionalBias[i + 1])) ReturnFalse; for(int i = cBKey.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBKey[i]).CalcHiddenGradients(cBKey[i + 1])) ReturnFalse; for(int i = cBValue.Total() - 2; i >= 0; i--) if(!((CNeuronBaseOCL*)cBValue[i]).CalcHiddenGradients(cBValue[i + 1])) ReturnFalse; if(!cDistance.CalcHiddenGradients(cBKey[0])) ReturnFalse; CBufferFloat *temp = cDistance.getGradient(); if(!cDistance.SetGradient(GetPointer(cTemp), false) || !cDistance.CalcHiddenGradients(cBValue[0]) || !SumAndNormilize(temp, GetPointer(cTemp), temp, iUnits, false, 0, 0, 0, 1) || !cDistance.SetGradient(temp, false) ) ReturnFalse; neuron = cKVProjection[cKVProjection.Total() - 1]; if(!neuron || !MatMulGrad(neuron.getOutput(), neuron.getGradient(), cTranspose.getOutput(), cTranspose.getGradient(), temp, iUnitsKV, iWindow, iUnits, 1) ) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), Gradient, cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cTranspose.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1) || !DiversityLoss(NeuronOCL, iUnits, iWindow, true) ) ReturnFalse; temp = neuron.getGradient(); if(!neuron.SetGradient(GetPointer(cTemp), false) || !neuron.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormilize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.CalcHiddenGradients(cValue.AsObject()) || !SumAndNormilize(temp, GetPointer(cTemp), temp, iWindow, false, 0, 0, 0, 1) || !neuron.SetGradient(temp, false) ) ReturnFalse; for(int i = cKVProjection.Total() - 2; i >= 0; i--) { neuron = cKVProjection[i]; if(!neuron || !neuron.CalcHiddenGradients(cKVProjection[i + 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { CNeuronBaseOCL *neuron = NULL; for(int i = 1; i < cKVProjection.Total(); i++) { neuron = cKVProjection[i]; if(!neuron || !neuron.UpdateInputWeights(cKVProjection[i - 1]) ) ReturnFalse; } if(!cQuery.UpdateInputWeights(NeuronOCL) || !cKey.UpdateInputWeights(neuron) || !cValue.UpdateInputWeights(neuron) ) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cBKey[0]).UpdateInputWeights(cDistance.AsObject()) || !((CNeuronBaseOCL*)cBValue[0]).UpdateInputWeights(cDistance.AsObject()) ) ReturnFalse; for(int i = 1; i < cBKey.Total(); i++) if(!((CNeuronBaseOCL*)cBKey[i]).UpdateInputWeights(cBKey[i - 1])) ReturnFalse; for(int i = 1; i < cBValue.Total(); i++) if(!((CNeuronBaseOCL*)cBValue[i]).UpdateInputWeights(cBValue[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalContentBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalContentBias[i]).UpdateInputWeights(cGlobalContentBias[i - 1])) ReturnFalse; for(int i = 1; i < cGlobalPositionalBias.Total(); i++) if(!((CNeuronBaseOCL*)cGlobalPositionalBias[i]).UpdateInputWeights(cGlobalPositionalBias[i - 1])) ReturnFalse; if(!AttentionOut()) ReturnFalse; for(int i = 1; i < cMHAttentionPooling.Total(); i++) if(!((CNeuronBaseOCL*)cMHAttentionPooling[i]).UpdateInputWeights(cMHAttentionPooling[i - 1])) ReturnFalse; if(!((CNeuronBaseOCL*)cScale[0]).UpdateInputWeights(cMHAttentionPooling[cMHAttentionPooling.Total() - 1])) ReturnFalse; for(int i = 1; i < cScale.Total(); i++) if(!((CNeuronBaseOCL*)cScale[i]).UpdateInputWeights(cScale[i - 1])) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronRelativeSelfAttention::WeightsUpdate(source, tau)) ReturnFalse; CNeuronRelativeCrossAttention *Source = source; if(!cKVProjection.WeightsUpdate(GetPointer(Source.cKVProjection), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Save(const int file_handle) { if(!CNeuronRelativeSelfAttention::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnitsKV) < INT_VALUE) ReturnFalse; if(!cKVProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRelativeCrossAttention::Load(const int file_handle) { if(!CNeuronRelativeSelfAttention::Load(file_handle)) ReturnFalse; iUnitsKV = (uint)FileReadInteger(file_handle); SetOpenCL(OpenCL); if(!cKVProjection.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMotifEncoder : public CNeuronRMAT { public: CNeuronMotifEncoder(void) {}; ~CNeuronMotifEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronMotifEncoder; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMotifEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units_count < 3) ReturnFalse; //--- cLayers.Clear(); //--- int bars_to_paattern = (units_count > 10 ? 3 : 2); CNeuronConvOCL *conv = new CNeuronConvOCL(); int idx = 0; int units = (int)units_count - bars_to_paattern + 1; if(!conv || !conv.Init(0, idx, open_cl, bars_to_paattern * window, window, window, units, 1, optimization_type, batch) || !cLayers.Add(conv) ) ReturnFalse; conv.SetActivationFunction(SIGMOID); idx++; units = units - bars_to_paattern + 1; CNeuronMotifs *motifs = new CNeuronMotifs(); if(!motifs || !motifs.Init(0, idx, open_cl, window, bars_to_paattern, 1, units, optimization_type, batch) || !cLayers.Add(motifs) ) ReturnFalse; motifs.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, motifs.Neurons(), optimization_type, batch)) ReturnFalse; cLayers.SetOpenCL(OpenCL); //--- CNeuronRelativeSelfAttention *attention = NULL; CResidualConv *ff = NULL; units = int(motifs.Neurons() / window); for(uint i = 0; i < layers; i++) { idx++; attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, idx, OpenCL, window, window_key, units, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, units, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPropertyAwareAttention : public CNeuronRMAT { protected: CBufferFloat cTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPropertyAwareAttention(void) {}; ~CNeuronPropertyAwareAttention(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronPropertyAwareAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * properties, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronBaseOCL *neuron = NULL; CNeuronRelativeSelfAttention *self_attention = NULL; CNeuronRelativeCrossAttention *cross_attention = NULL; CResidualConv *ff = NULL; //--- int idx = 0; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(window * properties, idx, OpenCL, 1, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; CBufferFloat *temp = neuron.getOutput(); if(!temp.Fill(1)) ReturnFalse; idx++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, idx, OpenCL, window * properties, optimization, iBatch) || !cLayers.Add(neuron)) ReturnFalse; //--- for(uint i = 0; i < layers; i++) { idx++; self_attention = new CNeuronRelativeSelfAttention(); if(!self_attention || !self_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, optimization, iBatch) || !cLayers.Add(self_attention) ) { DeleteObj(self_attention); ReturnFalse; } idx++; cross_attention = new CNeuronRelativeCrossAttention(); if(!cross_attention || !cross_attention.Init(0, idx, OpenCL, window, window_key, properties, heads, window, units_count, optimization, iBatch) || !cLayers.Add(cross_attention) ) { DeleteObj(cross_attention); ReturnFalse; } idx++; ff = new CResidualConv(); if(!ff || !ff.Init(0, idx, OpenCL, window, window, properties, optimization, iBatch) || !cLayers.Add(ff) ) { DeleteObj(ff); ReturnFalse; } } //--- if(!SetOutput(ff.getOutput()) || !SetGradient(ff.getGradient())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- if(bTrain) { neuron = cLayers[1]; if(!neuron || !neuron.FeedForward(cLayers[0])) ReturnFalse; } for(int i = 2; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.FeedForward(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(cTemp.GetIndex() < 0 || cTemp.Total() < NeuronOCL.Neurons()) { cTemp.BufferFree(); if(!cTemp.BufferInit(NeuronOCL.Neurons(), 0) || !cTemp.BufferCreate(OpenCL)) ReturnFalse; } if(!NeuronOCL.getGradient() || !NeuronOCL.getGradient().Fill(0)) ReturnFalse;; //--- CNeuronBaseOCL *neuron = NULL; for(int i = cLayers.Total() - 2; i > 0; i--) { neuron = cLayers[i]; if(!neuron.CalcHiddenGradients(cLayers[i + 1], NeuronOCL.getOutput(), GetPointer(cTemp), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(neuron.Type() == defNeuronRelativeCrossAttention) if(!SumAndNormilize(NeuronOCL.getGradient(), GetPointer(cTemp), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPropertyAwareAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = NULL; //--- for(int i = 1; i < cLayers.Total(); i++) { neuron = cLayers[i]; if(!neuron.UpdateInputWeights(cLayers[i - 1], NeuronOCL.getOutput())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAMCT : public CNeuronBaseOCL { protected: CNeuronRMAT cAtomEncoder; CNeuronMotifEncoder cMotifEncoder; CLayer cMotifProjection; CNeuronPropertyAwareAttention cPropertyDecoder; CLayer cPropertyProjection; CNeuronBaseOCL cConcatenate; CNeuronMHAttentionPooling cPooling; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAMCT(void) {}; ~CNeuronAMCT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronAMCT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint properties, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- int idx = 0; if(!cAtomEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; idx++; if(!cMotifEncoder.Init(0, idx, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; //--- cMotifProjection.Clear(); cMotifProjection.SetOpenCL(OpenCL); int motifs = int(cMotifEncoder.Neurons() / window); idx++; CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, motifs, window, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; idx++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, motifs, motifs, units_count, 1, window, optimization, iBatch) || !cMotifProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cMotifProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cPropertyDecoder.Init(0, idx, OpenCL, window, window_key, properties, motifs, heads, layers, optimization, iBatch)) ReturnFalse; //--- cPropertyProjection.Clear(); cPropertyProjection.SetOpenCL(OpenCL); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, properties, window, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; idx++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, idx, OpenCL, properties, properties, units_count, 1, window, optimization, iBatch) || !cPropertyProjection.Add(conv)) ReturnFalse; conv.SetActivationFunction((ENUM_ACTIVATION)cAtomEncoder.Activation()); idx++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, idx, OpenCL, window, units_count, optimization, iBatch) || !cPropertyProjection.Add(transp)) ReturnFalse; transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- idx++; if(!cConcatenate.Init(0, idx, OpenCL, 3 * window * units_count, optimization, iBatch)) ReturnFalse; idx++; if(!cPooling.Init(0, idx, OpenCL, window, units_count, 3, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cPooling.getOutput(), true) || !SetGradient(cPooling.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cAtomEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.FeedForward(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.FeedForward(prev, NULL)) ReturnFalse; prev = current; } //--- Concatenate uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); prev = cMotifProjection[cMotifProjection.Total() - 1]; if(!Concat(cAtomEncoder.getOutput(), prev.getOutput(), current.getOutput(), cConcatenate.getOutput(), window, window, window, units)) ReturnFalse; //--- Out if(!cPooling.FeedForward(cConcatenate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cConcatenate.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- uint window = cAtomEncoder.GetWindow(); uint units = cAtomEncoder.GetUnits(); CNeuronBaseOCL *motifs = cMotifProjection[cMotifProjection.Total() - 1]; CNeuronBaseOCL *prop = cPropertyProjection[cPropertyProjection.Total() - 1]; if(!motifs || !prop || !DeConcat(cAtomEncoder.getGradient(), motifs.getGradient(), prop.getGradient(), cConcatenate.getGradient(), window, window, window, units)) ReturnFalse; //--- if(cAtomEncoder.Activation() != None) if(!DeActivation(cAtomEncoder.getOutput(), cAtomEncoder.getGradient(), cAtomEncoder.getGradient(), cAtomEncoder.Activation())) ReturnFalse; if(motifs.Activation() != None) if(!DeActivation(motifs.getOutput(), motifs.getGradient(), motifs.getGradient(), motifs.Activation())) ReturnFalse; if(prop.Activation() != None) if(!DeActivation(prop.getOutput(), prop.getGradient(), prop.getGradient(), prop.Activation())) ReturnFalse; //--- if(!motifs.calcAlignmentGradient(cAtomEncoder.AsObject(), true)) ReturnFalse; for(int i = cMotifProjection.Total() - 2; i >= 0; i--) { motifs = cMotifProjection[i]; if(!motifs || !motifs.CalcHiddenGradients(cMotifProjection[i + 1])) ReturnFalse; } //--- for(int i = cPropertyProjection.Total() - 2; i >= 0; i--) { prop = cPropertyProjection[i]; if(!prop || !prop.CalcHiddenGradients(cPropertyProjection[i + 1])) ReturnFalse; } //--- if(!cPropertyDecoder.CalcHiddenGradients(cPropertyProjection[0]) || !cMotifEncoder.CalcHiddenGradients(cPropertyDecoder.AsObject())) ReturnFalse; CBufferFloat *temp = cMotifEncoder.getGradient(); if(!cMotifEncoder.SetGradient(cConcatenate.getGradient(), false) || !cMotifEncoder.CalcHiddenGradients(cMotifProjection[0]) || !SumAndNormilize(temp, cMotifEncoder.getGradient(), temp, window, false, 0, 0, 0, 1) || !cMotifEncoder.SetGradient(temp, false) || !NeuronOCL.CalcHiddenGradients(cMotifEncoder.AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cConcatenate.getGradient(), false) || !NeuronOCL.CalcHiddenGradients(cAtomEncoder.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, window, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cAtomEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMotifEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPropertyDecoder.UpdateInputWeights(cMotifEncoder.AsObject())) ReturnFalse; //--- Motifs projection CNeuronBaseOCL *prev = cMotifEncoder.AsObject(); CNeuronBaseOCL *current = NULL; for(int i = 0; i < cMotifProjection.Total(); i++) { current = cMotifProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Property projection prev = cPropertyDecoder.AsObject(); for(int i = 0; i < cPropertyProjection.Total(); i++) { current = cPropertyProjection[i]; if(!current || !current.UpdateInputWeights(prev, NULL)) ReturnFalse; prev = current; } //--- Out if(!cPooling.UpdateInputWeights(cConcatenate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAMCT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cAtomEncoder.SetOpenCL(OpenCL); cMotifEncoder.SetOpenCL(OpenCL); cMotifProjection.SetOpenCL(OpenCL); cPropertyDecoder.SetOpenCL(OpenCL); cPropertyProjection.SetOpenCL(OpenCL); cConcatenate.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAtomEncoder.Save(file_handle)) ReturnFalse; if(!cMotifEncoder.Save(file_handle)) ReturnFalse; if(!cMotifProjection.Save(file_handle)) ReturnFalse; if(!cPropertyDecoder.Save(file_handle)) ReturnFalse; if(!cPropertyProjection.Save(file_handle)) ReturnFalse; if(!cConcatenate.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAMCT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cAtomEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMotifEncoder.AsObject())) ReturnFalse; if(!cMotifProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cPropertyDecoder.AsObject())) ReturnFalse; if(!cPropertyProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronNAFS : public CNeuronBaseOCL { protected: uint iDimension; uint iSmoothing; uint iUnits; //--- CNeuronBaseOCL cFeatureSmoothing; CNeuronTransposeOCL cTranspose; CNeuronBaseOCL cDistance; CNeuronSoftMaxOCL cAdaptation; //--- virtual bool FeatureSmoothing(const CNeuronBaseOCL *neuron, const CNeuronBaseOCL *smoothing); virtual bool FeatureSmoothingGradient(const CNeuronBaseOCL *neuron, const CNeuronBaseOCL *smoothing); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } public: CNeuronNAFS(void) {}; ~CNeuronNAFS(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronNAFS; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothing(const CNeuronBaseOCL * neuron, const CNeuronBaseOCL * smoothing) { if(!OpenCL || !neuron || !neuron.getOutput() || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothing; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getOutputIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getOutputIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::FeatureSmoothingGradient(const CNeuronBaseOCL * neuron, const CNeuronBaseOCL * smoothing) { if(!OpenCL || !neuron || !smoothing) ReturnFalse; uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iDimension}; int kernel = def_k_FeatureSmoothingGradient; ResetLastError(); setBuffer(kernel, def_k_fs_feature, neuron.getGradientIndex()) setBuffer(kernel, def_k_fs_outputs, smoothing.getGradientIndex()) setArgument(kernel, def_k_fs_smoothing, iSmoothing) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint dimension, uint smoothing, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units_count, optimization_type, batch)) ReturnFalse; //--- iDimension = dimension; iSmoothing = smoothing; iUnits = units_count; //--- if(!cFeatureSmoothing.Init(0, 0, OpenCL, (iSmoothing + 1)*iUnits * iDimension, optimization, iBatch)) ReturnFalse; cFeatureSmoothing.SetActivationFunction(None); if(!cTranspose.Init(0, 1, OpenCL, (iSmoothing + 1)*iUnits, iDimension, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); if(!cDistance.Init(0, 2, OpenCL, (iSmoothing + 1)*iUnits, optimization, iBatch)) ReturnFalse; cDistance.SetActivationFunction(None); if(!cAdaptation.Init(0, 3, OpenCL, cDistance.Neurons(), optimization, iBatch)) ReturnFalse; cAdaptation.SetActivationFunction(None); cAdaptation.SetHeads(iUnits); //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!FeatureSmoothing(NeuronOCL, cFeatureSmoothing.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cFeatureSmoothing.AsObject())) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cDistance.getOutput(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cAdaptation.FeedForward(cDistance.AsObject())) ReturnFalse; if(!MatMul(cAdaptation.getOutput(), cFeatureSmoothing.getOutput(), Output, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cAdaptation.getOutput(), cAdaptation.getGradient(), cFeatureSmoothing.getOutput(), cFeatureSmoothing.getPrevOutput(), Gradient, 1, iSmoothing + 1, iDimension, iUnits)) ReturnFalse; if(!cDistance.CalcHiddenGradients(cAdaptation.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getGradient(), cDistance.getGradient(), 1, iDimension, iSmoothing + 1, iUnits)) ReturnFalse; if(!cFeatureSmoothing.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormilize(cFeatureSmoothing.getGradient(), cFeatureSmoothing.getPrevOutput(), cFeatureSmoothing.getGradient(), iDimension, false, 0, 0, 0, 1) ) ReturnFalse; if(!FeatureSmoothingGradient(NeuronOCL, cFeatureSmoothing.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cFeatureSmoothing.getPrevOutput(), NeuronOCL.getGradient(), iDimension, false, 0, 0, 0, 1) || !DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation()) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronNAFS::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cFeatureSmoothing.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cDistance.SetOpenCL(OpenCL); cAdaptation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Save constants if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSmoothing)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; //--- Save objects if(!cFeatureSmoothing.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cDistance.Save(file_handle)) ReturnFalse; if(!cAdaptation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronNAFS::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Load constants if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSmoothing = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); //--- Load objects if(!LoadInsideLayer(file_handle, cFeatureSmoothing.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistance.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAdaptation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBatchNormWithNoise : public CNeuronBatchNormOCL { protected: CBufferFloat cNoise; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); public: CNeuronBatchNormWithNoise(void) {}; ~CNeuronBatchNormWithNoise(void) {}; //--- virtual int Type(void) const { return defNeuronBatchNormWithNoise; }///< Identificator of class.@return Type of class }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBatchNormWithNoise::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!bTrain) return CNeuronBatchNormOCL::feedForward(NeuronOCL); //--- if(!OpenCL || !NeuronOCL) ReturnFalse; //--- PrevLayer = NeuronOCL; if(iBatchSize <= 1) { activation = (ENUM_ACTIVATION)NeuronOCL.Activation(); return true; } //--- double random[]; if(!MathRandomNormal(0, 1, Neurons(), random)) ReturnFalse; if(cNoise.Total() != Neurons() || cNoise.GetOpenCL() != OpenCL) { cNoise.BufferFree(); if(!cNoise.AssignArray(random)) ReturnFalse; if(!cNoise.BufferCreate(OpenCL)) ReturnFalse; } else { if(!cNoise.AssignArray(random)) ReturnFalse; if(!cNoise.BufferWrite()) ReturnFalse; } //--- iBatchCount = MathMin(iBatchCount, iBatchSize); float noise_alpha = float(1.0 - MathRand() / 32767.0 * 0.01); //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; global_work_size[0] = Neurons(); int kernel = def_k_BatchFeedForwardAddNoise; ResetLastError(); setBuffer(kernel, def_k_normwithnoise_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_normwithnoise_noise, cNoise.GetIndex()) setBuffer(kernel, def_k_normwithnoise_options, BatchOptions.GetIndex()) setBuffer(kernel, def_k_normwithnoise_output, Output.GetIndex()) setArgument(kernel, def_k_normwithnoise_activation, int(activation)) setArgument(kernel, def_k_normwithnoise_alpha, noise_alpha) setArgument(kernel, def_k_normwithnoise_batch, iBatchCount) setArgument(kernel, def_k_normwithnoise_optimization, int(optimization)) //--- kernelExecute(kernel, global_work_offset, global_work_size) iBatchCount++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDiffusion : public CNeuronUShapeAttention { protected: CNeuronBatchNormWithNoise cAddNoise; CNeuronBaseOCL cResidual; CNeuronRevINDenormOCL cRevIn; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronDiffusion(void) {}; ~CNeuronDiffusion(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronDiffusion; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag) { bTrain = flag; cAddNoise.TrainMode(bTrain); } ///< Set Training Mode Flag }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint layers, uint inside_block, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; if(!cAddNoise.Init(0, 0, OpenCL, window * units_count, iBatch, optimization)) ReturnFalse; cAddNoise.SetActivationFunction(None); if(!cAttention[0].Init(0, 1, OpenCL, window, window_key, heads, units_count, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[0].Init(0, 2, OpenCL, 2 * window, 2 * window, window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(inside_block > 0) { CNeuronDiffusion *temp = new CNeuronDiffusion(); if(!temp) ReturnFalse; if(!temp.Init(0, 3, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, inside_block - 1, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } else { CNeuronConvOCL *temp = new CNeuronConvOCL(); if(!temp) ReturnFalse; if(!temp.Init(0, 3, OpenCL, window, window, window, (units_count + 1) / 2, optimization, iBatch)) { DeleteObj(temp); ReturnFalse; } cNeck = temp; } if(!cAttention[1].Init(0, 4, OpenCL, window, window_key, heads, (units_count + 1) / 2, layers, optimization, iBatch)) ReturnFalse; if(!cMergeSplit[1].Init(0, 5, OpenCL, window, window, 2 * window, (units_count + 1) / 2, optimization, iBatch)) ReturnFalse; if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)cMergeSplit[1].Activation()); //--- if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject())) ReturnFalse; //--- if(!SetOutput(cRevIn.getOutput(), true)) ReturnFalse; //--- cRevIn.SetActivationFunction(None); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cAddNoise.FeedForward(NeuronOCL)) ReturnFalse; if(!cAttention[0].FeedForward(cAddNoise.AsObject())) ReturnFalse; if(!cMergeSplit[0].FeedForward(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.FeedForward(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].FeedForward(cNeck)) ReturnFalse; if(!cMergeSplit[1].FeedForward(cAttention[1].AsObject())) ReturnFalse; if(!SumAndNormilize(cAddNoise.getOutput(), cMergeSplit[1].getOutput(), cResidual.getOutput(), 1, true, 0, 0, 0, 1)) ReturnFalse; if(!cRevIn.FeedForward(cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- float error = 1; if(!cRevIn.calcOutputGradients(prevLayer.getOutput(), error) || !SumAndNormilize(cRevIn.getGradient(), Gradient, cRevIn.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cResidual.CalcHiddenGradients(cRevIn.AsObject())) ReturnFalse; if(!cAttention[1].CalcHiddenGradients(cMergeSplit[1].AsObject())) ReturnFalse; if(!cNeck.CalcHiddenGradients(cAttention[1].AsObject())) ReturnFalse; if(!cMergeSplit[0].CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!cAttention[0].CalcHiddenGradients(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAddNoise.CalcHiddenGradients(cAttention[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cAddNoise.getGradient(), cResidual.getGradient(), cAddNoise.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cAddNoise.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cAddNoise.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cAttention[0].UpdateInputWeights(cAddNoise.AsObject())) ReturnFalse; if(!cMergeSplit[0].UpdateInputWeights(cAttention[0].AsObject())) ReturnFalse; if(!cNeck.UpdateInputWeights(cMergeSplit[0].AsObject())) ReturnFalse; if(!cAttention[1].UpdateInputWeights(cNeck)) ReturnFalse; if(!cMergeSplit[1].UpdateInputWeights(cAttention[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDiffusion::SetOpenCL(COpenCLMy * obj) { CNeuronUShapeAttention::SetOpenCL(obj); cAddNoise.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cRevIn.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Save(const int file_handle) { if(!CNeuronUShapeAttention::Save(file_handle)) ReturnFalse; if(!cAddNoise.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDiffusion::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!LoadInsideLayer(file_handle, cAttention[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMergeSplit[i].AsObject())) ReturnFalse; } //--- int type = FileReadInteger(file_handle); if(!!cNeck) { if(cNeck.Type() != type) DeleteObj(cNeck); } //--- if(!cNeck) { switch(type) { case defNeuronUShapeAttention: cNeck = new CNeuronUShapeAttention(); if(!cNeck) ReturnFalse; break; case defNeuronDiffusion: cNeck = new CNeuronDiffusion(); if(!cNeck || !((CNeuronDiffusion*)cNeck).Init(0, 0, OpenCL, 1, 1, 1, 1, 1, 0, ADAM, 1)) ReturnFalse; break; case defNeuronConvOCL: cNeck = new CNeuronConvOCL(); if(!cNeck) ReturnFalse; break; default: ReturnFalse; } } cNeck.SetOpenCL(OpenCL); if(!cNeck.Load(file_handle)) ReturnFalse; //--- if(!cAddNoise.Init(0, 0, OpenCL, Neurons(), iBatch, optimization) || !LoadInsideLayer(file_handle, cAddNoise.AsObject())) ReturnFalse; //--- if(!cResidual.Init(0, 6, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(cMergeSplit[1].getGradient(), true)) ReturnFalse; if(!cRevIn.Init(0, 7, OpenCL, Neurons(), 0, cAddNoise.AsObject())) ReturnFalse; //--- if(!SetOutput(cRevIn.getOutput(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHyperProjection : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } public: CNeuronHyperProjection(void) : iWindow(-1), iUnits(-1) {}; ~CNeuronHyperProjection(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHyperProjection; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + 1)*units_count, optimization_type, batch)) ReturnFalse; iWindow = window; iUnits = units_count; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; uint local_work_size[2] = {1, iWindow}; int kernel = def_k_HyperProjection; ResetLastError(); setBuffer(kernel, def_k_lp_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_lp_outputs, getOutputIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnits, iWindow}; int kernel = def_k_HyperProjectionGrad; ResetLastError(); setBuffer(kernel, def_k_lpg_inputs, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_lpg_inputs_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_lpg_outputs_gr, getGradientIndex()) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperProjection::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHyperboloids : public CNeuronBaseOCL { protected: uint iWindows; uint iUnits; uint iCentroids; //--- CLayer cHyperCentroids; CLayer cHyperCurvatures; //--- int iProducts; int iDistances; int iNormes; //--- virtual bool LogMap(CNeuronBaseOCL *featers, CNeuronBaseOCL *centroids, CNeuronBaseOCL *curvatures, CNeuronBaseOCL *outputs); virtual bool LogMapGrad(CNeuronBaseOCL *featers, CNeuronBaseOCL *centroids, CNeuronBaseOCL *curvatures, CNeuronBaseOCL *outputs); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); public: CNeuronHyperboloids(void) : iWindows(0), iUnits(0), iCentroids(0), iProducts(-1), iDistances(-1), iNormes(-1) {}; ~CNeuronHyperboloids(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHyperboloids; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); virtual void TrainMode(bool flag); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * centroids, optimization_type, batch)) ReturnFalse; //--- iWindows = window; iUnits = units_count; iCentroids = centroids; //--- cHyperCentroids.Clear(); cHyperCurvatures.Clear(); cHyperCentroids.SetOpenCL(OpenCL); cHyperCurvatures.SetOpenCL(OpenCL); //--- CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 0, OpenCL, iUnits, iWindows, optimization, iBatch) || !cHyperCentroids.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction(None); //--- CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 1, OpenCL, iUnits, iUnits, iCentroids, iWindows, 1, optimization, iBatch) || !cHyperCentroids.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 2, OpenCL, iCentroids, iCentroids, iCentroids, 1, iWindows, optimization, iBatch) || !cHyperCentroids.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, 3, OpenCL, iWindows, iCentroids, optimization, iBatch) || !cHyperCentroids.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 4, OpenCL, iWindows, iWindows, iWindows, iCentroids, 1, optimization, iBatch) || !cHyperCurvatures.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, 5, OpenCL, iWindows, iWindows, 1, 1, iCentroids, optimization, iBatch) || !cHyperCurvatures.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- uint size = iCentroids * iUnits * sizeof(float); iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iProducts < 0) ReturnFalse; iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iDistances < 0) ReturnFalse; iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); if(iNormes < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::LogMap(CNeuronBaseOCL * featers, CNeuronBaseOCL * centroids, CNeuronBaseOCL * curvatures, CNeuronBaseOCL * outputs) { if(!featers || !centroids || !curvatures || !outputs) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iCentroids, iWindows}; uint local_work_size[3] = {1, 1, iWindows}; int kernel = def_k_LogMap; ResetLastError(); setBuffer(kernel, def_k_logmap_centroids, centroids.getOutputIndex()) setBuffer(kernel, def_k_logmap_curvatures, curvatures.getOutputIndex()) setBuffer(kernel, def_k_logmap_features, featers.getOutputIndex()) setBuffer(kernel, def_k_logmap_outputs, outputs.getOutputIndex()) setBuffer(kernel, def_k_logmap_distance, iDistances) setBuffer(kernel, def_k_logmap_product, iProducts) setBuffer(kernel, def_k_logmap_norma, iNormes) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::LogMapGrad(CNeuronBaseOCL * featers, CNeuronBaseOCL * centroids, CNeuronBaseOCL * curvatures, CNeuronBaseOCL * outputs) { if(!featers || !centroids || !curvatures || !outputs) ReturnFalse; //--- //--- CBufferFloat *temp = featers.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; temp = centroids.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; temp = curvatures.getGradient(); if(!temp || !temp.Fill(0) || !temp.BufferWrite()) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, iCentroids, iWindows}; uint local_work_size[3] = {1, 1, iWindows}; int kernel = def_k_LogMapGrad; ResetLastError(); setBuffer(kernel, def_k_logmapgr_centroids, centroids.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_centroids_gr, centroids.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_curvatures, curvatures.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_curvatures_gr, curvatures.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_features, featers.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_features_gr, featers.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_outputs, outputs.getOutputIndex()) setBuffer(kernel, def_k_logmapgr_outputs_gr, outputs.getGradientIndex()) setBuffer(kernel, def_k_logmapgr_distance, iDistances) setBuffer(kernel, def_k_logmapgr_product, iProducts) setBuffer(kernel, def_k_logmapgr_norma, iNormes) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *centroids = NULL; CNeuronBaseOCL *curvatures = NULL; //--- Centroids for(int i = 0; i < cHyperCentroids.Total(); i++) { centroids = cHyperCentroids[i]; if(!centroids || !centroids.FeedForward(prev)) ReturnFalse; prev = centroids; } //--- Curvatures for(int i = 0; i < cHyperCurvatures.Total(); i++) { curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.FeedForward(prev)) ReturnFalse; prev = curvatures; } if(!LogMap(NeuronOCL, centroids, curvatures, AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- CObject *next = NULL; CNeuronBaseOCL *centroids = cHyperCentroids[-1]; CNeuronBaseOCL *curvatures = cHyperCurvatures[-1]; //--- if(!LogMapGrad(prevLayer, centroids, curvatures, AsObject())) ReturnFalse; //--- Curvatures for(int i = cHyperCurvatures.Total() - 2; i >= 0; i--) { next = curvatures; curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.CalcHiddenGradients(next)) ReturnFalse; } CBufferFloat *temp = centroids.getGradient(); if(centroids.Activation() != None) if(!DeActivation(centroids.getOutput(), temp, temp, centroids.Activation())) ReturnFalse; if(!centroids.SetGradient(centroids.getPrevOutput(), false) || !centroids.CalcHiddenGradients(curvatures.AsObject()) || !SumAndNormilize(temp, centroids.getGradient(), temp, iWindows, false, 0, 0, 0, 1) || !centroids.SetGradient(temp, false) ) ReturnFalse; //--- Centroids for(int i = cHyperCentroids.Total() - 2; i >= 0; i--) { next = centroids; centroids = cHyperCentroids[i]; if(!centroids || !centroids.CalcHiddenGradients(next)) ReturnFalse; } //--- temp = prevLayer.getGradient(); if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), temp, temp, prevLayer.Activation())) ReturnFalse; if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) || !prevLayer.CalcHiddenGradients(centroids.AsObject()) || !SumAndNormilize(temp, prevLayer.getGradient(), temp, iWindows, false, 0, 0, 0, 1) || !prevLayer.SetGradient(temp, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *centroids = NULL; CNeuronBaseOCL *curvatures = NULL; //--- Centroids for(int i = 0; i < cHyperCentroids.Total(); i++) { centroids = cHyperCentroids[i]; if(!centroids || !centroids.UpdateInputWeights(prev)) ReturnFalse; prev = centroids; } //--- Curvatures for(int i = 0; i < cHyperCurvatures.Total(); i++) { curvatures = cHyperCurvatures[i]; if(!curvatures || !curvatures.UpdateInputWeights(prev)) ReturnFalse; prev = curvatures; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHyperboloids::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL) { OpenCL.BufferFree(iProducts); OpenCL.BufferFree(iDistances); OpenCL.BufferFree(iNormes); } //--- CNeuronBaseOCL::SetOpenCL(obj); cHyperCentroids.SetOpenCL(OpenCL); cHyperCurvatures.SetOpenCL(OpenCL); //--- uint size = iCentroids * iUnits * sizeof(float); iProducts = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); iDistances = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); iNormes = OpenCL.AddBuffer(size, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cHyperCentroids.Save(file_handle)) ReturnFalse; if(!cHyperCurvatures.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindows)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCentroids)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHyperboloids::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cHyperCentroids.Load(file_handle)) ReturnFalse; if(!cHyperCurvatures.Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindows = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iCentroids = (int)FileReadInteger(file_handle); SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHyperboloids::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); CNeuronBaseOCL *neuron = NULL; //--- for(int i = 0; i < cHyperCentroids.Total(); i++) { neuron = cHyperCentroids[i]; if(!neuron) continue; neuron.TrainMode(bTrain); } //--- for(int i = 0; i < cHyperCurvatures.Total(); i++) { neuron = cHyperCurvatures[i]; if(!neuron) continue; neuron.TrainMode(bTrain); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHypDiff : public CNeuronRMAT { public: CNeuronHypDiff(void) {}; ~CNeuronHypDiff(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHypDiff; } //--- virtual uint GetWindow(void) const override { CNeuronRMAT* neuron = cLayers[1]; return (!neuron ? 0 : neuron.GetWindow() - 1); } virtual uint GetUnits(void) const override { CNeuronRMAT* neuron = cLayers[1]; return (!neuron ? 0 : neuron.GetUnits()); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint centroids, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); int layer = 0; //--- Projection CNeuronHyperProjection *lorenz = new CNeuronHyperProjection(); if(!lorenz || !lorenz.Init(0, layer, OpenCL, window, units_count, optimization, iBatch) || !cLayers.Add(lorenz)) { DeleteObj(lorenz); ReturnFalse; } layer++; //--- Encoder CNeuronRMAT *rmat = new CNeuronRMAT(); if(!rmat || !rmat.Init(0, layer, OpenCL, window + 1, window_key, units_count, heads, layers, optimization, iBatch) || !cLayers.Add(rmat)) { DeleteObj(rmat); ReturnFalse; } layer++; //--- CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, window + 1, window + 1, 2 * window, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } layer++; conv.SetActivationFunction(TANH); //--- conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, 2 * window, 2 * window, 3, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } layer++; //--- LogMap projecction CNeuronHyperboloids *logmap = new CNeuronHyperboloids(); if(!logmap || !logmap.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) || !cLayers.Add(logmap)) { DeleteObj(logmap); ReturnFalse; } layer++; //--- Diffusion model CNeuronDiffusion *diff = new CNeuronDiffusion(); if(!diff || !diff.Init(0, layer, OpenCL, 3, window_key, heads, units_count * centroids, 2, layers, optimization, iBatch) || !cLayers.Add(diff)) { DeleteObj(diff); ReturnFalse; } layer++; //--- Pooling CNeuronMHAttentionPooling *pooling = new CNeuronMHAttentionPooling(); if(!pooling || !pooling.Init(0, layer, OpenCL, 3, units_count, centroids, optimization, iBatch) || !cLayers.Add(pooling)) { DeleteObj(pooling); ReturnFalse; } layer++; //--- Resize to source size conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, layer, OpenCL, 3, 3, window, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- if(!SetOutput(conv.getOutput(), true) || !SetGradient(conv.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBaseSAMOCL : public CNeuronBaseOCL { protected: float fRho; CBufferFloat cWeightsSAM; //--- virtual bool calcEpsilonWeights(CNeuronBaseOCL *NeuronOCL); virtual bool feedForwardSAM(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronBaseSAMOCL(void) {}; ~CNeuronBaseSAMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, float rho, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) const override { return defNeuronBaseSAMOCL; } virtual int Activation(void) const override { return (fRho == 0 ? (int)None : (int)activation); } virtual int getWeightsSAMIndex(void) const override { return cWeightsSAM.GetIndex(); } //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return Init(numOutputs, myIndex, open_cl, numNeurons, 0.7f, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, float rho, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; fRho = fabs(rho); if(fRho == 0 || !Weights) return true; //--- if(!cWeightsSAM.BufferInit(Weights.Total(), 0) || !cWeightsSAM.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::calcEpsilonWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = { 0, 0 }; uint global_work_size[2] = { NeuronOCL.Neurons() + 1, Output.Total() }; uint local_work_size[2] = { global_work_size[0], 1 }; const int kernel = def_k_CalcEpsilonWeights; //--- setBuffer(kernel, def_k_epsw_matrix_w, NeuronOCL.getWeightsIndex()) setBuffer(kernel, def_k_epsw_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_epsw_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_epsw_matrix_epsw, NeuronOCL.getWeightsSAMIndex()) setArgument(kernel, def_k_epsw_rho, fRho) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::feedForwardSAM(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { Output.Total() }; //--- const int kernel = def_k_FeedForward; setBuffer(kernel, def_k_ff_matrix_w, NeuronOCL.getWeightsSAMIndex()) setBuffer(kernel, def_k_ff_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ff_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ff_inputs, NeuronOCL.Neurons()) setArgument(kernel, def_k_ff_activation, (int)activation) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(NeuronOCL.getWeightsSAMIndex() < 0 || fRho == 0) return CNeuronBaseOCL::updateInputWeights(NeuronOCL); //--- if(!SumAndNormilize(Gradient, Output, Gradient, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!calcEpsilonWeights(NeuronOCL)) ReturnFalse; if(!feedForwardSAM(NeuronOCL)) ReturnFalse; float error = 1; if(!calcOutputGradients(Gradient, error)) ReturnFalse; //--- return CNeuronBaseOCL::updateInputWeights(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::calcHiddenGradients(CNeuronBaseOCL * NeuronOCL) { if(activation == None) return CNeuronBaseOCL::calcHiddenGradients(NeuronOCL); //--- CBufferFloat *temp = Gradient; Gradient = PrevOutput; if(!DeActivation(Output, temp, Gradient, activation) || !CNeuronBaseOCL::calcHiddenGradients(NeuronOCL)) ReturnFalse; Gradient = temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBaseSAMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); if(cWeightsSAM.GetOpenCL() != OpenCL) cWeightsSAM.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteFloat(file_handle, fRho) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseSAMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; fRho = FileReadFloat(file_handle); if(fRho == 0 || !Weights) return true; cWeightsSAM.BufferFree(); if(!cWeightsSAM.BufferInit(Weights.Total(), 0) || !cWeightsSAM.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { return CNeuronConvSAMOCL::Init(numOutputs, myIndex, open_cl, window_in, step, window_out, units_count, variables, 0.7f, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint step, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window_in, step, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; fRho = fabs(rho); if(fRho == 0) return true; //--- cWeightsSAMConv.BufferFree(); if(!cWeightsSAMConv.BufferInit(WeightsConv.Total(), 0) || !cWeightsSAMConv.BufferCreate(OpenCL)) ReturnFalse; //--- cWeightsSAM.BufferFree(); if(!Weights) return true; if(!cWeightsSAM.BufferInit(Weights.Total(), 0) || !cWeightsSAM.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::calcEpsilonWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[3] = { 0, 0, 0 }; uint global_work_size[3] = { iWindow + 1, iWindowOut, iVariables }; uint local_work_size[3] = { global_work_size[0], 1, 1 }; const int kernel = def_k_CalcEpsilonWeightsConv; //--- setBuffer(kernel, def_k_epswconv_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_epswconv_matrix_epsw, cWeightsSAMConv.GetIndex()) setBuffer(kernel, def_k_epswconv_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_epswconv_matrix_w, WeightsConv.GetIndex()) setArgument(kernel, def_k_epswconv_rho, fRho) setArgument(kernel, def_k_epswconv_inputs, int(NeuronOCL.Neurons() / iVariables)) setArgument(kernel, def_k_epswconv_step, int(iStep)) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::feedForwardSAM(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {Output.Total() / (iWindowOut * iVariables), iWindowOut, iVariables }; //--- const int kernel = def_k_FeedForwardConv; setBuffer(kernel, def_k_ffc_matrix_w, cWeightsSAMConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffс_window_out, (int)iWindowOut) setArgument(kernel, def_k_ffc_activation, (int)activation) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(fRho == 0) return CNeuronConvOCL::updateInputWeights(NeuronOCL); //--- if(!SumAndNormilize(Gradient, Output, Gradient, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!calcEpsilonWeights(NeuronOCL)) ReturnFalse; if(!feedForwardSAM(NeuronOCL)) ReturnFalse; float error = 1; if(!calcOutputGradients(Gradient, error)) ReturnFalse; //--- return CNeuronConvOCL::updateInputWeights(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(activation == None) return CNeuronConvOCL::calcInputGradients(NeuronOCL); //--- CBufferFloat *temp = Gradient; Gradient = PrevOutput; if(!DeActivation(Output, temp, Gradient, activation) || !CNeuronConvOCL::calcInputGradients(NeuronOCL)) ReturnFalse; Gradient = temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConvSAMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); cWeightsSAMConv.BufferCreate(OpenCL); if(!!Weights) cWeightsSAM.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteFloat(file_handle, fRho) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; fRho = FileReadFloat(file_handle); //--- cWeightsSAMConv.BufferFree(); cWeightsSAM.BufferFree(); cWeightsSAMConv.Clear(); cWeightsSAM.Clear(); if(fRho <= 0) return true; //--- if(!cWeightsSAMConv.BufferInit(WeightsConv.Total(), 0) || !cWeightsSAMConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!Weights) return true; if(!cWeightsSAM.BufferInit(Weights.Total(), 0) || !cWeightsSAM.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CBufferFloat::BufferInitLike(CBufferFloat * master) { if(!master) ReturnFalse; if(!!OpenCL && m_myIndex >= 0) { if(!BufferFree()) ReturnFalse; } if(!BufferInit(master.Total(), 0)) ReturnFalse; if(!BufferCreate(master.GetOpenCL())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::InitBufferLike(CBufferFloat *&buffer, CBufferFloat * master) { if(!buffer) { buffer = new CBufferFloat(); if(!buffer) ReturnFalse; } //--- return buffer.BufferInitLike(master); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConvSAMOCL::ReplaceBuffer(CBufferFloat *&buffer, CBufferFloat * master) { if(buffer == master) return; if(!!buffer) { buffer.BufferFree(); DeleteObj(buffer); } //--- buffer = master; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConvSAMOCL::InitPS(CNeuronConvSAMOCL * master) { if(!master || master.Type() != Type() ) ReturnFalse; //--- alpha = master.alpha; iBatch = master.iBatch; t = master.t; m_myIndex = master.m_myIndex; activation = master.activation; optimization = master.optimization; iWindow = master.iWindow; iStep = master.iStep; iWindowOut = master.iWindowOut; iVariables = master.iVariables; bTrain = master.bTrain; fRho = master.fRho; //--- if(!InitBufferLike(Output, master.Output)) ReturnFalse; if(!!master.getPrevOutput()) if(!InitBufferLike(PrevOutput, master.getPrevOutput())) ReturnFalse; if(!InitBufferLike(Gradient, master.Gradient)) ReturnFalse; //--- ReplaceBuffer(Weights, master.Weights); ReplaceBuffer(DeltaWeights, master.DeltaWeights); ReplaceBuffer(FirstMomentum, master.FirstMomentum); ReplaceBuffer(SecondMomentum, master.SecondMomentum); //--- ReplaceBuffer(WeightsConv, master.WeightsConv); ReplaceBuffer(DeltaWeightsConv, master.DeltaWeightsConv); ReplaceBuffer(FirstMomentumConv, master.FirstMomentumConv); ReplaceBuffer(SecondMomentumConv, master.SecondMomentumConv); //--- if(master.cWeightsSAM.Total() > 0) { CBufferFloat *buf = GetPointer(cWeightsSAM); if(!InitBufferLike(buf, GetPointer(master.cWeightsSAM))) ReturnFalse; } else { cWeightsSAM.BufferFree(); cWeightsSAM.Clear(); } //--- if(fRho > 0) { CBufferFloat *buf = GetPointer(cWeightsSAMConv); if(!InitBufferLike(buf, GetPointer(master.cWeightsSAMConv))) ReturnFalse; } else { cWeightsSAMConv.BufferFree(); cWeightsSAMConv.Clear(); } //--- SetOpenCL(master.OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPSBlock : public CNeuronConvSAMOCL { protected: CNeuronConvSAMOCL acConvolution[2]; CNeuronBaseOCL cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPSBlock(void) {}; ~CNeuronPSBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool InitPS(CNeuronPSBlock *master); //--- virtual int Type(void) const override { return defNeuronPSBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, float rho, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvSAMOCL::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count, variables, rho, optimization_type, batch)) ReturnFalse; if(!acConvolution[0].Init(0, 0, OpenCL, iWindow, iWindow, iWindowOut, units_count, iVariables, fRho, optimization, iBatch)) ReturnFalse; acConvolution[0].SetActivationFunction(GELU); if(!acConvolution[1].Init(0, 1, OpenCL, iWindowOut, iWindowOut, iWindow, units_count, iVariables, fRho, optimization, iBatch)) ReturnFalse; acConvolution[1].SetActivationFunction(None); if(!cResidual.Init(0, 2, OpenCL, acConvolution[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acConvolution[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction(None); //--- SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::InitPS(CNeuronPSBlock * master) { if(!CNeuronConvSAMOCL::InitPS((CNeuronConvSAMOCL*)master)) ReturnFalse; for(int i = 0; i < 2; i++) if(!acConvolution[i].InitPS(master.acConvolution[i].AsObject())) ReturnFalse; if(!cResidual.Init(0, 2, OpenCL, acConvolution[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acConvolution[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!acConvolution[0].FeedForward(NeuronOCL)) ReturnFalse; if(!acConvolution[1].FeedForward(acConvolution[0].AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), acConvolution[1].getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronConvSAMOCL::feedForward(cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvSAMOCL::calcInputGradients(cResidual.AsObject())) ReturnFalse; if(!acConvolution[0].CalcHiddenGradients(acConvolution[1].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(acConvolution[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cResidual.getPrevOutput(), NeuronOCL.Activation()) || !SumAndNormilize(NeuronOCL.getGradient(), cResidual.getPrevOutput(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronConvSAMOCL::updateInputWeights(cResidual.AsObject())) ReturnFalse; if(!acConvolution[1].UpdateInputWeights(acConvolution[0].AsObject())) ReturnFalse; if(!acConvolution[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::Save(const int file_handle) { if(!CNeuronConvSAMOCL::Save(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) if(!acConvolution[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSBlock::Load(const int file_handle) { if(!CNeuronConvSAMOCL::Load(file_handle)) ReturnFalse; //--- for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, acConvolution[i].AsObject())) ReturnFalse; //--- if(!cResidual.Init(0, 2, OpenCL, acConvolution[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acConvolution[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSBlock::SetOpenCL(COpenCLMy * obj) { CNeuronConvSAMOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) acConvolution[i].SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPSformer : public CNeuronBaseSAMOCL { protected: CNeuronTransposeOCL acTranspose[2]; CNeuronPSBlock acPSBlocks[3]; CNeuronRelativeSelfAttention acAttention[2]; CNeuronBaseOCL cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPSformer(void) {}; ~CNeuronPSformer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint segments, float rho, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronPSformer; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint segments, float rho, ENUM_OPTIMIZATION optimization_type, uint batch) { if((window * units_count) % segments > 0) ReturnFalse; if(!CNeuronBaseSAMOCL::Init(numOutputs, myIndex, open_cl, window * units_count, rho, optimization_type, batch)) ReturnFalse; //--- uint count = Neurons() / segments; //--- if(!acTranspose[0].Init(0, 0, OpenCL, segments, count, optimization, iBatch)) ReturnFalse; acTranspose[0].SetActivationFunction(None); //--- if(!acPSBlocks[0].Init(0, 1, OpenCL, segments, segments, units_count / segments, 1, fRho, optimization, iBatch)) ReturnFalse; for(int i = 0; i < 2; i++) { if(!acAttention[i].Init(0, i + 2, OpenCL, segments, segments, units_count / segments, 2, optimization, iBatch)) ReturnFalse; if(!acPSBlocks[i + 1].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject())) ReturnFalse; } //--- if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acAttention[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation()); //--- if(!acTranspose[1].Init(0, 5, OpenCL, count, segments, optimization, iBatch)) ReturnFalse; acTranspose[1].SetActivationFunction((ENUM_ACTIVATION)acPSBlocks[2].Activation()); //--- if(!SetOutput(acTranspose[1].getOutput(), true) || !SetGradient(acTranspose[1].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Dimension Transformation if(!acTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; //--- Segment Attention CObject* prev = acTranspose[0].AsObject(); for(int i = 0; i < 2; i++) { if(!acPSBlocks[i].FeedForward(prev)) ReturnFalse; if(!acAttention[i].FeedForward(acPSBlocks[i].AsObject())) ReturnFalse; prev = acAttention[i].AsObject(); } //--- Residual Add if(!SumAndNormilize(acTranspose[0].getOutput(), acAttention[1].getOutput(), cResidual.getOutput(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- PS Block if(!acPSBlocks[2].FeedForward(cResidual.AsObject())) ReturnFalse; //--- Inverse Transformation if(!acTranspose[1].FeedForward(acPSBlocks[2].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!acPSBlocks[2].CalcHiddenGradients(acTranspose[1].AsObject())) ReturnFalse; //--- if(!cResidual.CalcHiddenGradients(acPSBlocks[2].AsObject())) ReturnFalse; //--- if(!acPSBlocks[1].CalcHiddenGradients(acAttention[1].AsObject())) ReturnFalse; if(!acAttention[0].CalcHiddenGradients(acPSBlocks[1].AsObject())) ReturnFalse; if(!acPSBlocks[0].CalcHiddenGradients(acAttention[0].AsObject())) ReturnFalse; //--- if(!acTranspose[0].CalcHiddenGradients(acPSBlocks[0].AsObject())) ReturnFalse; if(acTranspose[0].Activation() == None) { if(!SumAndNormilize(acTranspose[0].getGradient(), cResidual.getGradient(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(acTranspose[0].getOutput(), cResidual.getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].Activation()) || !SumAndNormilize(acTranspose[0].getGradient(), acTranspose[0].getPrevOutput(), acTranspose[0].getGradient(), acAttention[1].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(acTranspose[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!acPSBlocks[2].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; //--- CObject* prev = acAttention[0].AsObject(); for(int i = 1; i >= 0; i--) { if(!acAttention[i].UpdateInputWeights(acPSBlocks[i].AsObject())) ReturnFalse; if(!acPSBlocks[i].UpdateInputWeights(prev)) ReturnFalse; prev = acTranspose[0].AsObject(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseSAMOCL::SetOpenCL(obj); for(int i = 0; i < 2; i++) { acTranspose[i].SetOpenCL(OpenCL); acPSBlocks[i].SetOpenCL(OpenCL); acAttention[i].SetOpenCL(OpenCL); } acPSBlocks[2].SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::Save(const int file_handle) { if(!CNeuronBaseSAMOCL::Save(file_handle)) ReturnFalse; if(!acPSBlocks[0].Save(file_handle)) ReturnFalse; for(int i = 0; i < 2; i++) if(!acTranspose[i].Save(file_handle) || !acAttention[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSformer::Load(const int file_handle) { if(!CNeuronBaseSAMOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, acPSBlocks[0].AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!LoadInsideLayer(file_handle, acTranspose[i].AsObject()) || !LoadInsideLayer(file_handle, acAttention[i].AsObject())) ReturnFalse; //--- for(int i = 1; i < 3; i++) if(!acPSBlocks[i].InitPS((CNeuronPSBlock*)acPSBlocks[0].AsObject())) ReturnFalse; //--- if(!cResidual.Init(0, 4, OpenCL, acAttention[1].Neurons(), optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(acAttention[1].getGradient(), true)) ReturnFalse; cResidual.SetActivationFunction((ENUM_ACTIVATION)acAttention[1].Activation()); //--- if(!SetOutput(acTranspose[1].getOutput(), true) || !SetGradient(acTranspose[1].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMarketObserver : public CNeuronRMAT { public: CNeuronMarketObserver(void) {}; ~CNeuronMarketObserver(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMarketObserver; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMarketObserver::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint forecast, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Init parent object if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * forecast, optimization_type, batch)) ReturnFalse; //--- Clear layers' array cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- Tranpose input data int lay_count = 0; CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, lay_count, OpenCL, units_count, window, optimization, iBatch) || !cLayers.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Piecewise linear representation lay_count++; CNeuronPLROCL *plr = new CNeuronPLROCL(); if(!plr || !plr.Init(0, lay_count, OpenCL, units_count, window, false, optimization, iBatch) || !cLayers.Add(plr)) { DeleteObj(plr); ReturnFalse; } //--- Self-Attention for Variables lay_count++; CNeuronRMAT *att = new CNeuronRMAT(); if(!att || !att.Init(0, lay_count, OpenCL, units_count, window_key, window, heads, layers, optimization, iBatch) || !cLayers.Add(att)) { DeleteObj(att); ReturnFalse; } //--- Forecast mapping lay_count++; CResidualConv *conv = new CResidualConv(); if(!conv || !conv.Init(0, lay_count, OpenCL, units_count, forecast, window, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- Back transpose forecast lay_count++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, lay_count, OpenCL, window, forecast, optimization, iBatch) || !cLayers.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- if(!SetOutput(transp.getOutput(), true) || !SetGradient(transp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRLAgent : public CNeuronRMAT { public: CNeuronRLAgent(void) {}; ~CNeuronRLAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint segments, float rho, uint layers, uint n_actions, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronRLAgent; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRLAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint segments, float rho, uint layers, uint n_actions, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Init parent object if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, n_actions, optimization_type, batch)) ReturnFalse; //--- Clear layers' array cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- State observation int lay_count = 0; for(uint i = 0; i < layers; i++) { CNeuronPSformer *psf = new CNeuronPSformer(); if(!psf || !psf.Init(0, lay_count, OpenCL, window, units_count, segments, rho, optimization, iBatch) || !cLayers.Add(psf)) { DeleteObj(psf); ReturnFalse; } lay_count++; } //--- CNeuronConvSAMOCL *conv = new CNeuronConvSAMOCL(); if(!conv || !conv.Init(n_actions, lay_count, OpenCL, window, window, 1, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(GELU); lay_count++; CNeuronBaseSAMOCL *flat = new CNeuronBaseSAMOCL(); if(!flat || !flat.Init(0, lay_count, OpenCL, n_actions, optimization, iBatch) || !cLayers.Add(flat)) { DeleteObj(flat); ReturnFalse; } SetActivationFunction(SIGMOID); //--- if(!SetOutput(flat.getOutput(), true) || !SetGradient(flat.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRMAT::SetActivationFunction(ENUM_ACTIVATION value) { CNeuronBaseOCL *neuron = cLayers[-1]; if(!neuron) return; neuron.SetActivationFunction(value); CNeuronBaseOCL::SetActivationFunction((ENUM_ACTIVATION)neuron.Activation()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronControlAgent : public CNeuronRMAT { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronControlAgent(void) {}; ~CNeuronControlAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronControlAgent; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronControlAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint window_kv, uint units_kv, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- Init parent object if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- Clear layers' array cLayers.Clear(); cLayers.SetOpenCL(OpenCL); //--- Second buffer prepare int lay_count = 0; CNeuronBaseOCL *flat = new CNeuronBaseOCL(); if(!flat || !flat.Init(0, lay_count, OpenCL, window_kv * units_kv, optimization, iBatch) || !cLayers.Add(flat)) { DeleteObj(flat); ReturnFalse; } lay_count++; CNeuronTransposeOCL *transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, lay_count, OpenCL, units_kv, window_kv, optimization, iBatch) || !cLayers.Add(transp)) { DeleteObj(transp); ReturnFalse; } lay_count++; //--- Attention Action To Observation for(uint i = 0; i < layers; i++) { if(units_count > 1) { CNeuronRelativeSelfAttention *self = new CNeuronRelativeSelfAttention(); if(!self || !self.Init(0, lay_count, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(self)) { DeleteObj(self); ReturnFalse; } lay_count++; } CNeuronRelativeCrossAttention *cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, lay_count, OpenCL, window, window_key, units_count, heads, units_kv, window_kv, optimization, iBatch) || !cLayers.Add(cross)) { DeleteObj(cross); ReturnFalse; } lay_count++; CResidualConv *ffn = new CResidualConv(); if(!ffn || !ffn.Init(0, lay_count, OpenCL, window, window, units_count, optimization, iBatch) || !cLayers.Add(ffn)) { DeleteObj(ffn); ReturnFalse; } lay_count++; } CNeuronConvSAMOCL *conv = new CNeuronConvSAMOCL(); if(!conv || !conv.Init(0, lay_count, OpenCL, window, window, window, units_count, 1, optimization, iBatch) || !cLayers.Add(conv)) { DeleteObj(conv); ReturnFalse; } SetActivationFunction(SIGMOID); //--- if(!SetOutput(conv.getOutput(), true) || !SetGradient(conv.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronControlAgent::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!SecondInput) ReturnFalse; //--- CNeuronBaseOCL *second = cLayers[0]; if(!second) ReturnFalse; if(!second.SetOutput(SecondInput, true)) ReturnFalse; //--- second = cLayers[1]; if(!second || !second.FeedForward(cLayers[0])) ReturnFalse; //--- CNeuronBaseOCL *first = NeuronOCL; CNeuronBaseOCL *main = NULL; for(int i = 2; i < cLayers.Total(); i++) { main = cLayers[i]; if(!main || !main.FeedForward(first, second.getOutput())) ReturnFalse; first = main; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronControlAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL *main = cLayers[0]; if(!main) ReturnFalse; if(!main.SetGradient(SecondGradient, true)) ReturnFalse; main.SetActivationFunction(SecondActivation); //--- CNeuronBaseOCL *second = cLayers[1]; if(!second) ReturnFalse; second.SetActivationFunction(SecondActivation); CBufferFloat *second_out = second.getOutput(); CBufferFloat *second_gr = second.getGradient(); CBufferFloat *temp = second.getPrevOutput(); if(!second_gr.Fill(0)) ReturnFalse; //--- for(int i = cLayers.Total() - 2; i >= 2; i--) { main = cLayers[i]; if(!main) ReturnFalse; if(cLayers[i + 1].Type() == defNeuronRelativeCrossAttention) { if(!main.CalcHiddenGradients(cLayers[i + 1], second_out, temp, SecondActivation) || !SumAndNormilize(temp, second_gr, second_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!main.CalcHiddenGradients(cLayers[i + 1])) ReturnFalse; } } //--- if(!NeuronOCL.CalcHiddenGradients(main.AsObject(), second_out, temp, SecondActivation)) ReturnFalse; if(main.Type() == defNeuronRelativeCrossAttention) { if(!SumAndNormilize(temp, second_gr, second_gr, 1, false, 0, 0, 0, 1)) ReturnFalse; } main = cLayers[0]; if(!main.CalcHiddenGradients(second.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronControlAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { CNeuronBaseOCL *second = cLayers[1]; CNeuronBaseOCL *main = NULL; for(int i = cLayers.Total() - 1; i > 2; i--) { main = cLayers[i]; if(!main || !main.UpdateInputWeights(cLayers[i - 1], second.getOutput())) ReturnFalse; } main = cLayers[2]; if(!main || !main.UpdateInputWeights(NeuronOCL, second.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMASA : public CNeuronBaseSAMOCL { protected: CNeuronMarketObserver cMarketObserver; CNeuronRevINDenormOCL cRevIN; CNeuronRLAgent cRLAgent; CNeuronControlAgent cControlAgent; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMASA(void) {}; ~CNeuronMASA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers_mo, uint forecast, uint segments_rl, float rho, uint layers_rl, uint n_actions, uint heads_contr, uint layers_contr, int NormLayer, CNeuronBatchNormOCL *normLayer, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMASA; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override; //--- virtual int GetNormLayer(void) { return cRevIN.GetNormLayer(); } virtual bool SetNormLayer(int NormLayer, CNeuronBatchNormOCL *normLayer); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads_mo, uint layers_mo, uint forecast, uint segments_rl, float rho, uint layers_rl, uint n_actions, uint heads_contr, uint layers_contr, int NormLayer, CNeuronBatchNormOCL * normLayer, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseSAMOCL::Init(numOutputs, myIndex, open_cl, n_actions, rho, optimization_type, batch)) ReturnFalse; //--- Market Observation if(!cMarketObserver.Init(0, 0, OpenCL, window, window_key, units_count, heads_mo, layers_mo, forecast, optimization, iBatch)) ReturnFalse; if(!cRevIN.Init(0, 1, OpenCL, cMarketObserver.Neurons(), NormLayer, normLayer)) ReturnFalse; //--- RL Agent if(!cRLAgent.Init(0, 2, OpenCL, window, units_count, segments_rl, fRho, layers_rl, n_actions, optimization, iBatch)) ReturnFalse; //--- if(!cControlAgent.Init(0, 3, OpenCL, 3, window_key, n_actions / 3, heads_contr, window, forecast, layers_contr, optimization, iBatch)) ReturnFalse; //--- if(!SetOutput(cControlAgent.getOutput(), true) || !SetGradient(cControlAgent.getGradient(), true)) ReturnFalse; SetActivationFunction(SIGMOID); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMASA::SetActivationFunction(ENUM_ACTIVATION value) { cControlAgent.SetActivationFunction(value); cRLAgent.SetActivationFunction((ENUM_ACTIVATION)cControlAgent.Activation()); CNeuronBaseSAMOCL::SetActivationFunction((ENUM_ACTIVATION)cControlAgent.Activation()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cMarketObserver.FeedForward(NeuronOCL.AsObject())) ReturnFalse; if(!cRLAgent.FeedForward(NeuronOCL.AsObject())) ReturnFalse; if(!cControlAgent.FeedForward(cRLAgent.AsObject(), cMarketObserver.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- if(!cRevIN.FeedForward(cMarketObserver.AsObject())) ReturnFalse; float error = 1.0f; if(!cRevIN.calcOutputGradients(SecondGradient, error)) ReturnFalse; if(!cMarketObserver.CalcHiddenGradients(cRevIN.AsObject())) ReturnFalse; if(!cRLAgent.CalcHiddenGradients(cControlAgent.AsObject(), cMarketObserver.getOutput(), cMarketObserver.getPrevOutput(), (ENUM_ACTIVATION)cMarketObserver.Activation()) || !SumAndNormilize(cMarketObserver.getGradient(), cMarketObserver.getPrevOutput(), cMarketObserver.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- CBufferFloat *temp = cRLAgent.getGradient(); if(!cRLAgent.SetGradient(cRLAgent.getPrevOutput(), false) || !cRLAgent.calcOutputGradients(cControlAgent.getOutput(), error) || !SumAndNormilize(temp, cRLAgent.getPrevOutput(), temp, 1, false, 0, 0, 0, 1) || !cRLAgent.SetGradient(temp, false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cMarketObserver.AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.calcOutputGradients(cRLAgent.getOutput(), error) || !SumAndNormilize(temp, NeuronOCL.getPrevOutput(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cControlAgent.UpdateInputWeights(cRLAgent.AsObject(), cMarketObserver.getOutput())) ReturnFalse; if(!cRLAgent.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMarketObserver.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMASA::SetOpenCL(COpenCLMy * obj) { CNeuronBaseSAMOCL::SetOpenCL(obj); cMarketObserver.SetOpenCL(OpenCL); cRevIN.SetOpenCL(OpenCL); cRLAgent.SetOpenCL(OpenCL); cControlAgent.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::Save(const int file_handle) { if(!CNeuronBaseSAMOCL::Save(file_handle)) ReturnFalse; if(!cMarketObserver.Save(file_handle)) ReturnFalse; if(!cRevIN.Save(file_handle)) ReturnFalse; if(!cRLAgent.Save(file_handle)) ReturnFalse; if(!cControlAgent.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::Load(const int file_handle) { if(!CNeuronBaseSAMOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMarketObserver.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRevIN.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRLAgent.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cControlAgent.AsObject())) ReturnFalse; //--- if(!SetOutput(cControlAgent.getOutput(), true) || !SetGradient(cControlAgent.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASA::SetNormLayer(int NormLayer, CNeuronBatchNormOCL * normLayer) { return cRevIN.Init(0, 1, OpenCL, cMarketObserver.Neurons(), NormLayer, normLayer); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPLRMultiAgentsOCL : public CNeuronPLROCL { protected: int iAgents; CBufferFloat cMinDistance; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronPLRMultiAgentsOCL(void) : iAgents(1) {}; ~CNeuronPLRMultiAgentsOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint units_count, bool transpose, vector &min_distance, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronPLRMultiAgentsOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLRMultiAgentsOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint units_count, bool transpose, vector &min_distance, ENUM_OPTIMIZATION optimization_type, uint batch) { iAgents = (int)min_distance.Size(); if(iAgents <= 0) ReturnFalse; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_in * units_count * iAgents, optimization_type, batch)) ReturnFalse; //--- iVariables = (int)window_in; iCount = (int)units_count; bTranspose = transpose; //--- icIsTTP = OpenCL.AddBuffer(sizeof(int) * Neurons(), CL_MEM_READ_WRITE); if(icIsTTP < 0) ReturnFalse; //--- if(!cMinDistance.AssignArray(min_distance) || !cMinDistance.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLRMultiAgentsOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput()) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iVariables, iAgents}; uint local_work_size[3] = {iCount, 1, 1}; ResetLastError(); const int kernel = def_k_PLRMultiAgents; setBuffer(kernel, def_k_plr_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_plr_outputs, getOutputIndex()) setBuffer(kernel, def_k_plt_isttp, icIsTTP) setArgument(kernel, def_k_plr_transpose, (int)bTranspose) setBuffer(kernel, def_k_plr_step, cMinDistance.GetIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLRMultiAgentsOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!OpenCL || !prevLayer || !prevLayer.getGradient()) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iVariables}; ResetLastError(); const int kernel = def_k_PLRMultiAgentsGrad; setBuffer(kernel, def_k_plrg_inputs_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_plrg_outputs, getOutputIndex()) setBuffer(kernel, def_k_plrg_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_plrg_transpose, (int)bTranspose) setArgument(kernel, def_k_plrg_agents, (int)iAgents) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLRMultiAgentsOCL::Save(const int file_handle) { if(!CNeuronPLROCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iAgents) < INT_VALUE) ReturnFalse; if(!cMinDistance.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPLRMultiAgentsOCL::Load(const int file_handle) { if(!CNeuronPLROCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iAgents = FileReadInteger(file_handle); if(!cMinDistance.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPLRMultiAgentsOCL::SetOpenCL(COpenCLMy * obj) { //--- CNeuronPLROCL::SetOpenCL(obj); cMinDistance.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCrossSectionalAnalysis : public CNeuronMVMHAttentionMLKV { protected: CNeuronConvOCL cEmbeding; CNeuronTransposeRCDOCL cTransposeRCD; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCrossSectionalAnalysis(void) {}; ~CNeuronCrossSectionalAnalysis(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) const override { return defNeuronCrossSectionalAnalysis; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMVMHAttentionMLKV::Init(numOutputs, myIndex, open_cl, window_key, window_key, heads, heads_kv, variables, layers, layers_to_one_kv, units_count, optimization_type, batch)) ReturnFalse; //--- if(!cEmbeding.Init(0, 0, OpenCL, window, window, window_key, units_count, variables, optimization, iBatch)) ReturnFalse; cEmbeding.SetActivationFunction(GELU); if(!cTransposeRCD.Init(0, 1, OpenCL, variables, units_count, window_key, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cEmbeding.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeRCD.FeedForward(cEmbeding.AsObject())) ReturnFalse; //--- return CNeuronMVMHAttentionMLKV::feedForward(cTransposeRCD.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronMVMHAttentionMLKV::calcInputGradients(cTransposeRCD.AsObject())) ReturnFalse; if(!cEmbeding.CalcHiddenGradients(cTransposeRCD.AsObject())) ReturnFalse; if(cEmbeding.Activation() != None) { if(!DeActivation(cEmbeding.getOutput(), cEmbeding.getGradient(), cEmbeding.getGradient(), cEmbeding.Activation())) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(cEmbeding.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMVMHAttentionMLKV::updateInputWeights(cTransposeRCD.AsObject())) ReturnFalse; if(!cEmbeding.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCrossSectionalAnalysis::SetOpenCL(COpenCLMy * obj) { CNeuronMVMHAttentionMLKV::SetOpenCL(obj); cEmbeding.SetOpenCL(OpenCL); cTransposeRCD.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::Save(const int file_handle) { if(!CNeuronMVMHAttentionMLKV::Save(file_handle)) ReturnFalse; if(!cEmbeding.Save(file_handle)) ReturnFalse; if(!cTransposeRCD.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossSectionalAnalysis::Load(const int file_handle) { if(!CNeuronMVMHAttentionMLKV::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEmbeding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeRCD.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTransposeVRCOCL : public CNeuronTransposeOCL { protected: CNeuronTransposeOCL cTranspose; CNeuronTransposeRCDOCL cTransposeRCD; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; public: CNeuronTransposeVRCOCL(void) {}; ~CNeuronTransposeVRCOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronTransposeVRCOCL; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetWindow(void) override const { return cTransposeRCD.GetWindow(); } virtual uint GetCount(void) override const { return cTransposeRCD.GetCount(); } virtual uint GetVariables(void) const { return iWindow; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint variables, uint count, uint window, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, count * window, variables, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, variables, count * window, optimization, iBatch)) ReturnFalse; if(!cTransposeRCD.Init(0, 1, OpenCL, count, window, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeRCD.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cTransposeRCD.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronTransposeOCL::calcInputGradients(cTransposeRCD.AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cTransposeRCD.AsObject())) ReturnFalse; //--- return prevLayer.CalcHiddenGradients(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cTransposeRCD.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTransposeVRCOCL::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeRCD.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTransposeVRCOCL::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cTransposeRCD.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTemporalAnalysis : public CNeuronCrossSectionalAnalysis { protected: CNeuronTransposeVRCOCL cTranspose; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTemporalAnalysis(void) {}; ~CNeuronTemporalAnalysis(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) const override { return defNeuronTemporalAnalysis; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossSectionalAnalysis::Init(numOutputs, myIndex, open_cl, 3 * units_count, window_key, heads, heads_kv, window / 3, layers, layers_to_one_kv, variables, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, variables, units_count, window, optimization_type, batch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- return CNeuronCrossSectionalAnalysis::feedForward(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronCrossSectionalAnalysis::calcInputGradients(cTranspose.AsObject())) ReturnFalse; //--- return prevLayer.CalcHiddenGradients(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { return CNeuronCrossSectionalAnalysis::updateInputWeights(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTemporalAnalysis::SetOpenCL(COpenCLMy * obj) { CNeuronCrossSectionalAnalysis::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::Save(const int file_handle) { if(!CNeuronCrossSectionalAnalysis::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTemporalAnalysis::Load(const int file_handle) { if(!CNeuronCrossSectionalAnalysis::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPortfolioGenerator : public CNeuronBaseOCL { protected: uint iAssets; uint iTimePoints; uint iAgents; uint iDimension; //--- CNeuronBaseOCL cAssetTime[2]; CNeuronTransposeVRCOCL cTransposeVRC; CNeuronSoftMaxOCL cSoftMax; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPortfolioGenerator(void) {}; ~CNeuronPortfolioGenerator(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint assets, uint time_points, uint dimension, uint agents, uint projection, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronPortfolioGenerator; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint assets, uint time_points, uint dimension, uint agents, uint projection, ENUM_OPTIMIZATION optimization_type, uint batch) { if(assets <= 0 || time_points <= 0 || dimension <= 0 || agents <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, projection, optimization_type, batch)) ReturnFalse; //--- iAssets = assets; iTimePoints = time_points; iDimension = dimension; iAgents = agents; //--- if(!cTransposeVRC.Init(0, 0, OpenCL, iAgents, iTimePoints, iDimension, optimization, iBatch)) ReturnFalse; if(!cAssetTime[0].Init(0, 1, OpenCL, iAssets * iTimePoints * iAgents, optimization, iBatch)) ReturnFalse; cAssetTime[0].SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAssetTime[0].Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iAssets * iAgents); if(!cAssetTime[1].Init(Neurons(), 3, OpenCL, iAssets * iDimension * iAgents, optimization, iBatch)) ReturnFalse; cAssetTime[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!SecondInput) ReturnFalse; //--- if(!cTransposeVRC.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!MatMul(SecondInput, cTransposeVRC.getOutput(), cAssetTime[0].getOutput(), iAssets, iDimension, iTimePoints, iAgents)) ReturnFalse; if(!cSoftMax.FeedForward(cAssetTime[0].AsObject())) ReturnFalse; if(!MatMul(cSoftMax.getOutput(), NeuronOCL.getOutput(), cAssetTime[1].getOutput(), iAssets, iTimePoints, iDimension, iAgents)) ReturnFalse; //--- return CNeuronBaseOCL::feedForward(cAssetTime[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient || !SecondInput) ReturnFalse; if(!CNeuronBaseOCL::calcInputGradients(cAssetTime[1].AsObject())) ReturnFalse; if(!MatMulGrad(cSoftMax.getOutput(), cSoftMax.getGradient(), NeuronOCL.getOutput(), cTransposeVRC.getPrevOutput(), cAssetTime[1].getGradient(), iAssets, iTimePoints, iDimension, iAgents)) ReturnFalse; if(!cAssetTime[0].CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!MatMulGrad(SecondInput, SecondGradient, cTransposeVRC.getOutput(), cTransposeVRC.getGradient(), cAssetTime[0].getGradient(), iAssets, iDimension, iTimePoints, iAgents)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTransposeVRC.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cTransposeVRC.getPrevOutput(), NeuronOCL.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cTransposeVRC.getPrevOutput(), cTransposeVRC.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { return CNeuronBaseOCL::updateInputWeights(cAssetTime[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPortfolioGenerator::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < cAssetTime.Size(); i++) cAssetTime[i].SetOpenCL(OpenCL); cTransposeVRC.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < cAssetTime.Size(); i++) if(!cAssetTime[i].Save(file_handle)) ReturnFalse; if(!cTransposeVRC.Save(file_handle)) ReturnFalse; if(!cSoftMax.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iAssets) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iTimePoints) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iAgents) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPortfolioGenerator::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < cAssetTime.Size(); i++) if(!LoadInsideLayer(file_handle, cAssetTime[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeVRC.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSoftMax.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iAssets = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iTimePoints = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iAgents = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMASAAT : public CNeuronPortfolioGenerator { protected: CNeuronTransposeOCL cTranspose; CNeuronPLRMultiAgentsOCL cPLR; CNeuronBaseOCL cConcat; CNeuronCrossSectionalAnalysis cCrossSectionalAnalysis; CNeuronTemporalAnalysis cTemporalAnalysis; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMASAAT(void) {}; ~CNeuronMASAAT(void) {}; //--- //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_cout, uint layers, vector &min_distance, uint projection, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronMASAAT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_cout, uint layers, vector &min_distance, uint projection, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronPortfolioGenerator::Init(numOutputs, myIndex, open_cl, window, units_cout / 3, window_key, (uint)min_distance.Size() + 1, projection, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, units_cout, window, optimization, iBatch)) ReturnFalse; if(!cPLR.Init(0, 1, OpenCL, window, units_cout, false, min_distance, optimization, iBatch)) ReturnFalse; if(!cConcat.Init(0, 2, OpenCL, cTranspose.Neurons() + cPLR.Neurons(), optimization, iBatch)) ReturnFalse; if(!cCrossSectionalAnalysis.Init(0, 3, OpenCL, units_cout, window_key, heads, heads / 2, window, layers, 1, iAgents, optimization, iBatch)) ReturnFalse; if(!cTemporalAnalysis.Init(0, 4, OpenCL, units_cout, window_key, heads, heads / 2, window, layers, 1, iAgents, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cPLR.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!Concat(cTranspose.getOutput(), cPLR.getOutput(), cConcat.getOutput(), cTranspose.Neurons(), cPLR.Neurons(), 1)) ReturnFalse; if(!cCrossSectionalAnalysis.FeedForward(cConcat.AsObject())) ReturnFalse; if(!cTemporalAnalysis.FeedForward(cConcat.AsObject())) ReturnFalse; //--- return CNeuronPortfolioGenerator::feedForward(cTemporalAnalysis.AsObject(), cCrossSectionalAnalysis.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronPortfolioGenerator::calcInputGradients(cTemporalAnalysis.AsObject(), cCrossSectionalAnalysis.getOutput(), cCrossSectionalAnalysis.getGradient(), (ENUM_ACTIVATION)cCrossSectionalAnalysis.Activation())) ReturnFalse; if(!cConcat.CalcHiddenGradients(cCrossSectionalAnalysis.AsObject())) ReturnFalse; CBufferFloat *grad = cConcat.getGradient(); if(!cConcat.SetGradient(cConcat.getPrevOutput(), false) || !cConcat.CalcHiddenGradients(cTemporalAnalysis.AsObject()) || !SumAndNormilize(grad, cConcat.getGradient(), grad, 1, 0, 0, 0, 0, 1) || !cConcat.SetGradient(grad, false)) ReturnFalse; //--- if(!DeConcat(cTranspose.getPrevOutput(), cPLR.getGradient(), cConcat.getGradient(), cTranspose.Neurons(), cPLR.Neurons(), 1)) ReturnFalse; if(cPLR.Activation() != None) if(!DeActivation(cPLR.getOutput(), cPLR.getGradient(), cPLR.getGradient(), cPLR.Activation())) ReturnFalse; //--- if(!cTranspose.CalcHiddenGradients(cPLR.AsObject()) || !SumAndNormilize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; if(cTranspose.Activation() != None) if(!DeActivation(cTranspose.getOutput(), cTranspose.getGradient(), cTranspose.getGradient(), cTranspose.Activation())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cCrossSectionalAnalysis.UpdateInputWeights(cConcat.AsObject())) ReturnFalse; if(!cTemporalAnalysis.UpdateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return CNeuronPortfolioGenerator::updateInputWeights(cTemporalAnalysis.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMASAAT::SetOpenCL(COpenCLMy * obj) { CNeuronPortfolioGenerator::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cPLR.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); cCrossSectionalAnalysis.SetOpenCL(OpenCL); cTemporalAnalysis.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::Save(const int file_handle) { if(!CNeuronPortfolioGenerator::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cPLR.Save(file_handle)) ReturnFalse; if(!cCrossSectionalAnalysis.Save(file_handle)) ReturnFalse; if(!cTemporalAnalysis.Save(file_handle)) ReturnFalse; return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMASAAT::Load(const int file_handle) { if(!CNeuronPortfolioGenerator::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPLR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossSectionalAnalysis.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalAnalysis.AsObject())) ReturnFalse; //--- if(!cConcat.Init(0, 2, OpenCL, cTranspose.Neurons() + cPLR.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHConvOCL : public CNeuronConvOCL { protected: uint iHeads; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMHConvOCL(void) : iHeads(1) {}; ~CNeuronMHConvOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronMHConvOCL; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronProofOCL::Init(numOutputs, myIndex, open_cl, window, step, units_count * window_out * variables, ADAM, batch)) ReturnFalse; //--- iWindowOut = window_out; iVariables = variables; iHeads = MathMax(MathMin(heads, window), 1); //--- const int window_h = int((iWindow + heads - 1) / heads); const int count = (int)((window_h + 1) * iWindowOut * iVariables); if(!WeightsConv) { WeightsConv = new CBufferFloat(); if(!WeightsConv) ReturnFalse; } if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(window_h + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!FirstMomentumConv) { FirstMomentumConv = new CBufferFloat(); if(!FirstMomentumConv) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(!SecondMomentumConv) { SecondMomentumConv = new CBufferFloat(); if(!SecondMomentumConv) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; DeleteObj(DeltaWeightsConv); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3]; global_work_size[0] = Output.Total() / (iWindowOut * iVariables); global_work_size[1] = iHeads; global_work_size[2] = iVariables; //--- ResetLastError(); const int kernel = def_k_FeedForwardMHConv; setBuffer(kernel, def_k_ffc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffс_window_out, (int)iWindowOut) setArgument(kernel, def_k_ffc_activation, (int)activation) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / iVariables; global_work_size[1] = iVariables; ResetLastError(); const int kernel = def_k_CalcHiddenGradientMHConv; setBuffer(kernel, def_k_chgc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_g, Gradient.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_chgc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgc_step, (int)iStep) setArgument(kernel, def_k_chgc_window_in, (int)iWindow) setArgument(kernel, def_k_chgc_window_out, (int)iWindowOut) setArgument(kernel, def_k_chgc_activation, (int)NeuronOCL.Activation()) setArgument(kernel, def_k_chgc_shift_out, (int)0) setArgument(kernel, def_k_chgc_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { WeightsConv.Total() }; float lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); ResetLastError(); const int kernel = def_k_UpdateWeightsMHConvAdam; setBuffer(kernel, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwca_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwca_l, lt) setArgument(kernel, def_k_uwca_b1, b1) setArgument(kernel, def_k_uwca_b2, b2) setArgument(kernel, def_k_uwca_window_in, (int)iWindow) setArgument(kernel, def_k_uwca_window_out, (int)iWindowOut) setArgument(kernel, def_k_uwca_step, (int)iStep) setArgument(kernel, def_k_uwca_heads, (int)iHeads) //--- kernelExecute(kernel, global_work_offset, global_work_size) t++; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!FileWriteInteger(file_handle, (int)iHeads)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHConvOCL::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHFeedForward : public CNeuronBaseOCL { protected: CNeuronMHConvOCL acConvolutions[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMHFeedForward(void) {}; ~CNeuronMHFeedForward(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const override { return defNeuronMHFeedForward; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true) { CNeuronBaseOCL *last = GetPointer(acConvolutions[acConvolutions.Size() - 1]); if(!last || !last.SetGradient(buffer, delete_prev) || !CNeuronBaseOCL::SetGradient(last.getGradient(), delete_prev)) ReturnFalse; return true; } virtual bool SetOutput(CBufferFloat *buffer, bool delete_prev = true) { CNeuronBaseOCL *last = GetPointer(acConvolutions[acConvolutions.Size() - 1]); if(!last || !last.SetOutput(buffer, delete_prev) || !CNeuronBaseOCL::SetOutput(last.getOutput(), delete_prev)) ReturnFalse; return true; } virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- if(!acConvolutions[0].Init(0, 0, OpenCL, window, window, window_out, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[0].SetActivationFunction(GELU); //--- if(!acConvolutions[1].Init(0, 1, OpenCL, window_out, window_out, window, units_count, variables, heads, optimization, iBatch)) ReturnFalse; acConvolutions[1].SetActivationFunction(None); //--- if(!SetGradient(acConvolutions[1].getGradient(), true)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::feedForward(CNeuronBaseOCL * NeuronOCL) { CObject *prev = NeuronOCL; for(uint i = 0; i < acConvolutions.Size(); i++) { if(!acConvolutions[i].FeedForward(prev)) ReturnFalse; prev = GetPointer(acConvolutions[i]); } //--- if(!SumAndNormilize(NeuronOCL.getOutput(), acConvolutions[acConvolutions.Size() - 1].getOutput(), Output, acConvolutions[0].GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; for(int i = (int)acConvolutions.Size() - 2; i >= 0; i--) { if(!acConvolutions[i].CalcHiddenGradients(acConvolutions[i + 1].AsObject())) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(acConvolutions[0].AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormilize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), Gradient, NeuronOCL.Activation()) || !SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), acConvolutions[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { for(int i = (int)acConvolutions.Size() - 1; i > 0; i--) { if(!acConvolutions[i].UpdateInputWeights(acConvolutions[i - 1].AsObject())) ReturnFalse; } if(!acConvolutions[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMHFeedForward* Source = source; if(acConvolutions.Size() != Source.acConvolutions.Size()) ReturnFalse; for(int i = 0; i < (int)acConvolutions.Size(); i++) if(!acConvolutions[i].WeightsUpdate(Source.acConvolutions[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!acConvolutions[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < acConvolutions.Size(); i++) if(!LoadInsideLayer(file_handle, acConvolutions[i].AsObject())) ReturnFalse; //--- if(!SetGradient(acConvolutions[acConvolutions.Size() - 1].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFeedForward::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < acConvolutions.Size(); i++) acConvolutions[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDMHAttention : public CNeuronRMAT { public: CNeuronDMHAttention(void) {}; ~CNeuronDMHAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) override; virtual int Type(void) override const { return defNeuronDMHAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronMHFeedForward *conv = NULL; for(uint i = 0; i < layers; i++) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, i * 2, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCrossDMHAttention : public CNeuronRMAT { protected: //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronCrossDMHAttention(void) {}; ~CNeuronCrossDMHAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCrossDMHAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronRelativeCrossAttention *cross = NULL; CNeuronMHFeedForward *conv = NULL; bool use_self = units_count > 0; int layer = 0; for(uint i = 0; i < layers; i++) { if(use_self) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, layer, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } layer++; } cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, layer, OpenCL, window, window_key, units_count, heads, window_cross, units_cross, optimization, iBatch) || !cLayers.Add(cross) ) { DeleteObj(cross); ReturnFalse; } layer++; conv = new CNeuronMHFeedForward(); if(!conv || !conv.Init(0, layer, OpenCL, window, 2 * window, units_count, 1, heads, optimization, iBatch) || !cLayers.Add(conv) ) { DeleteObj(conv); ReturnFalse; } layer++; } //--- SetOutput(conv.getOutput(), true); SetGradient(conv.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { CObject *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cLayers.Total(); i++) { current = cLayers[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(PrevOutput.Total() != SecondGradient.Total()) { PrevOutput.BufferFree(); if(!PrevOutput.BufferInit(SecondGradient.Total(), 0) || !PrevOutput.BufferCreate(OpenCL)) ReturnFalse; } if(!SecondGradient.Fill(0) || !PrevOutput.Fill(0)) ReturnFalse; //--- CNeuronBaseOCL *next = cLayers[-1]; CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 2; i >= 0; i--) { current = cLayers[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormilize(SecondGradient, PrevOutput, SecondGradient, 1, true, 0, 0, 0, 1)) ReturnFalse; next = current; } //--- if(!NeuronOCL.CalcHiddenGradients(next, SecondInput, PrevOutput, SecondActivation)) ReturnFalse; if(!SumAndNormilize(SecondGradient, PrevOutput, SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCrossDMHAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { CNeuronBaseOCL *current = NULL; for(int i = cLayers.Total() - 1; i > 0; i--) { current = cLayers[i]; if(!current || !current.UpdateInputWeights(cLayers[i - 1], SecondInput)) ReturnFalse; } //--- if(!((CNeuronBaseOCL*)cLayers[0]).UpdateInputWeights(NeuronOCL, SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLegendreWaveletsHL : public CNeuronConvOCL { protected: virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } ///< Method for updating weights.@param NeuronOCL Pointer to previos layer. public: CNeuronLegendreWaveletsHL(void) {}; ~CNeuronLegendreWaveletsHL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, uint filters, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) const { return defNeuronLegendreWaveletsHL; }///< Identificator of class.@return Type of class //--- virtual uint GetFilters(void) const {return (iWindowOut / 2); } virtual uint GetVariables(void) const {return (iVariables); } virtual uint GetUnits(void) const {return (Neurons() / (iVariables * iWindowOut)); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLegendreWaveletsHL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint units_count, uint filters, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, step, 2 * filters, units_count, variables, optimization_type, batch)) ReturnFalse; //--- WeightsConv.BufferInit(WeightsConv.Total(), 0); const uint shift_hight = (iWindow + 1) * filters; for(uint i = 0; i < iWindow; i++) { uint shift = i; float k = float(2.0 * i - 1.0) / iWindow; for(uint f = 1; f <= filters; f++) { float value = 0; switch(f) { case 1: value = k; break; case 2: value = (3 * k * k - 1) / 2; break; default: value = ((2 * f - 1) * k * WeightsConv.At(shift - (iWindow + 1)) - (f - 1) * WeightsConv.At(shift - 2 * (iWindow + 1))) / f; break; } for(uint v = 0; v < iVariables; v++) { uint shift_var = 2 * shift_hight * v; if(!WeightsConv.Update(shift + shift_var, value)) ReturnFalse; if(!WeightsConv.Update(shift + shift_var + shift_hight, MathPow(-1.0f, float(i))*value)) ReturnFalse; } shift += iWindow + 1; } } if(!!OpenCL) { if(!WeightsConv.BufferWrite()) ReturnFalse; uint global_work_size[] = {iWindowOut * iVariables}; uint global_work_offset[] = {0}; setBuffer(def_k_NormilizeWeights, def_k_norm_buffer, WeightsConv.GetIndex()); setArgument(def_k_NormilizeWeights, def_k_norm_dimension, (int)iWindow + 1); kernelExecute(def_k_NormilizeWeights, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDecouplingFlow : public CNeuronLegendreWaveletsHL { protected: CNeuronTransposeOCL cTranspose; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronDecouplingFlow(void) {}; ~CNeuronDecouplingFlow(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, uint filters, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) const { return defNeuronDecouplingFlow; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDecouplingFlow::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint units_count, uint filters, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint units_out = (units_count - window + step) / step; if(!CNeuronLegendreWaveletsHL::Init(numOutputs, myIndex, open_cl, window, step, units_out, filters, variables, optimization_type, batch)) ReturnFalse; if(!cTranspose.Init(0, 0, OpenCL, units_count, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDecouplingFlow::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronLegendreWaveletsHL::feedForward(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDecouplingFlow::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronLegendreWaveletsHL::calcInputGradients(cTranspose.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDecouplingFlow::SetOpenCL(COpenCLMy * obj) { CNeuronLegendreWaveletsHL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDecouplingFlow::Save(const int file_handle) { if(!CNeuronLegendreWaveletsHL::Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDecouplingFlow::Load(const int file_handle) { if(!CNeuronLegendreWaveletsHL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDilatedCasualConv : public CNeuronRMAT { public: CNeuronDilatedCasualConv(void) {}; ~CNeuronDilatedCasualConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint dimension, uint units_count, uint variables, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronDilatedCasualConv; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDilatedCasualConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint dimension, uint units_count, uint variables, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 1, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); uint units = units_count; CNeuronConvOCL *conv = NULL; CNeuronS3 *s3 = NULL; for(uint i = 0; i < layers; i++) { s3 = new CNeuronS3(); if(!s3 || !s3.Init(0, i * 2, OpenCL, dimension, dimension * units * variables, optimization, iBatch) || !cLayers.Add(s3)) { DeleteObj(s3); ReturnFalse; } s3.SetActivationFunction(None); //--- conv = new CNeuronConvOCL(); units = MathMax((units - window + step) / step, 1); if(!conv || !conv.Init(0, i * 2 + 1, OpenCL, window * dimension, step * dimension, dimension, units, variables, optimization, iBatch) || !cLayers.Add(conv)) { if(!!conv) DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(GELU); } //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, OpenCL, conv.Neurons(), optimization_type, batch)) ReturnFalse; //--- if(!SetGradient(conv.getGradient(), true) || !SetOutput(conv.getOutput(), true)) ReturnFalse; SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultitaskStockformer : public CNeuronBaseOCL { protected: CNeuronDecouplingFlow cDecouplingFlow; CNeuronBaseOCL cLowFreqSignal; CNeuronBaseOCL cHighFreqSignal; CNeuronRMAT cTemporalAttention; CNeuronDilatedCasualConv cDilatedCasualConvolution; CNeuronLearnabledPE cLowFreqPE; CNeuronLearnabledPE cHighFreqPE; CNeuronNAFS cLowFreqGraphAttention; CNeuronNAFS cHighFreqGraphAttention; CNeuronDMHAttention cLowFreqFusionDecoder; CNeuronCrossDMHAttention cLowHighFreqFusionDecoder; CNeuronBaseOCL cLowHigh; CNeuronConvOCL cProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultitaskStockformer(void) {}; ~CNeuronMultitaskStockformer(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint neurons_out, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultitaskStockformer; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint neurons_out, uint filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, neurons_out, optimization_type, batch)) ReturnFalse; //--- Decoupling Flow uint index = 0; uint wave_window = MathMin(24, units_count); if(!cDecouplingFlow.Init(0, index, OpenCL, wave_window, 2, units_count, filters, window, optimization, iBatch)) ReturnFalse; cDecouplingFlow.SetActivationFunction(None); //--- Dual-Frequency Spatiotemporal Encoder uint wave_units_out = cDecouplingFlow.GetUnits(); index++; if(!cLowFreqSignal.Init(0, index, OpenCL, cDecouplingFlow.Neurons() / 2, optimization, iBatch)) ReturnFalse; cLowFreqSignal.SetActivationFunction(None); index++; if(!cHighFreqSignal.Init(0, index, OpenCL, cDecouplingFlow.Neurons() / 2, optimization, iBatch)) ReturnFalse; cHighFreqSignal.SetActivationFunction(None); index++; if(!cTemporalAttention.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, heads, layers, optimization, iBatch)) ReturnFalse; cTemporalAttention.SetActivationFunction(None); index++; if(!cDilatedCasualConvolution.Init(0, index, OpenCL, 2, 2, filters, wave_units_out, window, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowFreqPE.Init(0, index, OpenCL, cTemporalAttention.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cHighFreqPE.Init(0, index, OpenCL, cDilatedCasualConvolution.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cLowFreqGraphAttention.Init(0, index, OpenCL, filters, 3, wave_units_out * window, optimization, iBatch)) ReturnFalse; index++; if(!cHighFreqGraphAttention.Init(0, index, OpenCL, filters, 3, cDilatedCasualConvolution.Neurons() / filters, optimization, iBatch)) ReturnFalse; index++; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, heads, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowHighFreqFusionDecoder.Init(0, index, OpenCL, filters, window_key, wave_units_out * window, filters, cDilatedCasualConvolution.Neurons() / filters, heads, layers, optimization, iBatch)) ReturnFalse; index++; if(!cLowHigh.Init(0, index, OpenCL, cLowFreqFusionDecoder.Neurons(), optimization, iBatch)) ReturnFalse; CBufferFloat *grad = cLowFreqFusionDecoder.getGradient(); if(!grad || !cLowHigh.SetGradient(grad, true) || !cLowHighFreqFusionDecoder.SetGradient(grad, true)) ReturnFalse; index++; if(!cProjection.Init(Neurons(), index, OpenCL, filters, filters, 3, wave_units_out, window, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Decoupling Flow if(!cDecouplingFlow.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cLowFreqSignal.getOutput(), cHighFreqSignal.getOutput(), cDecouplingFlow.getOutput(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetUnits()*cDecouplingFlow.GetVariables())) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cTemporalAttention.FeedForward(cLowFreqSignal.AsObject())) ReturnFalse; if(!cLowFreqPE.FeedForward(cTemporalAttention.AsObject())) ReturnFalse; if(!cLowFreqGraphAttention.FeedForward(cLowFreqPE.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cDilatedCasualConvolution.FeedForward(cHighFreqSignal.AsObject())) ReturnFalse; if(!cHighFreqPE.FeedForward(cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!cHighFreqGraphAttention.FeedForward(cHighFreqPE.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.FeedForward(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cLowHighFreqFusionDecoder.FeedForward(cLowFreqGraphAttention.AsObject(), cHighFreqGraphAttention.getOutput())) ReturnFalse; if(!SumAndNormilize(cLowFreqFusionDecoder.getOutput(), cLowHighFreqFusionDecoder.getOutput(), cLowHigh.getOutput(), cLowFreqFusionDecoder.GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- if(!cProjection.FeedForward(cLowHigh.AsObject())) ReturnFalse; //--- return CNeuronBaseOCL::feedForward(cProjection.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronBaseOCL::calcInputGradients(cProjection.AsObject())) ReturnFalse; if(!cLowHigh.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqGraphAttention.CalcHiddenGradients(cLowFreqFusionDecoder.AsObject())) ReturnFalse; CBufferFloat *grad = cLowFreqGraphAttention.getGradient(); if(!cLowFreqGraphAttention.SetGradient(cLowFreqGraphAttention.getPrevOutput(), false) || !cLowFreqGraphAttention.CalcHiddenGradients(cLowHighFreqFusionDecoder.AsObject(), cHighFreqGraphAttention.getOutput(), cHighFreqGraphAttention.getGradient(), (ENUM_ACTIVATION)cHighFreqGraphAttention.Activation()) || !SumAndNormilize(grad, cLowFreqGraphAttention.getGradient(), grad, 1, false, 0, 0, 0, 1) || !cLowFreqGraphAttention.SetGradient(grad, false)) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cLowFreqPE.CalcHiddenGradients(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cTemporalAttention.CalcHiddenGradients(cLowFreqPE.AsObject())) ReturnFalse; if(!cLowFreqSignal.CalcHiddenGradients(cTemporalAttention.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cHighFreqPE.CalcHiddenGradients(cHighFreqGraphAttention.AsObject())) ReturnFalse; if(!cDilatedCasualConvolution.CalcHiddenGradients(cHighFreqPE.AsObject())) ReturnFalse; if(!cHighFreqSignal.CalcHiddenGradients(cDilatedCasualConvolution.AsObject())) ReturnFalse; //--- Decoupling Flow if(!Concat(cLowFreqSignal.getGradient(), cHighFreqSignal.getGradient(), cDecouplingFlow.getGradient(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetFilters(), cDecouplingFlow.GetUnits()*cDecouplingFlow.GetVariables())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cDecouplingFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- if(!CNeuronBaseOCL::updateInputWeights(cProjection.AsObject())) ReturnFalse; if(!cProjection.UpdateInputWeights(cLowHigh.AsObject())) ReturnFalse; //--- Dual-Frequency Fusion Decoder if(!cLowFreqFusionDecoder.UpdateInputWeights(cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!cLowHighFreqFusionDecoder.UpdateInputWeights(cLowFreqGraphAttention.AsObject(), cHighFreqGraphAttention.getOutput())) ReturnFalse; //--- Dual-Frequency Spatiotemporal Encoder //--- Low Frequency Encoder if(!cLowFreqPE.UpdateInputWeights(cTemporalAttention.AsObject())) ReturnFalse; if(!cTemporalAttention.UpdateInputWeights(cLowFreqSignal.AsObject())) ReturnFalse; //--- High Frequency Encoder if(!cHighFreqPE.UpdateInputWeights(cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!cDilatedCasualConvolution.UpdateInputWeights(cHighFreqSignal.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cDecouplingFlow.Save(file_handle)) ReturnFalse; if(!cLowFreqSignal.Save(file_handle)) ReturnFalse; if(!cHighFreqSignal.Save(file_handle)) ReturnFalse; if(!cTemporalAttention.Save(file_handle)) ReturnFalse; if(!cDilatedCasualConvolution.Save(file_handle)) ReturnFalse; if(!cLowFreqPE.Save(file_handle)) ReturnFalse; if(!cHighFreqPE.Save(file_handle)) ReturnFalse; if(!cLowFreqGraphAttention.Save(file_handle)) ReturnFalse; if(!cHighFreqGraphAttention.Save(file_handle)) ReturnFalse; if(!cLowFreqFusionDecoder.Save(file_handle)) ReturnFalse; if(!cLowHighFreqFusionDecoder.Save(file_handle)) ReturnFalse; if(!cLowHigh.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultitaskStockformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cDecouplingFlow.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqSignal.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqSignal.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTemporalAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDilatedCasualConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqPE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqGraphAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighFreqGraphAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowFreqFusionDecoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowHighFreqFusionDecoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLowHigh.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; //--- CBufferFloat *grad = cLowFreqFusionDecoder.getGradient(); if(!grad || !cLowHigh.SetGradient(grad, true) || !cLowHighFreqFusionDecoder.SetGradient(grad, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultitaskStockformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDecouplingFlow.SetOpenCL(OpenCL); cLowFreqSignal.SetOpenCL(OpenCL); cHighFreqSignal.SetOpenCL(OpenCL); cTemporalAttention.SetOpenCL(OpenCL); cDilatedCasualConvolution.SetOpenCL(OpenCL); cLowFreqPE.SetOpenCL(OpenCL); cHighFreqPE.SetOpenCL(OpenCL); cLowFreqGraphAttention.SetOpenCL(OpenCL); cHighFreqGraphAttention.SetOpenCL(OpenCL); cLowFreqFusionDecoder.SetOpenCL(OpenCL); cLowHighFreqFusionDecoder.SetOpenCL(OpenCL); cLowHigh.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMemory : public CNeuronRelativeCrossAttention { protected: CNeuronLSTMOCL cLSTM; CNeuronMambaOCL cMamba; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return updateInputWeights(NeuronOCL); } public: CNeuronMemory(void) {}; ~CNeuronMemory(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronMemory; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, window, units_count, optimization_type, batch)) ReturnFalse; if(!cLSTM.Init(0, 0, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; if(!cMamba.Init(0, 1, OpenCL, iWindow, 2 * iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cLSTM.FeedForward(NeuronOCL)) ReturnFalse; if(!cMamba.FeedForward(NeuronOCL)) ReturnFalse; //--- return CNeuronRelativeCrossAttention::feedForward(cMamba.AsObject(), cLSTM.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronRelativeCrossAttention::calcInputGradients(cMamba.AsObject(), cLSTM.getOutput(), cLSTM.getGradient(), (ENUM_ACTIVATION)cLSTM.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMamba.AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cMamba.getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cLSTM.AsObject())) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false) || !SumAndNormilize(temp, cMamba.getPrevOutput(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronRelativeCrossAttention::updateInputWeights(cMamba.AsObject(), cLSTM.getOutput())) ReturnFalse; if(!cLSTM.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMamba.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMemory::SetOpenCL(COpenCLMy * obj) { CNeuronRelativeCrossAttention::SetOpenCL(obj); cLSTM.SetOpenCL(OpenCL); cMamba.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::Save(const int file_handle) { if(!CNeuronRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cLSTM.Save(file_handle)) ReturnFalse; if(!cMamba.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::Load(const int file_handle) { if(!CNeuronRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLSTM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMamba.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemory::Clear(void) { if(!CNeuronRelativeCrossAttention::Clear()) ReturnFalse; if(!cLSTM.Clear()) ReturnFalse; if(!cMamba.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFinMem : public CNeuronRelativeCrossAttention { protected: CNeuronTransposeOCL cTransposeState; CNeuronMemory cMemory[2]; CNeuronRelativeCrossAttention cCrossMemory; CNeuronRelativeCrossAttention cMemoryToAccount; CNeuronRelativeCrossAttention cActionToAccount; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronFinMem(void) {}; ~CNeuronFinMem(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint account_descr, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFinMem; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint account_descr, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, nactions / 2, window_key, 2, heads, window, units_count, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTransposeState.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; index++; if(!cMemory[0].Init(0, index, OpenCL, window, window_key, units_count, heads, optimization, iBatch)) ReturnFalse; index++; if(!cMemory[1].Init(0, index, OpenCL, units_count, window_key, window, heads, optimization, iBatch)) ReturnFalse; index++; if(!cCrossMemory.Init(0, index, OpenCL, window, window_key, units_count, heads, units_count, window, optimization, iBatch)) ReturnFalse; index++; if(!cMemoryToAccount.Init(0, index, OpenCL, window, window_key, units_count, heads, account_descr, 1, optimization, iBatch)) ReturnFalse; index++; if(!cActionToAccount.Init(0, index, OpenCL, nactions / 2, window_key, 2, heads, account_descr, 1, optimization, iBatch)) ReturnFalse; //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cTransposeState.FeedForward(NeuronOCL)) ReturnFalse; if(!cMemory[0].FeedForward(NeuronOCL) || !cMemory[1].FeedForward(cTransposeState.AsObject())) ReturnFalse; if(!cCrossMemory.FeedForward(cMemory[0].AsObject(), cMemory[1].getOutput())) ReturnFalse; if(!cMemoryToAccount.FeedForward(cCrossMemory.AsObject(), SecondInput)) ReturnFalse; if(!cActionToAccount.FeedForward(this.AsObject(), SecondInput)) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!CNeuronRelativeCrossAttention::feedForward(cActionToAccount.AsObject(), cMemoryToAccount.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronRelativeCrossAttention::calcInputGradients(cActionToAccount.AsObject(), cMemoryToAccount.getOutput(), cMemoryToAccount.getGradient(), (ENUM_ACTIVATION)cMemoryToAccount.Activation())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; CBufferFloat *temp = Gradient; if(!SetGradient(cMemoryToAccount.getPrevOutput(), false)) ReturnFalse; if(!calcHiddenGradients(cActionToAccount.AsObject(), SecondInput, SecondGradient, SecondActivation)) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; Gradient = temp; if(!cCrossMemory.CalcHiddenGradients(cMemoryToAccount.AsObject(), SecondInput, cMemoryToAccount.getPrevOutput(), SecondActivation)) ReturnFalse; if(!SumAndNormilize(SecondGradient, cMemoryToAccount.getPrevOutput(), SecondGradient, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cMemory[0].CalcHiddenGradients(cCrossMemory.AsObject(), cMemory[1].getOutput(), cMemory[1].getGradient(), (ENUM_ACTIVATION)cMemory[1].Activation())) ReturnFalse; if(!cTransposeState.CalcHiddenGradients(cMemory[1].AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMemory[0].AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cTransposeState.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cTransposeState.AsObject()) || !NeuronOCL.SetGradient(temp, false) || !SumAndNormilize(temp, cTransposeState.getPrevOutput(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; //--- if(!CNeuronRelativeCrossAttention::updateInputWeights(cActionToAccount.AsObject(), cMemoryToAccount.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!cActionToAccount.UpdateInputWeights(this.AsObject(), SecondInput)) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!cMemoryToAccount.UpdateInputWeights(cCrossMemory.AsObject(), SecondInput)) ReturnFalse; if(!cCrossMemory.UpdateInputWeights(cMemory[0].AsObject(), cMemory[1].getOutput())) ReturnFalse; if(!cMemory[1].UpdateInputWeights(cTransposeState.AsObject())) ReturnFalse; if(!cMemory[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::Clear(void) { if(!Output || !Output.Fill(0)) ReturnFalse; for(uint i = 0; i < cMemory.Size(); i++) if(!cMemory[i].Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFinMem::SetOpenCL(COpenCLMy * obj) { CNeuronRelativeCrossAttention::SetOpenCL(obj); cTransposeState.SetOpenCL(OpenCL); for(uint i = 0; i < cMemory.Size(); i++) cMemory[i].SetOpenCL(OpenCL); cCrossMemory.SetOpenCL(OpenCL); cMemoryToAccount.SetOpenCL(OpenCL); cActionToAccount.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::Save(const int file_handle) { if(!CNeuronRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cTransposeState.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cMemory.Size(); i++) if(!cMemory[i].Save(file_handle)) ReturnFalse; if(!cCrossMemory.Save(file_handle)) ReturnFalse; if(!cMemoryToAccount.Save(file_handle)) ReturnFalse; if(!cActionToAccount.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinMem::Load(const int file_handle) { if(!CNeuronRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeState.AsObject())) ReturnFalse; for(uint i = 0; i < cMemory.Size(); i++) if(!LoadInsideLayer(file_handle, cMemory[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossMemory.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMemoryToAccount.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionToAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLowLevelReflection : public CNeuronMemory { protected: CNeuronLSTMOCL cChangeLSTM; CNeuronMambaOCL cChangeMamba; CNeuronRelativeCrossAttention cCrossAttention[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronLowLevelReflection(void) {}; ~CNeuronLowLevelReflection(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronLowLevelReflection; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMemory::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cChangeLSTM.Init(0, index, OpenCL, window, units_count, optimization, iBatch)) ReturnFalse; index++; if(!cChangeMamba.Init(0, index, OpenCL, window, 2 * window, units_count, optimization, iBatch)) ReturnFalse; for(int i = 0; i < 2; i++) { index++; if(!cCrossAttention[i].Init(0, index, OpenCL, window, window_key, units_count, heads, window, units_count, optimization, iBatch)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cChangeLSTM.FeedForward(NeuronOCL)) ReturnFalse; if(!cChangeMamba.FeedForward(NeuronOCL)) ReturnFalse; if(!cCrossAttention[0].FeedForward(NeuronOCL, cChangeLSTM.getOutput())) ReturnFalse; if(!cCrossAttention[1].FeedForward(cCrossAttention[0].AsObject(), cChangeMamba.getOutput())) ReturnFalse; //--- return CNeuronMemory::feedForward(cCrossAttention[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMemory::calcInputGradients(cCrossAttention[1].AsObject())) ReturnFalse; if(!cCrossAttention[0].CalcHiddenGradients(cCrossAttention[1].AsObject(), cChangeMamba.getOutput(), cChangeMamba.getGradient(), (ENUM_ACTIVATION)cChangeMamba.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cCrossAttention[0].AsObject(), cChangeLSTM.getOutput(), cChangeLSTM.getGradient(), (ENUM_ACTIVATION)cChangeLSTM.Activation())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cChangeMamba.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cChangeMamba.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cChangeLSTM.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), temp, temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMemory::updateInputWeights(cCrossAttention[1].AsObject())) ReturnFalse; if(!cCrossAttention[1].UpdateInputWeights(cCrossAttention[0].AsObject(), cChangeMamba.getOutput())) ReturnFalse; if(!cCrossAttention[0].UpdateInputWeights(NeuronOCL, cChangeLSTM.getOutput())) ReturnFalse; if(!cChangeMamba.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cChangeLSTM.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::Save(const int file_handle) { if(!CNeuronMemory::Save(file_handle)) ReturnFalse; if(!cChangeLSTM.Save(file_handle)) ReturnFalse; if(!cChangeMamba.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cCrossAttention.Size(); i++) if(!cCrossAttention[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::Load(const int file_handle) { if(!CNeuronMemory::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cChangeLSTM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cChangeMamba.AsObject())) ReturnFalse; for(uint i = 0; i < cCrossAttention.Size(); i++) if(!LoadInsideLayer(file_handle, cCrossAttention[i].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLowLevelReflection::SetOpenCL(COpenCLMy * obj) { CNeuronMemory::SetOpenCL(obj); cChangeLSTM.SetOpenCL(OpenCL); cChangeMamba.SetOpenCL(OpenCL); for(uint i = 0; i < cCrossAttention.Size(); i++) cCrossAttention[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLowLevelReflection::Clear(void) { if(!CNeuronMemory::Clear()) ReturnFalse; if(!cChangeLSTM.Clear()) ReturnFalse; if(!cChangeMamba.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHighLevelReflection : public CNeuronMemory { protected: CNeuronBaseOCL cAccount; CNeuronLSTMOCL cHistoryAccount; CNeuronRelativeCrossAttention cActionReason; CNeuronLSTMOCL cHistoryActions; CNeuronRelativeCrossAttention cActionResult; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronHighLevelReflection(void) {}; ~CNeuronHighLevelReflection(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint desc_account, uint actions_state, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronHighLevelReflection; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint desc_account, uint actions_state, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMemory::Init(numOutputs, myIndex, open_cl, 3, window_key, actions_state / 3, heads, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cAccount.Init(0, index, OpenCL, desc_account, optimization, iBatch)) ReturnFalse; index++; if(!cHistoryAccount.Init(0, index, OpenCL, desc_account, 1, optimization, iBatch)) ReturnFalse; index++; if(!cActionReason.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, window, units_count, optimization, iBatch)) ReturnFalse; index++; if(!cHistoryActions.Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; index++; if(!cActionResult.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, desc_account, 1, optimization, iBatch)) ReturnFalse;; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; //--- if(cAccount.getOutput() != SecondInput) { if(cAccount.Neurons() != SecondInput.Total()) if(!cAccount.Init(0, 0, OpenCL, SecondInput.Total(), optimization, iBatch)) ReturnFalse; if(!cAccount.SetOutput(SecondInput, true)) ReturnFalse; } //--- if(!cHistoryAccount.FeedForward(cAccount.AsObject())) ReturnFalse; if(!cActionReason.FeedForward(this.AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!cHistoryActions.FeedForward(cActionReason.AsObject())) ReturnFalse; if(!cActionResult.FeedForward(cHistoryActions.AsObject(), cHistoryAccount.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- return CNeuronMemory::feedForward(cActionResult.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- if(cAccount.getGradient() != SecondGradient) if(!cAccount.SetGradient(SecondGradient, true)) ReturnFalse; cAccount.SetActivationFunction(SecondActivation); //--- if(!CNeuronMemory::calcInputGradients(cActionResult.AsObject())) ReturnFalse; if(!cHistoryActions.CalcHiddenGradients(cActionResult.AsObject(), cHistoryAccount.getOutput(), cHistoryAccount.getGradient(), (ENUM_ACTIVATION)cHistoryAccount.Activation())) ReturnFalse; if(!cActionReason.CalcHiddenGradients(cHistoryActions.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; CBufferFloat *temp = Gradient; Gradient = cActionReason.getPrevOutput(); if(!calcHiddenGradients(cActionReason.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; //--- Gradient = temp; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- if(!cAccount.CalcHiddenGradients(cHistoryAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!CNeuronMemory::updateInputWeights(cActionResult.AsObject())) ReturnFalse; if(!cActionResult.UpdateInputWeights(cHistoryActions.AsObject(), cHistoryAccount.getOutput())) ReturnFalse; if(!cHistoryActions.UpdateInputWeights(cActionReason.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!cActionReason.UpdateInputWeights(this.AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!cHistoryAccount.UpdateInputWeights(cAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::Save(const int file_handle) { if(!CNeuronMemory::Save(file_handle)) ReturnFalse; if(!cAccount.Save(file_handle)) ReturnFalse; if(!cHistoryAccount.Save(file_handle)) ReturnFalse; if(!cActionReason.Save(file_handle)) ReturnFalse; if(!cHistoryActions.Save(file_handle)) ReturnFalse; if(!cActionResult.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::Load(const int file_handle) { if(!CNeuronMemory::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cAccount.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistoryAccount.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionReason.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistoryActions.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionResult.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHighLevelReflection::SetOpenCL(COpenCLMy * obj) { CNeuronMemory::SetOpenCL(obj); cAccount.SetOpenCL(OpenCL); cHistoryAccount.SetOpenCL(OpenCL); cActionReason.SetOpenCL(OpenCL); cHistoryActions.SetOpenCL(OpenCL); cActionResult.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHighLevelReflection::Clear(void) { if(!CNeuronMemory::Clear()) ReturnFalse; if(!cAccount.Clear()) ReturnFalse; if(!cHistoryAccount.Clear()) ReturnFalse; if(!cActionReason.Clear()) ReturnFalse; if(!cHistoryActions.Clear()) ReturnFalse; if(!cActionResult.Clear()) ReturnFalse;; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMoreLessEqual : public CNeuronBaseOCL { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override {return true; } public: CNeuronMoreLessEqual(void) {}; ~CNeuronMoreLessEqual(void) {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoreLessEqual::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[1] = { 0 }; uint global_work_size[1] = { Neurons() }; ResetLastError(); const int kernel = def_k_MoreLessEqual; setBuffer(kernel, def_k_mle_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mle_outputs, getOutputIndex()) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoreLessEqual::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !NeuronOCL.getGradient()) ReturnFalse; return NeuronOCL.getGradient().Fill(0); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFinAgent : public CNeuronRelativeCrossAttention { protected: CNeuronTransposeOCL cTransposeState; CNeuronLowLevelReflection cLowLevelReflection[2]; CNeuronHighLevelReflection cHighLevelReflection; CNeuronMoreLessEqual cTools; CNeuronPSformer cMarketIntelligence; CNeuronMemory cMarketMemory; CNeuronRelativeCrossAttention cCrossLowLevel; CNeuronRelativeCrossAttention cMarketToLowLevel; CNeuronRelativeCrossAttention cMarketToTools; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronFinAgent(void) {}; ~CNeuronFinAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint account_descr, uint nactions, uint segments, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFinAgent; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint account_descr, uint nactions, uint segments, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, 3, window_key, nactions / 3, heads, window, units_count, optimization_type, batch)) ReturnFalse; int index = 0; if(!cMarketIntelligence.Init(0, index, OpenCL, window, units_count, segments, 0.2f, optimization, iBatch)) ReturnFalse; index++; if(!cMarketMemory.Init(0, index, OpenCL, window, window_key, units_count, heads, optimization, iBatch)) ReturnFalse; index++; if(!cTransposeState.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; index++; if(!cLowLevelReflection[0].Init(0, index, OpenCL, window, window_key, units_count, heads, optimization, iBatch)) ReturnFalse; index++; if(!cLowLevelReflection[1].Init(0, index, OpenCL, units_count, window_key, window, heads, optimization, iBatch)) ReturnFalse; index++; if(!cHighLevelReflection.Init(0, index, OpenCL, window, window_key, units_count, heads, account_descr, nactions, optimization, iBatch)) ReturnFalse; index++; if(!cTools.Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cTools.SetActivationFunction(None); index++; if(!cCrossLowLevel.Init(0, index, OpenCL, window, window_key, units_count, heads, units_count, window, optimization, iBatch)) ReturnFalse; index++; if(!cMarketToLowLevel.Init(0, index, OpenCL, window, window_key, units_count, heads, window, units_count, optimization, iBatch)) ReturnFalse; index++; if(!cMarketToTools.Init(0, index, OpenCL, window, window_key, units_count, heads, window, units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cMarketIntelligence.FeedForward(NeuronOCL)) ReturnFalse; if(!cMarketMemory.FeedForward(cMarketIntelligence.AsObject())) ReturnFalse; if(!cTransposeState.FeedForward(cMarketIntelligence.AsObject())) ReturnFalse; if(!cLowLevelReflection[0].FeedForward(cMarketIntelligence.AsObject())) ReturnFalse; if(!cLowLevelReflection[1].FeedForward(cTransposeState.AsObject())) ReturnFalse; if(!cHighLevelReflection.FeedForward(cMarketIntelligence.AsObject(), SecondInput)) ReturnFalse; if(!cTools.FeedForward(NeuronOCL)) ReturnFalse; if(!cCrossLowLevel.FeedForward(cLowLevelReflection[0].AsObject(), cLowLevelReflection[1].getOutput())) ReturnFalse; if(!cMarketToLowLevel.FeedForward(cMarketMemory.AsObject(), cCrossLowLevel.getOutput())) ReturnFalse; if(!cMarketToTools.FeedForward(cMarketToLowLevel.AsObject(), cTools.getOutput())) ReturnFalse; //--- return CNeuronRelativeCrossAttention::feedForward(cHighLevelReflection.AsObject(), cMarketToTools.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- if(!CNeuronRelativeCrossAttention::calcInputGradients(cHighLevelReflection.AsObject(), cMarketToTools.getOutput(), cMarketToTools.getGradient(), (ENUM_ACTIVATION)cMarketToTools.Activation())) ReturnFalse; //--- if(!cMarketToLowLevel.CalcHiddenGradients(cMarketToTools.AsObject(), cTools.getOutput(), cTools.getGradient(), (ENUM_ACTIVATION)cTools.Activation())) ReturnFalse; //--- if(!cMarketMemory.CalcHiddenGradients(cMarketToLowLevel.AsObject(), cCrossLowLevel.getOutput(), cCrossLowLevel.getGradient(), (ENUM_ACTIVATION)cCrossLowLevel.Activation())) ReturnFalse; //--- if(!cLowLevelReflection[0].CalcHiddenGradients(cCrossLowLevel.AsObject(), cLowLevelReflection[1].getOutput(), cLowLevelReflection[1].getGradient(), (ENUM_ACTIVATION)cLowLevelReflection[1].Activation())) ReturnFalse; if(!cTransposeState.CalcHiddenGradients(cLowLevelReflection[1].AsObject())) ReturnFalse; //--- if(!((CNeuronBaseOCL*)cMarketIntelligence.AsObject()).CalcHiddenGradients(cMarketMemory.AsObject())) ReturnFalse; CBufferFloat *temp = cMarketIntelligence.getGradient(); if(!cMarketIntelligence.SetGradient(cMarketIntelligence.getPrevOutput(), false) || !((CNeuronBaseOCL*)cMarketIntelligence.AsObject()).CalcHiddenGradients(cHighLevelReflection.AsObject(), SecondInput, SecondGradient, SecondActivation) || !SumAndNormilize(temp, cMarketIntelligence.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!((CNeuronBaseOCL*)cMarketIntelligence.AsObject()).CalcHiddenGradients(cLowLevelReflection[0].AsObject()) || !SumAndNormilize(temp, cMarketIntelligence.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!((CNeuronBaseOCL*)cMarketIntelligence.AsObject()).CalcHiddenGradients(cTransposeState.AsObject()) || !SumAndNormilize(temp, cMarketIntelligence.getGradient(), temp, 1, false, 0, 0, 0, 1) || !cMarketIntelligence.SetGradient(temp, false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cMarketIntelligence.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- if(!CNeuronRelativeCrossAttention::updateInputWeights(cHighLevelReflection.AsObject(), cMarketToTools.getOutput())) ReturnFalse; //--- if(!cMarketToTools.UpdateInputWeights(cMarketToLowLevel.AsObject(), cTools.getOutput())) ReturnFalse; //--- if(!cMarketToLowLevel.UpdateInputWeights(cMarketMemory.AsObject(), cCrossLowLevel.getOutput())) ReturnFalse; if(!cMarketMemory.UpdateInputWeights(cMarketIntelligence.AsObject())) ReturnFalse; //--- if(!cCrossLowLevel.UpdateInputWeights(cLowLevelReflection[0].AsObject(), cLowLevelReflection[1].getOutput())) ReturnFalse; //--- if(!cLowLevelReflection[0].UpdateInputWeights(cMarketIntelligence.AsObject())) ReturnFalse; if(!cLowLevelReflection[1].UpdateInputWeights(cTransposeState.AsObject())) ReturnFalse; //--- if(!cHighLevelReflection.UpdateInputWeights(cMarketIntelligence.AsObject(), SecondInput)) ReturnFalse; //--- if(!cMarketIntelligence.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::Save(const int file_handle) { if(!CNeuronRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cMarketIntelligence.Save(file_handle)) ReturnFalse; if(!cMarketMemory.Save(file_handle)) ReturnFalse; if(!cTransposeState.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cLowLevelReflection.Size(); i++) if(!cLowLevelReflection[i].Save(file_handle)) ReturnFalse; if(!cHighLevelReflection.Save(file_handle)) ReturnFalse; if(!cTools.Save(file_handle)) ReturnFalse; if(!cCrossLowLevel.Save(file_handle)) ReturnFalse; if(!cMarketToLowLevel.Save(file_handle)) ReturnFalse; if(!cMarketToTools.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::Load(const int file_handle) { if(!CNeuronRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMarketIntelligence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMarketMemory.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeState.AsObject())) ReturnFalse; for(uint i = 0; i < cLowLevelReflection.Size(); i++) if(!LoadInsideLayer(file_handle, cLowLevelReflection[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHighLevelReflection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTools.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossLowLevel.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMarketToLowLevel.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMarketToTools.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFinAgent::SetOpenCL(COpenCLMy * obj) { CNeuronRelativeCrossAttention::SetOpenCL(obj); cMarketIntelligence.SetOpenCL(OpenCL); cMarketMemory.SetOpenCL(OpenCL); cTransposeState.SetOpenCL(OpenCL); for(uint i = 0; i < cLowLevelReflection.Size(); i++) cLowLevelReflection[i].SetOpenCL(OpenCL); cHighLevelReflection.SetOpenCL(OpenCL); cTools.SetOpenCL(OpenCL); cCrossLowLevel.SetOpenCL(OpenCL); cMarketToLowLevel.SetOpenCL(OpenCL); cMarketToTools.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinAgent::Clear(void) { if(!CNeuronRelativeCrossAttention::Clear()) ReturnFalse; if(!cMarketIntelligence.Clear()) ReturnFalse; if(!cMarketMemory.Clear()) ReturnFalse; if(!cTransposeState.Clear()) ReturnFalse; for(uint i = 0; i < cLowLevelReflection.Size(); i++) if(!cLowLevelReflection[i].Clear()) ReturnFalse; if(!cHighLevelReflection.Clear()) ReturnFalse; if(!cTools.Clear()) ReturnFalse; if(!cCrossLowLevel.Clear()) ReturnFalse; if(!cMarketToLowLevel.Clear()) ReturnFalse; if(!cMarketToTools.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMemoryDistil : public CNeuronMemory { protected: CNeuronBaseOCL cConcatenated; CNeuronTransposeOCL cTransposeConc; CNeuronConvOCL cConvolution; CNeuronEmbeddingOCL cDistilMemory; CNeuronRelativeCrossAttention cCrossAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMemoryDistil(void) {}; ~CNeuronMemoryDistil(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMemoryDistil; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, window, 2 * stack_size, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cLSTM.Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; index++; if(!cMamba.Init(0, index, OpenCL, iWindow, 2 * iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- index++; if(!cConcatenated.Init(0, index, OpenCL, 2 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); index++; if(!cTransposeConc.Init(0, index, OpenCL, iUnits, 2 * iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cConvolution.Init(0, index, OpenCL, iUnits, iUnits, iWindowKey, iWindow, 1, optimization, iBatch)) ReturnFalse; cConvolution.SetActivationFunction(GELU); index++; uint windows[] = {iWindowKey * iWindow, iWindowKey * iWindow}; if(!cDistilMemory.Init(0, index, OpenCL, iUnitsKV / 2, iWindow, windows)) ReturnFalse; //--- index++; if(!cCrossAttention.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, iWindow, 2 * iUnits, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cLSTM.FeedForward(NeuronOCL)) ReturnFalse; if(!cMamba.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Concat(cLSTM.getOutput(), cMamba.getOutput(), cConcatenated.getOutput(), iWindow, iWindow, iUnits)) ReturnFalse; if(!cTransposeConc.FeedForward(cConcatenated.AsObject())) ReturnFalse; if(!cConvolution.FeedForward(cTransposeConc.AsObject())) ReturnFalse; if(!cDistilMemory.FeedForward(cConvolution.AsObject())) ReturnFalse; //--- if(!cCrossAttention.FeedForward(NeuronOCL, cConcatenated.getOutput())) ReturnFalse; //--- return CNeuronRelativeCrossAttention::feedForward(cCrossAttention.AsObject(), cDistilMemory.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronRelativeCrossAttention::calcInputGradients(cCrossAttention.AsObject(), cDistilMemory.getOutput(), cDistilMemory.getGradient(), (ENUM_ACTIVATION)cDistilMemory.Activation())) ReturnFalse; //--- if(!cConvolution.CalcHiddenGradients(cDistilMemory.AsObject())) ReturnFalse; if(!cTransposeConc.CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cTransposeConc.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cCrossAttention.AsObject(), cConcatenated.getOutput(), cConcatenated.getPrevOutput(), (ENUM_ACTIVATION)cConcatenated.Activation())) ReturnFalse; if(!SumAndNormilize(cConcatenated.getGradient(), cConcatenated.getPrevOutput(), cConcatenated.getGradient(), iWindow, false, 0, 0, 0, 1) || !DeConcat(cLSTM.getGradient(), cMamba.getGradient(), cConcatenated.getGradient(), iWindow, iWindow, iUnits)) ReturnFalse; //--- if(cLSTM.Activation() != None) if(!DeActivation(cLSTM.getOutput(), cLSTM.getGradient(), cLSTM.getGradient(), cLSTM.Activation())) ReturnFalse; if(cMamba.Activation() != None) if(!DeActivation(cMamba.getOutput(), cMamba.getGradient(), cMamba.getGradient(), cMamba.Activation())) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cConcatenated.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cLSTM.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMamba.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronRelativeCrossAttention::updateInputWeights(cCrossAttention.AsObject(), cDistilMemory.getOutput())) ReturnFalse; if(!cCrossAttention.UpdateInputWeights(NeuronOCL, cConcatenated.getOutput())) ReturnFalse; //--- if(!cDistilMemory.UpdateInputWeights(cConvolution.AsObject())) ReturnFalse; if(!cConvolution.UpdateInputWeights(cTransposeConc.AsObject())) ReturnFalse; //--- if(!cLSTM.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMamba.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMemoryDistil::SetOpenCL(COpenCLMy * obj) { CNeuronMemory::SetOpenCL(obj); cConcatenated.SetOpenCL(OpenCL); cTransposeConc.SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cDistilMemory.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::Clear(void) { if(!CNeuronMemory::Clear()) ReturnFalse; if(!cConcatenated.Clear()) ReturnFalse; if(!cTransposeConc.Clear()) ReturnFalse; if(!cConvolution.Clear()) ReturnFalse; if(!cDistilMemory.Clear()) ReturnFalse; if(!cCrossAttention.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::Save(const int file_handle) { if(!CNeuronMemory::Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cTransposeConc.Save(file_handle)) ReturnFalse; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cDistilMemory.Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMemoryDistil::Load(const int file_handle) { if(!CNeuronMemory::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeConc.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDistilMemory.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFinConAgent : public CNeuronRelativeCrossAttention { protected: CNeuronMemoryDistil cStatesMemory; CNeuronMemoryDistil cActionsMemory; CNeuronBaseOCL caRole[2]; CNeuronRelativeCrossAttention cStateToRole; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return updateInputWeights(NeuronOCL); } public: CNeuronFinConAgent(void) {}; ~CNeuronFinConAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFinConAgent; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, 3, window_key, action_space / 3, heads, window, units_count, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cStatesMemory.Init(0, index, OpenCL, window, iWindowKey, iUnitsKV, iHeads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cActionsMemory.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!caRole[0].Init(10 * iWindow, index, OpenCL, 1, optimization, iBatch)) ReturnFalse; CBufferFloat *out = caRole[0].getOutput(); if(!out || !out.Fill(1)) ReturnFalse; index++; if(!caRole[1].Init(0, index, OpenCL, 10 * iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cStateToRole.Init(0, index, OpenCL, window, iWindowKey, iUnitsKV, iHeads, iWindow, 10, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::feedForward(CNeuronBaseOCL * NeuronOCL) { if(bTrain && !caRole[1].FeedForward(caRole[0].AsObject())) ReturnFalse; //--- if(!cStateToRole.FeedForward(NeuronOCL, caRole[1].getOutput())) ReturnFalse; if(!cStatesMemory.FeedForward(cStateToRole.AsObject())) ReturnFalse; //--- if(!cActionsMemory.FeedForward(this.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- return CNeuronRelativeCrossAttention::feedForward(cActionsMemory.AsObject(), cStatesMemory.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronRelativeCrossAttention::calcInputGradients(cActionsMemory.AsObject(), cStatesMemory.getOutput(), cStatesMemory.getGradient(), (ENUM_ACTIVATION)cStatesMemory.Activation())) ReturnFalse; //--- CBufferFloat *temp = Gradient; if(!SwapBuffers(Output, PrevOutput) || !SetGradient(cActionsMemory.getPrevOutput(), false)) ReturnFalse; if(!calcHiddenGradients(cActionsMemory.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; Gradient = temp; //--- if(!cStateToRole.CalcHiddenGradients(cStatesMemory.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cStateToRole.AsObject(), caRole[1].getOutput(), caRole[1].getGradient(), (ENUM_ACTIVATION)caRole[1].Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronRelativeCrossAttention::updateInputWeights(cActionsMemory.AsObject(), cStatesMemory.getOutput())) ReturnFalse; //--- if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; if(!cActionsMemory.UpdateInputWeights(this.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- if(!cStatesMemory.UpdateInputWeights(cStateToRole.AsObject())) ReturnFalse; if(!cStateToRole.UpdateInputWeights(NeuronOCL, caRole[1].getOutput())) ReturnFalse; //--- if(!caRole[1].UpdateInputWeights(caRole[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFinConAgent::SetOpenCL(COpenCLMy * obj) { CNeuronRelativeCrossAttention::SetOpenCL(obj); cStatesMemory.SetOpenCL(OpenCL); cActionsMemory.SetOpenCL(OpenCL); caRole[0].SetOpenCL(OpenCL); caRole[1].SetOpenCL(OpenCL); cStateToRole.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::Clear(void) { if(!CNeuronRelativeCrossAttention::Clear()) ReturnFalse; if(!cStatesMemory.Clear()) ReturnFalse; if(!cActionsMemory.Clear()) ReturnFalse; if(!caRole[0].Clear()) ReturnFalse; if(!caRole[1].Clear()) ReturnFalse; if(!cStateToRole.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::Save(const int file_handle) { if(!CNeuronRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cStatesMemory.Save(file_handle)) ReturnFalse; if(!cActionsMemory.Save(file_handle)) ReturnFalse; if(!caRole[0].Save(file_handle)) ReturnFalse; if(!caRole[1].Save(file_handle)) ReturnFalse; if(!cStateToRole.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConAgent::Load(const int file_handle) { if(!CNeuronRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStatesMemory.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionsMemory.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caRole[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, caRole[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToRole.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFinConManager : public CNeuronFinConAgent { protected: CNeuronTransposeOCL cTransposeState; CNeuronFinConAgent caAgents[3]; CNeuronFinConAgent caTrAgents[3]; CNeuronFinConAgent cRiskAgent; CNeuronBaseOCL cConcatenatedAgents; CNeuronBaseOCL cAccount; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronFinConManager(void) {}; ~CNeuronFinConManager(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint account_descr, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFinConManager; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; virtual void TrainMode(bool flag); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint account_descr, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronFinConAgent::Init(numOutputs, myIndex, open_cl, action_space, window_key, caAgents.Size() + caTrAgents.Size() + 1, heads, stack_size, action_space, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTransposeState.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; //--- for(uint i = 0; i < caAgents.Size(); i++) { index++; if(!caAgents[i].Init(0, index, OpenCL, window, iWindowKey, units_count, iHeads, stack_size, action_space, optimization, iBatch)) ReturnFalse; } for(uint i = 0; i < caTrAgents.Size(); i++) { index++; if(!caTrAgents[i].Init(0, index, OpenCL, units_count, iWindowKey, window, iHeads, stack_size, action_space, optimization, iBatch)) ReturnFalse; } index++; if(!cRiskAgent.Init(0, index, OpenCL, account_descr, iWindowKey, 1, iHeads, stack_size, action_space, optimization, iBatch)) ReturnFalse; index++; if(!cConcatenatedAgents.Init(0, index, OpenCL, caAgents.Size()*caAgents[0].Neurons() + caTrAgents.Size()*caTrAgents[0].Neurons() + cRiskAgent.Neurons(), optimization, iBatch)) ReturnFalse; //--- index++; if(!cAccount.Init(0, index, OpenCL, account_descr, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(cAccount.getOutput() != SecondInput) { if(!cAccount.SetOutput(SecondInput, true)) ReturnFalse; } if(!cTransposeState.FeedForward(NeuronOCL)) ReturnFalse; //--- Agents for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].FeedForward(NeuronOCL)) ReturnFalse; for(uint i = 0; i < caTrAgents.Size(); i++) if(!caTrAgents[i].FeedForward(cTransposeState.AsObject())) ReturnFalse; if(!cRiskAgent.FeedForward(cAccount.AsObject())) ReturnFalse; //--- Concatenate if(!Concat(caAgents[0].getOutput(), caAgents[1].getOutput(), caAgents[2].getOutput(), cRiskAgent.getOutput(), cConcatenatedAgents.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1) || !Concat(caTrAgents[0].getOutput(), caTrAgents[1].getOutput(), caTrAgents[2].getOutput(), cConcatenatedAgents.getPrevOutput(), cConcatenatedAgents.getOutput(), Neurons(), Neurons(), Neurons(), 4 * Neurons(), 1)) ReturnFalse; //--- Manager return CNeuronFinConAgent::feedForward(cConcatenatedAgents.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; if(cAccount.getGradient() != SecondGradient) { if(!cAccount.SetGradient(SecondGradient, true)) ReturnFalse; cAccount.SetActivationFunction(SecondActivation); } //--- if(!CNeuronFinConAgent::calcInputGradients(cConcatenatedAgents.AsObject())) ReturnFalse; //--- if(!DeConcat(caTrAgents[0].getGradient(), caTrAgents[1].getGradient(), caTrAgents[2].getGradient(), cConcatenatedAgents.getPrevOutput(), cConcatenatedAgents.getGradient(), Neurons(), Neurons(), Neurons(), 4 * Neurons(), 1) || !DeConcat(caAgents[0].getGradient(), caAgents[1].getGradient(), caAgents[2].getGradient(), cRiskAgent.getGradient(), cConcatenatedAgents.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; //--- if(!cAccount.CalcHiddenGradients(cRiskAgent.AsObject())) ReturnFalse; if(!cTransposeState.CalcHiddenGradients(caTrAgents[0].AsObject())) ReturnFalse; CBufferFloat *temp = cTransposeState.getGradient(); if(!cTransposeState.SetGradient(cTransposeState.getPrevOutput(), false)) ReturnFalse; for(uint i = 1; i < caTrAgents.Size(); i++) { if(!cTransposeState.CalcHiddenGradients(caTrAgents[i].AsObject())) ReturnFalse; if(!SumAndNormilize(cTransposeState.getGradient(), temp, temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!cTransposeState.SetGradient(temp, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTransposeState.AsObject())) ReturnFalse; temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cTransposeState.getPrevOutput(), false)) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) { if(!NeuronOCL.CalcHiddenGradients(caAgents[i].AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), temp, temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { //--- Manager if(!CNeuronFinConAgent::updateInputWeights(cConcatenatedAgents.AsObject())) ReturnFalse; //--- Agents for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = 0; i < caTrAgents.Size(); i++) if(!caTrAgents[i].UpdateInputWeights(cTransposeState.AsObject())) ReturnFalse; if(!cRiskAgent.UpdateInputWeights(cAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFinConManager::SetOpenCL(COpenCLMy * obj) { CNeuronFinConAgent::SetOpenCL(obj); cTransposeState.SetOpenCL(OpenCL); for(uint i = 0; i < caAgents.Size(); i++) caAgents[i].SetOpenCL(OpenCL); for(uint i = 0; i < caTrAgents.Size(); i++) caTrAgents[i].SetOpenCL(OpenCL); cRiskAgent.SetOpenCL(OpenCL); cConcatenatedAgents.SetOpenCL(OpenCL); cAccount.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFinConManager::TrainMode(bool flag) { CNeuronFinConAgent::TrainMode(flag); cTransposeState.TrainMode(flag); for(uint i = 0; i < caAgents.Size(); i++) caAgents[i].TrainMode(flag); for(uint i = 0; i < caTrAgents.Size(); i++) caTrAgents[i].TrainMode(flag); cRiskAgent.TrainMode(flag); cConcatenatedAgents.TrainMode(flag); cAccount.TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::Save(const int file_handle) { if(!CNeuronFinConAgent::Save(file_handle)) ReturnFalse; if(!cTransposeState.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caTrAgents.Size(); i++) if(!caTrAgents[i].Save(file_handle)) ReturnFalse; if(!cRiskAgent.Save(file_handle)) ReturnFalse; if(!cConcatenatedAgents.Save(file_handle)) ReturnFalse; if(!cAccount.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::Load(const int file_handle) { if(!CNeuronFinConAgent::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeState.AsObject())) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caAgents[i].AsObject())) ReturnFalse; for(uint i = 0; i < caTrAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caTrAgents[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRiskAgent.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenatedAgents.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFinConManager::Clear(void) { if(!CNeuronFinConAgent::Clear()) ReturnFalse; if(!cTransposeState.Clear()) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].Clear()) ReturnFalse; for(uint i = 0; i < caTrAgents.Size(); i++) if(!caTrAgents[i].Clear()) ReturnFalse; if(!cRiskAgent.Clear()) ReturnFalse; if(!cConcatenatedAgents.Clear()) ReturnFalse; if(!cAccount.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMacroHFTHyperAgent : public CNeuronSoftMaxOCL { protected: CNeuronMemoryDistil cMemory; CNeuronRMAT cStatePrepare; CNeuronTransposeOCL cTranspose; CNeuronConvOCL cScale; CNeuronBaseOCL cMLP[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMacroHFTHyperAgent(void) {}; ~CNeuronMacroHFTHyperAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint agents, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMacroHFTHyperAgent; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint agents, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, agents, optimization_type, batch)) ReturnFalse; SetHeads(1); //--- int index = 0; if(!cMemory.Init(0, 0, OpenCL, window, window_key, units_count, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cStatePrepare.Init(0, index, OpenCL, window, window_key, units_count, heads, layers, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; index++; if(!cScale.Init(4 * agents, index, OpenCL, 3, 1, 1, units_count - 2, window, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); index++; if(!cMLP[0].Init(agents, index, OpenCL, 4 * agents, optimization, iBatch)) ReturnFalse; cMLP[0].SetActivationFunction(LReLU); index++; if(!cMLP[1].Init(0, index, OpenCL, agents, optimization, iBatch)) ReturnFalse; cMLP[0].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cMemory.FeedForward(NeuronOCL)) ReturnFalse; if(!cStatePrepare.FeedForward(cMemory.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cStatePrepare.AsObject())) ReturnFalse; if(!cScale.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cMLP[0].FeedForward(cScale.AsObject())) ReturnFalse; if(!cMLP[1].FeedForward(cMLP[0].AsObject())) ReturnFalse; //--- return CNeuronSoftMaxOCL::feedForward(cMLP[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSoftMaxOCL::calcInputGradients(cMLP[1].AsObject())) ReturnFalse; if(!cMLP[0].CalcHiddenGradients(cMLP[1].AsObject())) ReturnFalse; if(!cScale.CalcHiddenGradients(cMLP[0].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cScale.AsObject())) ReturnFalse; if(!cStatePrepare.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!cMemory.CalcHiddenGradients(cStatePrepare.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMemory.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cMLP[1].UpdateInputWeights(cMLP[0].AsObject())) ReturnFalse; if(!cMLP[0].UpdateInputWeights(cScale.AsObject())) ReturnFalse; if(!cScale.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; if(!cStatePrepare.UpdateInputWeights(cMemory.AsObject())) ReturnFalse; if(!cMemory.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; if(!cStatePrepare.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cMLP.Size(); i++) if(!cMLP[i].Save(file_handle)) ReturnFalse; if(!cMemory.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStatePrepare.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; for(uint i = 0; i < cMLP.Size(); i++) if(!LoadInsideLayer(file_handle, cMLP[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMemory.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMacroHFTHyperAgent::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); cStatePrepare.SetOpenCL(obj); cTranspose.SetOpenCL(obj); cScale.SetOpenCL(obj); for(uint i = 0; i < cMLP.Size(); i++) cMLP[i].SetOpenCL(obj); cMemory.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTHyperAgent::Clear(void) { if(!CNeuronSoftMaxOCL::Clear()) ReturnFalse; if(!cStatePrepare.Clear()) ReturnFalse; if(!cTranspose.Clear()) ReturnFalse; if(!cScale.Clear()) ReturnFalse; for(uint i = 0; i < cMLP.Size(); i++) if(!cMLP[i].Clear()) ReturnFalse; if(!cMemory.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMacroHFT : public CNeuronBaseOCL { protected: CNeuronTransposeOCL cTranspose; CNeuronFinConAgent caAgents[6]; CNeuronMacroHFTHyperAgent cHyperAgent; CNeuronBaseOCL cConcatenated; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMacroHFT(void) {}; ~CNeuronMacroHFT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMacroHFT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, nactions, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; uint half = (caAgents.Size() + 1) / 2; for(uint i = 0; i < half; i++) { index++; if(!caAgents[i].Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } for(uint i = half; i < caAgents.Size(); i++) { index++; if(!caAgents[i].Init(0, index, OpenCL, units_count, window_key, window, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } index++; if(!cHyperAgent.Init(0, index, OpenCL, window, window_key, units_count, heads, layers, caAgents.Size(), stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cConcatenated.Init(0, index, OpenCL, caAgents.Size()*nactions, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- uint total = caAgents.Size(); uint half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caAgents[i].FeedForward(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caAgents[i].FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cHyperAgent.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Concat(caAgents[0].getOutput(), caAgents[1].getOutput(), caAgents[2].getOutput(), caAgents[3].getOutput(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1) || !Concat(cConcatenated.getPrevOutput(), caAgents[4].getOutput(), caAgents[5].getOutput(), cConcatenated.getOutput(), 4 * Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; if(!MatMul(cHyperAgent.getOutput(), cConcatenated.getOutput(), Output, 1, total, Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint total = caAgents.Size(); if(!MatMulGrad(cHyperAgent.getOutput(), cHyperAgent.getGradient(), cConcatenated.getOutput(), cConcatenated.getGradient(), Gradient, 1, total, Neurons(), 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cHyperAgent.AsObject())) ReturnFalse; //--- if(!DeConcat(cConcatenated.getPrevOutput(), caAgents[4].getGradient(), caAgents[5].getGradient(), cConcatenated.getGradient(), 4 * Neurons(), Neurons(), Neurons(), 1) || !DeConcat(caAgents[0].getGradient(), caAgents[1].getGradient(), caAgents[2].getGradient(), caAgents[3].getGradient(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!temp || !NeuronOCL.SetGradient(cTranspose.getPrevOutput(), false)) ReturnFalse; uint half = (total + 1) / 2; for(uint i = 0; i < half; i++) { if(!NeuronOCL.CalcHiddenGradients(caAgents[i].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } for(uint i = half; i < total; i++) { if(!cTranspose.CalcHiddenGradients(caAgents[i].AsObject()) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cHyperAgent.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- uint total = caAgents.Size(); uint half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caAgents[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].Save(file_handle)) ReturnFalse; if(!cHyperAgent.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caAgents[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHyperAgent.AsObject())) ReturnFalse; //--- if(!cConcatenated.Init(0, 8, OpenCL, caAgents.Size()*Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMacroHFT::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(obj); for(uint i = 0; i < caAgents.Size(); i++) caAgents[i].SetOpenCL(obj); cHyperAgent.SetOpenCL(obj); cConcatenated.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFT::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cTranspose.Clear()) ReturnFalse; for(uint i = 0; i < caAgents.Size(); i++) if(!caAgents[i].Clear()) ReturnFalse; if(!cHyperAgent.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMacroHFTvsRiskManager : public CResidualConv { protected: CNeuronBaseOCL caAccountProjection[2]; CNeuronMemoryDistil cMemoryAccount; CNeuronMemoryDistil cMemoryAction; CNeuronRelativeCrossAttention cCrossAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; public: CNeuronMacroHFTvsRiskManager(void) {}; ~CNeuronMacroHFTvsRiskManager(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint nactions, uint account_decr, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMacroHFTvsRiskManager; } //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint nactions, uint account_decr, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, 3, 3, (nactions + 2) / 3, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!caAccountProjection[0].Init(0, index, OpenCL, account_decr, optimization, iBatch)) ReturnFalse; index++; if(!caAccountProjection[1].Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; index++; if(!cMemoryAccount.Init(caAccountProjection[1].Neurons(), index, OpenCL, account_decr, window_key, 1, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cMemoryAction.Init(0, index, OpenCL, 3, window_key, (nactions + 2) / 3, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cCrossAttention.Init(0, index, OpenCL, 3, window_key, (nactions + 2) / 3, heads, window, units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(caAccountProjection[0].getOutput() != SecondInput) { if(!caAccountProjection[0].SetOutput(SecondInput, true)) ReturnFalse; } //--- if(!cMemoryAccount.FeedForward(caAccountProjection[0].AsObject())) ReturnFalse; if(!caAccountProjection[1].FeedForward(cMemoryAccount.AsObject())) ReturnFalse; if(!cMemoryAction.FeedForward(NeuronOCL)) ReturnFalse; if(!cCrossAttention.FeedForward(cMemoryAction.AsObject(), caAccountProjection[1].getOutput())) ReturnFalse; //--- return CResidualConv::feedForward(cCrossAttention.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; if(caAccountProjection[0].getGradient() != SecondGradient) { if(!caAccountProjection[0].SetGradient(SecondGradient, true)) ReturnFalse; caAccountProjection[0].SetActivationFunction(SecondActivation); } //--- if(!CResidualConv::calcInputGradients(cCrossAttention.AsObject())) ReturnFalse; if(!cMemoryAction.CalcHiddenGradients(cCrossAttention.AsObject(), caAccountProjection[1].getOutput(), caAccountProjection[1].getGradient(), (ENUM_ACTIVATION)caAccountProjection[1].Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cMemoryAction.AsObject())) ReturnFalse; //--- if(!cMemoryAccount.CalcHiddenGradients(caAccountProjection[1].AsObject())) ReturnFalse; if(!caAccountProjection[0].CalcHiddenGradients(cMemoryAccount.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!CResidualConv::updateInputWeights(cCrossAttention.AsObject())) ReturnFalse; if(!cCrossAttention.UpdateInputWeights(cMemoryAction.AsObject(), caAccountProjection[1].getOutput())) ReturnFalse; //--- if(!cMemoryAction.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!caAccountProjection[1].UpdateInputWeights(cMemoryAccount.AsObject())) ReturnFalse; if(!cMemoryAccount.UpdateInputWeights(caAccountProjection[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; for(uint i = 0; i < caAccountProjection.Size(); i++) if(!caAccountProjection[i].Save(file_handle)) ReturnFalse; if(!cMemoryAccount.Save(file_handle)) ReturnFalse; if(!cMemoryAction.Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::Load(const int file_handle) { if(!CResidualConv::Load(file_handle)) ReturnFalse; for(uint i = 0; i < caAccountProjection.Size(); i++) if(!LoadInsideLayer(file_handle, caAccountProjection[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMemoryAccount.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMemoryAction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMacroHFTvsRiskManager::SetOpenCL(COpenCLMy * obj) { CResidualConv::SetOpenCL(obj); for(uint i = 0; i < caAccountProjection.Size(); i++) caAccountProjection[i].SetOpenCL(OpenCL); cMemoryAccount.SetOpenCL(OpenCL); cMemoryAction.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMacroHFTvsRiskManager::Clear(void) { if(!CResidualConv::Clear()) ReturnFalse; for(uint i = 0; i < caAccountProjection.Size(); i++) if(!caAccountProjection[i].Clear()) ReturnFalse; if(!cMemoryAccount.Clear()) ReturnFalse; if(!cMemoryAction.Clear()) ReturnFalse; if(!cCrossAttention.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleRelativeSelfAttention : public CNeuronRelativeSelfAttention { protected: //--- virtual bool AttentionOut(void); public: CNeuronMultiScaleRelativeSelfAttention(void) {}; ~CNeuronMultiScaleRelativeSelfAttention(void) {}; //--- virtual int Type(void) override const { return defNeuronMultiScaleRelativeSelfAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeSelfAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleRelativeCrossAttention : public CNeuronRelativeCrossAttention { protected: virtual bool AttentionOut(void); public: CNeuronMultiScaleRelativeCrossAttention(void) {}; ~CNeuronMultiScaleRelativeCrossAttention(void) {}; //--- virtual int Type(void) override const { return defNeuronMultiScaleRelativeCrossAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleRelativeCrossAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits/*Q units*/, cKey.Neurons() / (iHeads * iWindowKey), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; int kernel = def_k_MultiScaleRelativeAttentionOut; //--- ResetLastError(); setBuffer(kernel, def_k_rat_q, cQuery.getOutputIndex()) setBuffer(kernel, def_k_rat_k, cKey.getOutputIndex()) setBuffer(kernel, def_k_rat_v, cValue.getOutputIndex()) setBuffer(kernel, def_k_rat_score, iScore) setBuffer(kernel, def_k_rat_bk, ((CNeuronBaseOCL*)cBKey[cBKey.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_bv, ((CNeuronBaseOCL*)cBValue[cBValue.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gc, ((CNeuronBaseOCL*)cGlobalContentBias[cGlobalContentBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_gp, ((CNeuronBaseOCL*)cGlobalPositionalBias[cGlobalPositionalBias.Total() - 1]).getOutputIndex()) setBuffer(kernel, def_k_rat_out, ((CNeuronBaseOCL*)cMHAttentionPooling[0]).getOutputIndex()) setArgument(kernel, def_k_rat_dimension, (int)iWindowKey) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRecursiveAttention : public CNeuronMultiScaleRelativeCrossAttention { protected: CNeuronMultiScaleRelativeSelfAttention cSelfAttention; CNeuronTransposeOCL cTransposeSA; CNeuronConvOCL cConvolution; CNeuronEmbeddingOCL cHistory; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return updateInputWeights(NeuronOCL); } public: CNeuronRecursiveAttention(void) {}; ~CNeuronRecursiveAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint history_size, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRecursiveAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint history_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMultiScaleRelativeCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, units_count, heads, window_key, history_size, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cSelfAttention.Init(0, index, OpenCL, iWindow, iWindowKey, iUnits, iHeads, optimization, iBatch)) ReturnFalse; index++; if(!cTransposeSA.Init(0, index, OpenCL, iUnits, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cConvolution.Init(0, index, OpenCL, iUnits, iUnits, iWindowKey, 1, iWindow, optimization, iBatch)) ReturnFalse; index++; uint windows[] = { iWindowKey * iWindow }; if(!cHistory.Init(0, index, OpenCL, iUnitsKV, iWindowKey, windows)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!cTransposeSA.FeedForward(cSelfAttention.AsObject())) ReturnFalse; if(!cConvolution.FeedForward(cTransposeSA.AsObject())) ReturnFalse; if(!cHistory.FeedForward(cConvolution.AsObject())) ReturnFalse; //--- return CNeuronMultiScaleRelativeCrossAttention::feedForward(cSelfAttention.AsObject(), cHistory.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMultiScaleRelativeCrossAttention::calcInputGradients(cSelfAttention.AsObject(), cHistory.getOutput(), cHistory.getGradient(), (ENUM_ACTIVATION)cHistory.Activation())) ReturnFalse; if(!cConvolution.CalcHiddenGradients(cHistory.AsObject())) ReturnFalse; if(!cTransposeSA.CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; CBufferFloat *temp = cSelfAttention.getGradient(); if(!cSelfAttention.SetGradient(cTransposeSA.getPrevOutput(), false) || !cSelfAttention.CalcHiddenGradients(cTransposeSA.AsObject()) || !SumAndNormilize(temp, cSelfAttention.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cSelfAttention.SetGradient(temp, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSelfAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMultiScaleRelativeCrossAttention::updateInputWeights(cSelfAttention.AsObject(), cHistory.getOutput())) ReturnFalse; if(!cHistory.UpdateInputWeights(cConvolution.AsObject())) ReturnFalse; if(!cConvolution.UpdateInputWeights(cTransposeSA.AsObject())) ReturnFalse; if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Save(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; if(!cTransposeSA.Save(file_handle)) ReturnFalse; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cHistory.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Load(const int file_handle) { if(!CNeuronMultiScaleRelativeCrossAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeSA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHistory.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRecursiveAttention::Clear(void) { if(!CNeuronMultiScaleRelativeCrossAttention::Clear()) ReturnFalse; if(!cSelfAttention.Clear()) ReturnFalse; if(!cTransposeSA.Clear()) ReturnFalse; if(!cConvolution.Clear()) ReturnFalse; if(!cHistory.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRecursiveAttention::SetOpenCL(COpenCLMy * obj) { CNeuronMultiScaleRelativeCrossAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); cTransposeSA.SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cHistory.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronLinerAttention : public CNeuronBaseOCL { protected: uint iWindow; uint iWindowKey; uint iUnits; uint iVariables; //--- CNeuronConvOCL cQuery; CNeuronConvOCL cKey; CNeuronTransposeVRCOCL cKeyT; CNeuronBaseOCL cKeyValue; CNeuronBaseOCL cAttentionOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronLinerAttention(void) {}; ~CNeuronLinerAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronLinerAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = fmax(window_key, 1); iUnits = units_count; iVariables = variables; //--- int index = 0; if(!cQuery.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQuery.SetActivationFunction(SIGMOID); index++; if(!cKey.Init(0, index, OpenCL, iWindow, iWindow, iWindowKey, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cKey.SetActivationFunction(TANH); index++; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction(TANH); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cQuery.FeedForward(NeuronOCL)) ReturnFalse; if(!cKey.FeedForward(NeuronOCL) || !cKeyT.FeedForward(cKey.AsObject())) ReturnFalse; if(!MatMul(cKeyT.getOutput(), NeuronOCL.getOutput(), cKeyValue.getOutput(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!MatMul(cQuery.getOutput(), cKeyValue.getOutput(), cAttentionOut.getOutput(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cAttentionOut.getOutput(), Output, iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cQuery.getOutput(), cQuery.getGradient(), cKeyValue.getOutput(), cKeyValue.getGradient(), cAttentionOut.getGradient(), iUnits, iWindowKey, iWindow, iVariables)) ReturnFalse; if(!MatMulGrad(cKeyT.getOutput(), cKeyT.getGradient(), NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cKeyValue.getGradient(), iWindowKey, iUnits, iWindow, iVariables)) ReturnFalse; if(!SumAndNormilize(Gradient, cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(cKeyT.Activation() != None) if(!DeActivation(cKeyT.getOutput(), cKeyT.getGradient(), cKeyT.getGradient(), cKeyT.Activation())) ReturnFalse; if(cQuery.Activation() != None) if(!DeActivation(cQuery.getOutput(), cQuery.getGradient(), cQuery.getGradient(), cQuery.Activation())) ReturnFalse; //--- if(!cKey.CalcHiddenGradients(cKeyT.AsObject()) || !NeuronOCL.CalcHiddenGradients(cKey.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), cAttentionOut.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQuery.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), cAttentionOut.getPrevOutput(), NeuronOCL.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cQuery.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKey.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronLinerAttention* Source = source; if(!cQuery.WeightsUpdate(Source.cQuery.AsObject(), tau)) ReturnFalse; if(!cKey.WeightsUpdate(Source.cKey.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQuery.Save(file_handle)) ReturnFalse; if(!cKey.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronLinerAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (int)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (int)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQuery.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKey.AsObject())) ReturnFalse; //--- int index = 2; if(!cKeyT.Init(0, index, OpenCL, iVariables, iUnits, iWindowKey, optimization, iBatch)) ReturnFalse; cKeyT.SetActivationFunction((ENUM_ACTIVATION)cKey.Activation()); index++; if(!cKeyValue.Init(0, index, OpenCL, iWindow * iWindowKey * iVariables, optimization, iBatch)) ReturnFalse; cKeyValue.SetActivationFunction(None); index++; if(!cAttentionOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttentionOut.SetActivationFunction(None); //--- if(!SetGradient(cAttentionOut.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronLinerAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cQuery.SetOpenCL(OpenCL); cKey.SetOpenCL(OpenCL); cKeyT.SetOpenCL(OpenCL); cKeyValue.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHidformerTSAgent : public CResidualConv { protected: CNeuronBaseOCL caRole[2]; CNeuronRelativeCrossAttention cStateToRole; CNeuronS3 cShuffle; CNeuronRecursiveAttention cRecursiveState; CResidualConv cResidualState; CNeuronRecursiveAttention cRecursiveAction; CNeuronMultiScaleRelativeCrossAttention cActionToState; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHidformerTSAgent(void) {}; ~CNeuronHidformerTSAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHidformerTSAgent; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, 3, 3, (action_space + 2) / 3, optimization_type, batch)) ReturnFalse; //--- Role int index = 0; if(!caRole[0].Init(10 * window_key, index, OpenCL, 1, optimization, iBatch)) ReturnFalse; caRole[0].getOutput().Fill(1); index++; if(!caRole[1].Init(0, index, OpenCL, 10 * window_key, optimization, iBatch)) ReturnFalse; //--- State to Role index++; if(!cStateToRole.Init(0, index, OpenCL, window, window_key, units_count, heads, window_key, 10, optimization, iBatch)) ReturnFalse; //--- State index++; if(!cShuffle.Init(0, index, OpenCL, window, window * units_count, optimization, iBatch)) ReturnFalse; index++; if(!cRecursiveState.Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cResidualState.Init(0, index, OpenCL, window, window, units_count, optimization, iBatch)) ReturnFalse; //--- Action index++; if(!cRecursiveAction.Init(0, index, OpenCL, 3, window_key, (action_space + 2) / 3, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cActionToState.Init(0, index, OpenCL, 3, window_key, (action_space + 2) / 3, heads, window, units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::feedForward(CNeuronBaseOCL * NeuronOCL) { if(bTrain && !caRole[1].FeedForward(caRole[0].AsObject())) ReturnFalse; //--- State to Role if(!cStateToRole.FeedForward(NeuronOCL, caRole[1].getOutput())) ReturnFalse; //--- State if(!cShuffle.FeedForward(cStateToRole.AsObject())) ReturnFalse; if(!cRecursiveState.FeedForward(cShuffle.AsObject())) ReturnFalse; if(!cResidualState.FeedForward(cRecursiveState.AsObject())) ReturnFalse; //--- Action if(!cRecursiveAction.FeedForward(AsObject())) ReturnFalse; if(!cActionToState.FeedForward(cRecursiveAction.AsObject(), cResidualState.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- return CResidualConv::feedForward(cActionToState.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CResidualConv::calcInputGradients(cActionToState.AsObject())) ReturnFalse; if(!cRecursiveAction.CalcHiddenGradients(cActionToState.AsObject(), cResidualState.getOutput(), cResidualState.getGradient(), (ENUM_ACTIVATION)cResidualState.Activation())) ReturnFalse; //--- Action CBufferFloat *temp = Gradient; if(!SwapBuffers(Output, PrevOutput) || !SetGradient(cRecursiveAction.getPrevOutput(), false)) ReturnFalse; if(!calcHiddenGradients(cRecursiveAction.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; Gradient = temp; //--- State if(!cRecursiveState.CalcHiddenGradients(cResidualState.AsObject())) ReturnFalse; if(!cShuffle.CalcHiddenGradients(cRecursiveState.AsObject())) ReturnFalse; if(!cStateToRole.CalcHiddenGradients(cShuffle.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cStateToRole.AsObject(), caRole[1].getOutput(), caRole[1].getGradient(), (ENUM_ACTIVATION)caRole[1].Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CResidualConv::updateInputWeights(cActionToState.AsObject())) ReturnFalse; //--- Action if(!cActionToState.UpdateInputWeights(cRecursiveAction.AsObject())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput) || !cRecursiveAction.UpdateInputWeights(AsObject()) || !SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- State if(!cResidualState.UpdateInputWeights(cRecursiveState.AsObject())) ReturnFalse; if(!cResidualState.UpdateInputWeights(cShuffle.AsObject())) ReturnFalse; if(!cShuffle.UpdateInputWeights(cStateToRole.AsObject())) ReturnFalse; if(!cStateToRole.UpdateInputWeights(NeuronOCL, caRole[1].getOutput())) ReturnFalse; //--- Role if(!caRole[1].UpdateInputWeights(caRole[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caRole.Size(); i++) if(!caRole[i].Save(file_handle)) ReturnFalse; if(!cStateToRole.Save(file_handle)) ReturnFalse; if(!cShuffle.Save(file_handle)) ReturnFalse; if(!cRecursiveState.Save(file_handle)) ReturnFalse; if(!cResidualState.Save(file_handle)) ReturnFalse; if(!cRecursiveAction.Save(file_handle)) ReturnFalse; if(!cActionToState.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::Load(const int file_handle) { if(!CResidualConv::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caRole.Size(); i++) if(!LoadInsideLayer(file_handle, caRole[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStateToRole.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShuffle.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRecursiveState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidualState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRecursiveAction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionToState.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHidformerTSAgent::SetOpenCL(COpenCLMy * obj) { CResidualConv::SetOpenCL(obj); //--- for(uint i = 0; i < caRole.Size(); i++) caRole[i].SetOpenCL(OpenCL); cStateToRole.SetOpenCL(OpenCL); cShuffle.SetOpenCL(OpenCL); cRecursiveState.SetOpenCL(OpenCL); cResidualState.SetOpenCL(OpenCL); cRecursiveAction.SetOpenCL(OpenCL); cActionToState.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerTSAgent::Clear(void) { if(!CResidualConv::Clear()) ReturnFalse; //--- for(uint i = 0; i < caRole.Size(); i++) if(!caRole[i].Clear()) ReturnFalse; if(!cStateToRole.Clear()) ReturnFalse; if(!cShuffle.Clear()) ReturnFalse; if(!cRecursiveState.Clear()) ReturnFalse; if(!cResidualState.Clear()) ReturnFalse; if(!cRecursiveAction.Clear()) ReturnFalse; if(!cActionToState.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHidformerFreqAgent : public CResidualConv { protected: CNeuronTransposeOCL cTranspose; CNeuronLegendreWaveletsHL cLegendre; CNeuronTransposeRCDOCL cHLState; CNeuronLinerAttention cAttentionState; CResidualConv cResidualState; CNeuronS3 cShuffle; CNeuronRecursiveAttention cRecursiveAction; CNeuronMultiScaleRelativeCrossAttention cActionToState; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHidformerFreqAgent(void) {}; ~CNeuronHidformerFreqAgent(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint filters, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHidformerFreqAgent; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint filters, uint units_count, uint heads, uint stack_size, uint action_space, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, 3, 3, (action_space + 2) / 3, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; index++; uint wind = (units_count >= 20 ? (units_count + 3) / 4 : units_count); uint units = (units_count + wind - 1) / wind; if(!cLegendre.Init(0, index, OpenCL, wind, wind, units, filters, window, optimization, batch)) ReturnFalse; index++; if(!cHLState.Init(0, index, OpenCL, units * window, 2, filters, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionState.Init(0, index, OpenCL, filters, filters, units * window, 2, optimization, iBatch)) ReturnFalse; index++; if(!cResidualState.Init(0, index, OpenCL, filters, filters, 2 * units * window, optimization, iBatch)) ReturnFalse; index++; if(!cShuffle.Init(0, index, OpenCL, filters, cResidualState.Neurons(), optimization, iBatch)) ReturnFalse; //--- Action index++; if(!cRecursiveAction.Init(0, index, OpenCL, 3, filters, (action_space + 2) / 3, heads, stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cActionToState.Init(0, index, OpenCL, 3, filters, (action_space + 2) / 3, heads, filters, 2 * units * window, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- State if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cLegendre.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cHLState.FeedForward(cLegendre.AsObject())) ReturnFalse; if(!cAttentionState.FeedForward(cHLState.AsObject())) ReturnFalse; if(!cResidualState.FeedForward(cAttentionState.AsObject())) ReturnFalse; if(!cShuffle.FeedForward(cResidualState.AsObject())) ReturnFalse; //--- Action if(!cRecursiveAction.FeedForward(AsObject())) ReturnFalse; if(!cActionToState.FeedForward(cRecursiveAction.AsObject(), cShuffle.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput)) ReturnFalse; return CResidualConv::feedForward(cActionToState.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Action if(!CResidualConv::calcInputGradients(cActionToState.AsObject())) ReturnFalse; if(!cRecursiveAction.CalcHiddenGradients(cActionToState.AsObject(), cShuffle.getOutput(), cShuffle.getGradient(), (ENUM_ACTIVATION)cShuffle.Activation())) ReturnFalse; CBufferFloat *temp = Gradient; if(!SwapBuffers(Output, PrevOutput) || !SetGradient(cActionToState.getPrevOutput(), false) || !calcHiddenGradients(cRecursiveAction.AsObject()) || !SwapBuffers(Output, PrevOutput)) ReturnFalse; Gradient = temp; //--- State if(!cResidualState.CalcHiddenGradients(cShuffle.AsObject())) ReturnFalse; if(!cAttentionState.CalcHiddenGradients(cResidualState.AsObject())) ReturnFalse; if(!cHLState.CalcHiddenGradients(cAttentionState.AsObject())) ReturnFalse; if(!cLegendre.CalcHiddenGradients(cHLState.AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cLegendre.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- Action if(!CResidualConv::updateInputWeights(cActionToState.AsObject())) ReturnFalse; if(!cActionToState.UpdateInputWeights(cRecursiveAction.AsObject(), cShuffle.getOutput())) ReturnFalse; if(!SwapBuffers(Output, PrevOutput) || !cRecursiveAction.UpdateInputWeights(AsObject()) || !SwapBuffers(Output, PrevOutput)) ReturnFalse; //--- State if(!cShuffle.UpdateInputWeights(cResidualState.AsObject())) ReturnFalse; if(!cResidualState.UpdateInputWeights(cAttentionState.AsObject())) ReturnFalse; if(!cAttentionState.UpdateInputWeights(cHLState.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cLegendre.Save(file_handle)) ReturnFalse; if(!cHLState.Save(file_handle)) ReturnFalse; if(!cAttentionState.Save(file_handle)) ReturnFalse; if(!cResidualState.Save(file_handle)) ReturnFalse; if(!cShuffle.Save(file_handle)) ReturnFalse; //--- Action if(!cRecursiveAction.Save(file_handle)) ReturnFalse; if(!cActionToState.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::Load(const int file_handle) { if(!CResidualConv::Load(file_handle)) ReturnFalse; //--- State if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLegendre.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHLState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttentionState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidualState.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShuffle.AsObject())) ReturnFalse; //--- Action if(!LoadInsideLayer(file_handle, cRecursiveAction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cActionToState.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformerFreqAgent::Clear(void) { if(!CResidualConv::Clear()) ReturnFalse; //--- if(!cTranspose.Clear()) ReturnFalse; if(!cLegendre.Clear()) ReturnFalse; if(!cHLState.Clear()) ReturnFalse; if(!cAttentionState.Clear()) ReturnFalse; if(!cResidualState.Clear()) ReturnFalse; if(!cShuffle.Clear()) ReturnFalse; //--- Action if(!cRecursiveAction.Clear()) ReturnFalse; if(!cActionToState.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHidformerFreqAgent::SetOpenCL(COpenCLMy * obj) { CResidualConv::SetOpenCL(obj); //--- cTranspose.SetOpenCL(OpenCL); cLegendre.SetOpenCL(OpenCL); cHLState.SetOpenCL(OpenCL); cAttentionState.SetOpenCL(OpenCL); cResidualState.SetOpenCL(OpenCL); cShuffle.SetOpenCL(OpenCL); //--- Action cRecursiveAction.SetOpenCL(OpenCL); cActionToState.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHidformer : public CNeuronBaseOCL { protected: CNeuronTransposeOCL cTranspose; CNeuronHidformerTSAgent caTSAgents[4]; CNeuronHidformerFreqAgent caFreqAgents[2]; CNeuronMacroHFTHyperAgent cHyperAgent; CNeuronBaseOCL cConcatenated; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHidformer(void) {}; ~CNeuronHidformer(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHidformer; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, nactions, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; uint half = (caTSAgents.Size() + 1) / 2; for(uint i = 0; i < half; i++) { index++; if(!caTSAgents[i].Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } for(uint i = half; i < caTSAgents.Size(); i++) { index++; if(!caTSAgents[i].Init(0, index, OpenCL, units_count, window_key, window, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } half = (caFreqAgents.Size() + 1) / 2; for(uint i = 0; i < half; i++) { index++; if(!caFreqAgents[i].Init(0, index, OpenCL, window, window_key, units_count, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } for(uint i = half; i < caFreqAgents.Size(); i++) { index++; if(!caFreqAgents[i].Init(0, index, OpenCL, units_count, window_key, window, heads, stack_size, nactions, optimization, iBatch)) ReturnFalse; } index++; if(!cHyperAgent.Init(0, index, OpenCL, window, window_key, units_count, heads, layers, caTSAgents.Size() + caFreqAgents.Size(), stack_size, optimization, iBatch)) ReturnFalse; index++; if(!cConcatenated.Init(0, index, OpenCL, (caTSAgents.Size() + caFreqAgents.Size())*nactions, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- uint total_ts = caTSAgents.Size(); uint half = (total_ts + 1) / 2; for(uint i = 0; i < half; i++) { if(!caTSAgents[i].FeedForward(NeuronOCL)) ReturnFalse; } for(uint i = half; i < total_ts; i++) { if(!caTSAgents[i].FeedForward(cTranspose.AsObject())) ReturnFalse; } //--- uint total_fr = caFreqAgents.Size(); half = (total_fr + 1) / 2; for(uint i = 0; i < half; i++) { if(!caFreqAgents[i].FeedForward(NeuronOCL)) ReturnFalse; } for(uint i = half; i < total_fr; i++) { if(!caFreqAgents[i].FeedForward(cTranspose.AsObject())) ReturnFalse; } if(!cHyperAgent.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!Concat(caTSAgents[0].getOutput(), caTSAgents[1].getOutput(), caTSAgents[2].getOutput(), caTSAgents[3].getOutput(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1) || !Concat(cConcatenated.getPrevOutput(), caFreqAgents[0].getOutput(), caFreqAgents[1].getOutput(), cConcatenated.getOutput(), 4 * Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; if(!MatMul(cHyperAgent.getOutput(), cConcatenated.getOutput(), Output, 1, total_fr + total_ts, Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint total_ts = caTSAgents.Size(); uint total_fr = caFreqAgents.Size(); if(!MatMulGrad(cHyperAgent.getOutput(), cHyperAgent.getGradient(), cConcatenated.getOutput(), cConcatenated.getGradient(), Gradient, 1, total_ts + total_fr, Neurons(), 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cHyperAgent.AsObject())) ReturnFalse; //--- if(!DeConcat(cConcatenated.getPrevOutput(), caFreqAgents[0].getGradient(), caFreqAgents[1].getGradient(), cConcatenated.getGradient(), 4 * Neurons(), Neurons(), Neurons(), 1) || !DeConcat(caTSAgents[0].getGradient(), caTSAgents[1].getGradient(), caTSAgents[2].getGradient(), caTSAgents[3].getGradient(), cConcatenated.getPrevOutput(), Neurons(), Neurons(), Neurons(), Neurons(), 1)) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!temp || !NeuronOCL.SetGradient(cTranspose.getPrevOutput(), false)) ReturnFalse; uint half = (total_ts + 1) / 2; for(uint i = 0; i < half; i++) { if(!NeuronOCL.CalcHiddenGradients(caTSAgents[i].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } for(uint i = half; i < total_ts; i++) { if(!cTranspose.CalcHiddenGradients(caTSAgents[i].AsObject()) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } half = (total_fr + 1) / 2; for(uint i = 0; i < half; i++) { if(!NeuronOCL.CalcHiddenGradients(caFreqAgents[i].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } for(uint i = half; i < total_fr; i++) { if(!cTranspose.CalcHiddenGradients(caFreqAgents[i].AsObject()) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cHyperAgent.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- uint total = caTSAgents.Size(); uint half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caTSAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caTSAgents[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- total = caFreqAgents.Size(); half = (total + 1) / 2; for(uint i = 0; i < half; i++) if(!caFreqAgents[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; for(uint i = half; i < total; i++) if(!caFreqAgents[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!caTSAgents[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!caFreqAgents[i].Save(file_handle)) ReturnFalse; if(!cHyperAgent.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caTSAgents[i].AsObject())) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqAgents[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHyperAgent.AsObject())) ReturnFalse; //--- if(!cConcatenated.Init(0, 8, OpenCL, (caTSAgents.Size() + caFreqAgents.Size())*Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHidformer::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(obj); for(uint i = 0; i < caTSAgents.Size(); i++) caTSAgents[i].SetOpenCL(obj); for(uint i = 0; i < caFreqAgents.Size(); i++) caFreqAgents[i].SetOpenCL(obj); cHyperAgent.SetOpenCL(obj); cConcatenated.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHidformer::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cTranspose.Clear()) ReturnFalse; for(uint i = 0; i < caTSAgents.Size(); i++) if(!caTSAgents[i].Clear()) ReturnFalse; for(uint i = 0; i < caFreqAgents.Size(); i++) if(!caFreqAgents[i].Clear()) ReturnFalse; if(!cHyperAgent.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResNeXtBottleneck : public CNeuronConvOCL { protected: CNeuronConvOCL cProjectionIn; CNeuronBatchNormOCL cNormalizeIn; CNeuronTransposeRCDOCL cTransposeIn; CNeuronConvOCL cFeatureExtraction; CNeuronBatchNormOCL cNormalizeFeature; CNeuronTransposeRCDOCL cTransposeOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronResNeXtBottleneck(void) {}; ~CNeuronResNeXtBottleneck(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint window, uint step, uint units_count, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronResNeXtBottleneck; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint chanels_in, uint chanels_out, uint window, uint step, uint units_count, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch) { int units_out = ((int)units_count - (int)window + (int)step - 1) / (int)step + 1; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, group_size * groups, group_size * groups, chanels_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Projection In uint index = 0; if(!cProjectionIn.Init(0, index, OpenCL, chanels_in, chanels_in, group_size * groups, units_count, 1, optimization, iBatch)) ReturnFalse; index++; if(!cNormalizeIn.Init(0, index, OpenCL, cProjectionIn.Neurons(), iBatch, optimization)) ReturnFalse; cNormalizeIn.SetActivationFunction(LReLU); index++; if(!cTransposeIn.Init(0, index, OpenCL, units_count, groups, group_size, optimization, iBatch)) ReturnFalse; cTransposeIn.SetActivationFunction((ENUM_ACTIVATION)cNormalizeIn.Activation()); //--- Feature Extraction index++; if(!cFeatureExtraction.Init(0, index, OpenCL, group_size * window, group_size * step, group_size, units_out, groups, optimization, iBatch)) ReturnFalse; index++; if(!cNormalizeFeature.Init(0, index, OpenCL, cFeatureExtraction.Neurons(), iBatch, optimization)) ReturnFalse; cNormalizeFeature.SetActivationFunction(LReLU); //--- Projection Out index++; if(!cTransposeOut.Init(0, index, OpenCL, groups, units_out, group_size, optimization, iBatch)) ReturnFalse; cTransposeOut.SetActivationFunction((ENUM_ACTIVATION)cNormalizeFeature.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Projection In if(!cProjectionIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cNormalizeIn.FeedForward(cProjectionIn.AsObject())) ReturnFalse; if(!cTransposeIn.FeedForward(cNormalizeIn.AsObject())) ReturnFalse; //--- Feature Extraction if(!cFeatureExtraction.FeedForward(cTransposeIn.AsObject())) ReturnFalse; if(!cNormalizeFeature.FeedForward(cFeatureExtraction.AsObject())) ReturnFalse; //--- Projection Out if(!cTransposeOut.FeedForward(cNormalizeFeature.AsObject())) ReturnFalse; return CNeuronConvOCL::feedForward(cTransposeOut.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Projection Out if(!CNeuronConvOCL::calcInputGradients(cTransposeOut.AsObject())) ReturnFalse; //--- Feature Extraction\ if(!cNormalizeFeature.CalcHiddenGradients(cTransposeOut.AsObject())) ReturnFalse; if(!cFeatureExtraction.CalcHiddenGradients(cNormalizeFeature.AsObject())) ReturnFalse; //--- Projection In if(!cTransposeIn.CalcHiddenGradients(cFeatureExtraction.AsObject())) ReturnFalse; if(!cNormalizeIn.CalcHiddenGradients(cTransposeIn.AsObject())) ReturnFalse; if(!cProjectionIn.CalcHiddenGradients(cNormalizeIn.AsObject())) ReturnFalse; //--- Inputs if(!NeuronOCL.CalcHiddenGradients(cProjectionIn.AsObject())) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- Projection Out if(!CNeuronConvOCL::updateInputWeights(cTransposeOut.AsObject())) ReturnFalse; //--- Feature Extraction if(!cNormalizeFeature.UpdateInputWeights(cFeatureExtraction.AsObject())) ReturnFalse; if(!cFeatureExtraction.UpdateInputWeights(cTransposeIn.AsObject())) ReturnFalse; //--- Projection In if(!cNormalizeIn.UpdateInputWeights(cProjectionIn.AsObject())) ReturnFalse; if(!cProjectionIn.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cProjectionIn.Save(file_handle)) ReturnFalse; if(!cNormalizeIn.Save(file_handle)) ReturnFalse; if(!cTransposeIn.Save(file_handle)) ReturnFalse; if(!cFeatureExtraction.Save(file_handle)) ReturnFalse; if(!cNormalizeFeature.Save(file_handle)) ReturnFalse; if(!cTransposeOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBottleneck::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizeIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeatureExtraction.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizeFeature.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResNeXtBottleneck::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); cProjectionIn.SetOpenCL(obj); cNormalizeIn.SetOpenCL(obj); cTransposeIn.SetOpenCL(obj); cFeatureExtraction.SetOpenCL(obj); cNormalizeFeature.SetOpenCL(obj); cTransposeOut.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResNeXtResidual: public CNeuronConvOCL { protected: CNeuronTransposeOCL cTransposeIn; CNeuronConvOCL cProjectionTime; CNeuronBatchNormOCL cNormalizeTime; CNeuronTransposeOCL cTransposeOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronResNeXtResidual(void) {}; ~CNeuronResNeXtResidual(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronResNeXtResidual; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, chanels_in, chanels_in, chanels_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cTransposeIn.Init(0, index, OpenCL, units_in, chanels_in, optimization, iBatch)) ReturnFalse; index++; if(!cProjectionTime.Init(0, index, OpenCL, units_in, units_in, units_out, chanels_in, 1, optimization, iBatch)) ReturnFalse; index++; if(!cNormalizeTime.Init(0, index, OpenCL, cProjectionTime.Neurons(), iBatch, optimization)) ReturnFalse; index++; if(!cTransposeOut.Init(0, index, OpenCL, chanels_in, units_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Projection Timeline if(!cTransposeIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjectionTime.FeedForward(cTransposeIn.AsObject())) ReturnFalse; if(!cNormalizeTime.FeedForward(cProjectionTime.AsObject())) ReturnFalse; //--- Projection Chanels if(!cTransposeOut.FeedForward(cNormalizeTime.AsObject())) ReturnFalse; return CNeuronConvOCL::feedForward(cTransposeOut.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Projection Chanels if(!CNeuronConvOCL::calcInputGradients(cTransposeOut.AsObject())) ReturnFalse; //--- Projection Timeline if(!cNormalizeTime.CalcHiddenGradients(cTransposeOut.AsObject())) ReturnFalse; if(!cProjectionTime.CalcHiddenGradients(cNormalizeTime.AsObject())) ReturnFalse; if(!cTransposeIn.CalcHiddenGradients(cProjectionTime.AsObject())) ReturnFalse; //--- Inputs return NeuronOCL.CalcHiddenGradients(cTransposeIn.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- Projection Chanels if(!CNeuronConvOCL::updateInputWeights(cTransposeOut.AsObject())) ReturnFalse; //--- Projection Timeline if(!cNormalizeTime.UpdateInputWeights(cProjectionTime.AsObject())) ReturnFalse; if(!cProjectionTime.UpdateInputWeights(cTransposeIn.AsObject())) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- if(!cTransposeIn.Save(file_handle)) ReturnFalse; if(!cProjectionTime.Save(file_handle)) ReturnFalse; if(!cNormalizeTime.Save(file_handle)) ReturnFalse; if(!cTransposeOut.Save(file_handle)) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtResidual::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTransposeIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionTime.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizeTime.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTransposeOut.AsObject())) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResNeXtResidual::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); //--- cTransposeIn.SetOpenCL(OpenCL); cProjectionTime.SetOpenCL(OpenCL); cNormalizeTime.SetOpenCL(OpenCL); cTransposeOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResNeXtBlock : public CNeuronBaseOCL { protected: uint iChanelsOut; CNeuronResNeXtBottleneck cBottleneck; CNeuronResNeXtResidual cResidual; CBufferFloat cBuffer; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronResNeXtBlock(void) {}; ~CNeuronResNeXtBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint window, uint step, uint units_count, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronResNeXtBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint chanels_in, uint chanels_out, uint window, uint step, uint units_count, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch) { int units_out = ((int)units_count - (int)window + (int)step - 1) / (int)step + 1; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_out * chanels_out, optimization_type, batch)) ReturnFalse; //--- iChanelsOut = chanels_out; //--- int index = 0; if(!cBottleneck.Init(0, index, OpenCL, chanels_in, chanels_out, window, step, units_count, group_size, groups, optimization, iBatch)) ReturnFalse; index++; if(!cResidual.Init(0, index, OpenCL, chanels_in, chanels_out, units_count, units_out, optimization, iBatch)) ReturnFalse; //--- if(!cResidual.SetGradient(cBottleneck.getGradient(), true)) ReturnFalse; if(!SetGradient(cBottleneck.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cBottleneck.FeedForward(NeuronOCL)) ReturnFalse; if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(cBottleneck.getOutput(), cResidual.getOutput(), Output, iChanelsOut, true, 0, 0, 0, 1)) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cBottleneck.AsObject())) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(cBuffer.GetOpenCL() != OpenCL || cBuffer.Total() != temp.Total()) { if(!cBuffer.BufferInitLike(temp)) ReturnFalse; } if(!NeuronOCL.SetGradient(GetPointer(cBuffer), false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cBottleneck.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iChanelsOut)) < INT_VALUE) ReturnFalse; //--- if(!cBottleneck.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResNeXtBlock::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iChanelsOut = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cBottleneck.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResNeXtBlock::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cBottleneck.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuron2DSSMOCL : public CNeuronBaseOCL { protected: uint iWindowOut; uint iUnitsOut; CNeuronBaseOCL cHiddenStates; CLayer cProjectionX_Time; CLayer cProjectionX_Variable; CNeuronConvOCL cA; CNeuronConvOCL cB_Time; CNeuronConvOCL cB_Variable; CNeuronConvOCL cC_Time; CNeuronConvOCL cC_Variable; CNeuronConvOCL cDelta_Time; CNeuronConvOCL cDelta_Variable; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForwardSSM2D(void); //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradientsSSM2D(void); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuron2DSSMOCL(void) {}; ~CNeuron2DSSMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuron2DSSMOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_out, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindowOut = window_out; iUnitsOut = units_out; //--- int index = 0; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- Projection Time cProjectionX_Time.Clear(); cProjectionX_Time.SetOpenCL(OpenCL); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, window_in, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iUnitsOut, window_in, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- Projection Variables cProjectionX_Variable.Clear(); cProjectionX_Variable.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iUnitsOut, units_in, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Variable.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- HiddenState index++; if(!cHiddenStates.Init(0, index, OpenCL, 2 * iUnitsOut * iWindowOut, optimization, iBatch)) ReturnFalse; //--- A*H index++; if(!cA.Init(0, index, OpenCL, iWindowOut, iWindowOut, 2 * iWindowOut, iUnitsOut, 2, optimization, iBatch)) ReturnFalse; if(!SumAndNormilize(cA.GetWeightsConv(), cA.GetWeightsConv(), cA.GetWeightsConv(), iWindowOut, false, 0, 0, 0, 0.05f)) ReturnFalse; cA.SetActivationFunction(MinusSoftPlus); //--- B index++; if(!cB_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cB_Time.SetActivationFunction(TANH); index++; if(!cB_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cB_Variable.SetActivationFunction(TANH); //--- C index++; if(!cC_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cC_Time.SetActivationFunction(TANH); index++; if(!cC_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cC_Variable.SetActivationFunction(TANH); //--- Delta index++; if(!cDelta_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cDelta_Time.SetActivationFunction(SoftPlus); index++; if(!cDelta_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; cDelta_Variable.SetActivationFunction(SoftPlus); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; //--- Projection Time int total = cProjectionX_Time.Total(); for(int i = 0; i < total; i++) { x_time = cProjectionX_Time.At(i); if(!x_time || !x_time.FeedForward(inp)) ReturnFalse; inp = x_time; } //--- Projection Variable inp = NeuronOCL; total = cProjectionX_Variable.Total(); for(int i = 0; i < total; i++) { x_var = cProjectionX_Variable.At(i); if(!x_var || !x_var.FeedForward(inp)) ReturnFalse; inp = x_var; } //--- if(!cA.FeedForward(cHiddenStates.AsObject())) ReturnFalse; if(!cB_Time.FeedForward(x_time) || !cB_Variable.FeedForward(x_var)) ReturnFalse; if(!cC_Time.FeedForward(x_time) || !cC_Variable.FeedForward(x_var)) ReturnFalse; if(!cDelta_Time.FeedForward(x_time) || !cDelta_Variable.FeedForward(x_var)) ReturnFalse; //--- if(!cHiddenStates.SwapOutputs()) ReturnFalse; //--- return feedForwardSSM2D(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::feedForwardSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {global_work_size[0], 1}; int kernel = def_k_SSM2D_FeedForward; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2d_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_y, getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::calcInputGradientsSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {1, global_work_size[1]}; int kernel = def_k_SSM2D_CalcHiddenGradient; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2dhg_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_ah, cA.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_time, cB_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_var, cB_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_time, cC_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_var, cC_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_time, cDelta_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_var, cDelta_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_time, x_time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_var, x_var.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_y, getGradientIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!calcInputGradientsSSM2D()) ReturnFalse; //--- Deactivation CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!x_time || !x_var) ReturnFalse; if(x_time.Activation() != None) if(!DeActivation(x_time.getOutput(), x_time.getGradient(), x_time.getGradient(), x_time.Activation())) ReturnFalse; if(x_var.Activation() != None) if(!DeActivation(x_var.getOutput(), x_var.getGradient(), x_var.getGradient(), x_var.Activation())) ReturnFalse; if(cB_Time.Activation() != None) if(!DeActivation(cB_Time.getOutput(), cB_Time.getGradient(), cB_Time.getGradient(), cB_Time.Activation())) ReturnFalse; if(cB_Variable.Activation() != None) if(!DeActivation(cB_Variable.getOutput(), cB_Variable.getGradient(), cB_Variable.getGradient(), cB_Variable.Activation())) ReturnFalse; if(cC_Time.Activation() != None) if(!DeActivation(cC_Time.getOutput(), cC_Time.getGradient(), cC_Time.getGradient(), cC_Time.Activation())) ReturnFalse; if(cC_Variable.Activation() != None) if(!DeActivation(cC_Variable.getOutput(), cC_Variable.getGradient(), cC_Variable.getGradient(), cC_Variable.Activation())) ReturnFalse; if(cDelta_Time.Activation() != None) if(!DeActivation(cDelta_Time.getOutput(), cDelta_Time.getGradient(), cDelta_Time.getGradient(), cDelta_Time.Activation())) ReturnFalse; if(cDelta_Variable.Activation() != None) if(!DeActivation(cDelta_Variable.getOutput(), cDelta_Variable.getGradient(), cDelta_Variable.getGradient(), cDelta_Variable.Activation())) ReturnFalse; if(cA.Activation() != None) if(!DeActivation(cA.getOutput(), cA.getGradient(), cA.getGradient(), cA.Activation())) ReturnFalse; //--- Gradient to projections X CBufferFloat *grad_x_time = x_time.getGradient(); CBufferFloat *grad_x_var = x_var.getGradient(); if(!x_time.SetGradient(x_time.getPrevOutput(), false) || !x_var.SetGradient(x_var.getPrevOutput(), false)) ReturnFalse; //--- B -> X if(!x_time.CalcHiddenGradients(cB_Time.AsObject()) || !SumAndNormilize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cB_Variable.AsObject()) || !SumAndNormilize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; //--- C -> X if(!x_time.CalcHiddenGradients(cC_Time.AsObject()) || !SumAndNormilize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cC_Variable.AsObject()) || !SumAndNormilize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; //--- Delta -> X if(!x_time.CalcHiddenGradients(cDelta_Time.AsObject()) || !SumAndNormilize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cDelta_Variable.AsObject()) || !SumAndNormilize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_time.SetGradient(grad_x_time, false) || !x_var.SetGradient(grad_x_var, false)) ReturnFalse; //--- Projection Variable int total = cProjectionX_Variable.Total() - 2; for(int i = total; i >= 0; i--) { x_var = cProjectionX_Variable[i]; if(!x_var || !x_var.CalcHiddenGradients(cProjectionX_Variable[i + 1])) ReturnFalse; } //--- Projection Time total = cProjectionX_Time.Total() - 2; for(int i = total; i >= 0; i--) { x_time = cProjectionX_Time[i]; if(!x_time || !x_time.CalcHiddenGradients(cProjectionX_Time[i + 1])) ReturnFalse; } //--- Projections -> inputs if(!NeuronOCL.CalcHiddenGradients(x_var.AsObject())) ReturnFalse; grad_x_time = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(x_time.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(x_time.AsObject()) || !SumAndNormilize(grad_x_time, NeuronOCL.getGradient(), grad_x_time, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(grad_x_time, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cB_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cB_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- C -> X if(!cC_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cC_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- Delta -> X if(!cDelta_Time.UpdateInputWeights(cProjectionX_Time[-1])) ReturnFalse; if(!cDelta_Variable.UpdateInputWeights(cProjectionX_Variable[-1])) ReturnFalse; //--- Projection Variable CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; int total = cProjectionX_Variable.Total() - 1; for(int i = total; i > 0; i--) { x_var = cProjectionX_Variable[i]; if(!x_var || !x_var.UpdateInputWeights(cProjectionX_Variable[i - 1])) ReturnFalse; } //--- Projection Time total = cProjectionX_Time.Total() - 1; for(int i = total; i > 0; i--) { x_time = cProjectionX_Time[i]; if(!x_time || !x_time.UpdateInputWeights(cProjectionX_Time[i - 1])) ReturnFalse; } //--- Projections -> inputs x_var = cProjectionX_Variable[0]; x_time = cProjectionX_Time[0]; if(!x_var.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!x_time.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- A*H if(!cHiddenStates.SwapOutputs() || !cA.UpdateInputWeights(cHiddenStates.AsObject()) || !cHiddenStates.SwapOutputs()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuron2DSSMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cHiddenStates.SetOpenCL(OpenCL); cProjectionX_Time.SetOpenCL(OpenCL); cProjectionX_Variable.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cB_Time.SetOpenCL(OpenCL); cB_Variable.SetOpenCL(OpenCL); cC_Time.SetOpenCL(OpenCL); cC_Variable.SetOpenCL(OpenCL); cDelta_Time.SetOpenCL(OpenCL); cDelta_Variable.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cHiddenStates.Save(file_handle)) ReturnFalse; if(!cProjectionX_Time.Save(file_handle)) ReturnFalse; if(!cProjectionX_Variable.Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cB_Time.Save(file_handle)) ReturnFalse; if(!cB_Variable.Save(file_handle)) ReturnFalse; if(!cC_Time.Save(file_handle)) ReturnFalse; if(!cC_Variable.Save(file_handle)) ReturnFalse; if(!cDelta_Time.Save(file_handle)) ReturnFalse; if(!cDelta_Variable.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cHiddenStates.AsObject())) ReturnFalse; if(!cProjectionX_Time.Load(file_handle)) ReturnFalse; if(!cProjectionX_Variable.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cB_Variable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cC_Variable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_Time.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_Variable.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuron2DSSMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cHiddenStates.Clear()) ReturnFalse; if(!cA.Clear()) ReturnFalse; if(!cB_Time.Clear()) ReturnFalse; if(!cB_Variable.Clear()) ReturnFalse; if(!cC_Time.Clear()) ReturnFalse; if(!cC_Variable.Clear()) ReturnFalse; if(!cDelta_Time.Clear()) ReturnFalse; if(!cDelta_Variable.Clear()) ReturnFalse; //--- for(int i = 0; i < cProjectionX_Time.Total(); i++) { if(!cProjectionX_Time[i] || !((CNeuronBaseOCL*)cProjectionX_Time[i]).Clear()) ReturnFalse; } for(int i = 0; i < cProjectionX_Variable.Total(); i++) { if(!cProjectionX_Variable[i] || !((CNeuronBaseOCL*)cProjectionX_Variable[i]).Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronChimera : public CNeuronBaseOCL { protected: CNeuron2DSSMOCL caSSM[2]; CNeuronConvOCL cDiscretization; CLayer cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronChimera(void) {}; ~CNeuronChimera(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronChimera; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_out * window_out, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; for(int i = 0; i < 2; i++) { if(!caSSM[i].Init(0, index, OpenCL, window_in, (i + 1)*window_out, units_in, units_out, optimization, iBatch)) ReturnFalse; index++; } if(!cDiscretization.Init(0, index, OpenCL, 2 * window_out, 2 * window_out, window_out, units_out, 1, optimization, iBatch)) ReturnFalse; cDiscretization.SetActivationFunction(None); //--- Residual cResidual.Clear(); cResidual.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, window_in, optimization, iBatch) || !cResidual.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, units_out, window_in, 1, optimization, iBatch) || !cResidual.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_in, units_out, optimization, iBatch) || !cResidual.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, window_out, units_out, 1, optimization, iBatch) || !cResidual.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); //--- if(!SetGradient(caSSM[0].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::feedForward(CNeuronBaseOCL * NeuronOCL) { for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].FeedForward(NeuronOCL)) ReturnFalse; } if(!cDiscretization.FeedForward(caSSM[1].AsObject())) ReturnFalse; //--- CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- if(!SumAndNormilize(caSSM[0].getOutput(), cDiscretization.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormilize(Output, current.getOutput(), Output, cDiscretization.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cDiscretization.getOutput(), cDiscretization.getGradient(), Gradient, cDiscretization.Activation())) ReturnFalse; if(!caSSM[1].CalcHiddenGradients(cDiscretization.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *residual = cResidual[-1]; if(!residual) ReturnFalse; if(!DeActivation(residual.getOutput(), residual.getGradient(), Gradient, residual.Activation())) ReturnFalse; for(int i = cResidual.Total() - 2; i >= 0; i--) { residual = cResidual[i]; if(!residual || !residual.CalcHiddenGradients(cResidual[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(caSSM[0].AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(residual.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(caSSM[1].AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients((CObject*)residual) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cDiscretization.UpdateInputWeights(caSSM[1].AsObject())) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; } //--- CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.UpdateInputWeights(inp)) ReturnFalse; inp = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!caSSM[i].Save(file_handle)) ReturnFalse; if(!cDiscretization.Save(file_handle)) ReturnFalse; //--- if(!cResidual.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!LoadInsideLayer(file_handle, caSSM[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDiscretization.AsObject())) ReturnFalse; //--- if(!cResidual.Load(file_handle)) ReturnFalse; //--- if(!SetGradient(caSSM[0].getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronChimera::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < caSSM.Size(); i++) caSSM[i].SetOpenCL(OpenCL); cDiscretization.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimera::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- for(uint i = 0; i < caSSM.Size(); i++) if(!caSSM[i].Clear()) ReturnFalse; if(!cDiscretization.Clear()) ReturnFalse; //--- for(int i = 0; i < cResidual.Total(); i++) if(!((CNeuronBaseOCL*)cResidual[i]).Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMoT : public CNeuronMHAttentionPooling { protected: CNeuronConvOCL cNodesTokenizer; CNeuronConvOCL cEdgesTokenizer; CNeuronConvOCL cSubGraphsTokenizer; CLayer cUnitarSubGraphsTokenizer; CNeuronBaseOCL cConcatenate; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMoT(void) {}; ~CNeuronMoT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMoT; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, window, units_count, 4, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cNodesTokenizer.Init(0, index, OpenCL, iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cNodesTokenizer.SetActivationFunction(SoftPlus); //--- index++; if(!cEdgesTokenizer.Init(0, index, OpenCL, 2 * iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cEdgesTokenizer.SetActivationFunction(SoftPlus); //--- index++; if(!cSubGraphsTokenizer.Init(0, index, OpenCL, 3 * iWindow, iWindow, iWindow, iUnits, 1, optimization, iBatch)) ReturnFalse; cSubGraphsTokenizer.SetActivationFunction(SoftPlus); //--- cUnitarSubGraphsTokenizer.Clear(); cUnitarSubGraphsTokenizer.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, iUnits, iWindow, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 3, 1, 1, iUnits, iWindow, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch) || !cUnitarSubGraphsTokenizer.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- index++; if(!cConcatenate.Init(0, index, OpenCL, 4 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatenate.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cNodesTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!cEdgesTokenizer.FeedForward(NeuronOCL)) ReturnFalse; if(!cSubGraphsTokenizer.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL, *current = NULL; for(int i = 0; i < cUnitarSubGraphsTokenizer.Total(); i++) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- if(!Concat(cNodesTokenizer.getOutputIndex(), cEdgesTokenizer.getOutputIndex(), cSubGraphsTokenizer.getOutputIndex(), current.getOutputIndex(), cConcatenate.getOutputIndex(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- return CNeuronMHAttentionPooling::feedForward(cConcatenate.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcatenate.AsObject())) ReturnFalse; //--- CNeuronBaseOCL *current = cUnitarSubGraphsTokenizer[-1]; if(!current || !DeConcat(cNodesTokenizer.getGradient(), cEdgesTokenizer.getGradient(), cSubGraphsTokenizer.getGradient(), current.getGradient(), cConcatenate.getGradient(), iWindow, iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- if(current.Activation() != None && !DeActivation(current.getOutput(), current.getGradient(), current.getGradient(), current.Activation())) ReturnFalse; for(int i = cUnitarSubGraphsTokenizer.Total() - 2; i >= 0; i--) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.CalcHiddenGradients(cUnitarSubGraphsTokenizer[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(current.AsObject())) ReturnFalse; //--- CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(current.getPrevOutput(), false)) ReturnFalse; //--- if(cNodesTokenizer.Activation() != None && !DeActivation(cNodesTokenizer.getOutput(), cNodesTokenizer.getGradient(), cNodesTokenizer.getGradient(), cNodesTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNodesTokenizer.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(cEdgesTokenizer.Activation() != None && !DeActivation(cEdgesTokenizer.getOutput(), cEdgesTokenizer.getGradient(), cEdgesTokenizer.getGradient(), cEdgesTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEdgesTokenizer.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(cSubGraphsTokenizer.Activation() != None && !DeActivation(cSubGraphsTokenizer.getOutput(), cSubGraphsTokenizer.getGradient(), cSubGraphsTokenizer.getGradient(), cSubGraphsTokenizer.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSubGraphsTokenizer.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronMHAttentionPooling::updateInputWeights(cConcatenate.AsObject())) ReturnFalse; //--- if(!cNodesTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cEdgesTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSubGraphsTokenizer.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL, *current = NULL; for(int i = 0; i < cUnitarSubGraphsTokenizer.Total(); i++) { current = cUnitarSubGraphsTokenizer[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cNodesTokenizer.Save(file_handle)) ReturnFalse; if(!cEdgesTokenizer.Save(file_handle)) ReturnFalse; if(!cSubGraphsTokenizer.Save(file_handle)) ReturnFalse; if(!cUnitarSubGraphsTokenizer.Save(file_handle)) ReturnFalse; if(!cConcatenate.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoT::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cNodesTokenizer.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEdgesTokenizer.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSubGraphsTokenizer.AsObject())) ReturnFalse; if(!cUnitarSubGraphsTokenizer.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMoT::SetOpenCL(COpenCLMy * obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cNodesTokenizer.SetOpenCL(OpenCL); cEdgesTokenizer.SetOpenCL(OpenCL); cSubGraphsTokenizer.SetOpenCL(OpenCL); cUnitarSubGraphsTokenizer.SetOpenCL(OpenCL); cConcatenate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronChimeraPlus : public CNeuronChimera { protected: CNeuron2DSSMOCL cSSMPlus; CLayer cDiscretizationPlus; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronChimeraPlus(void) {}; ~CNeuronChimeraPlus(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronChimeraPlus; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronChimera::Init(numOutputs, myIndex, open_cl, window_in, window_out, units_in, units_out, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cSSMPlus.Init(0, index, OpenCL, window_in, window_out, units_in, 2 * units_out, optimization, iBatch)) ReturnFalse; //--- CNeuronTransposeOCL *transp = NULL; CNeuronConvOCL *conv = NULL; cDiscretizationPlus.Clear(); cDiscretizationPlus.SetOpenCL(OpenCL); //--- index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, 2 * units_out, window_out, optimization, iBatch) || !cDiscretizationPlus.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 2 * units_out, 2 * units_out, units_out, window_out, 1, optimization, iBatch) || !cDiscretizationPlus.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_out, units_out, optimization, iBatch) || !cDiscretizationPlus.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::feedForward(CNeuronBaseOCL * NeuronOCL) { for(uint i = 0; i < caSSM.Size(); i++) { if(!caSSM[i].FeedForward(NeuronOCL)) ReturnFalse; } if(!cSSMPlus.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!cDiscretization.FeedForward(caSSM[1].AsObject())) ReturnFalse; CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- inp = NeuronOCL; for(int i = 0; i < cResidual.Total(); i++) { current = cResidual[i]; if(!current || !current.FeedForward(inp)) ReturnFalse; inp = current; } //--- inp = cDiscretizationPlus[-1]; if(!SumAndNormilize(caSSM[0].getOutput(), cDiscretization.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormilize(Output, inp.getOutput(), Output, 1, false, 0, 0, 0, 1) || !SumAndNormilize(Output, current.getOutput(), Output, cDiscretization.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronChimera::calcInputGradients(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *current = cDiscretizationPlus[-1]; if(!current || !DeActivation(current.getOutput(), current.getGradient(), Gradient, current.Activation())) ReturnFalse; for(int i = cDiscretizationPlus.Total() - 2; i >= 0; i--) { current = cDiscretizationPlus[i]; if(!current || !current.CalcHiddenGradients(cDiscretizationPlus[i + 1])) ReturnFalse; } if(!cSSMPlus.CalcHiddenGradients(current.AsObject())) ReturnFalse; //--- current = cResidual[0]; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(current.getGradient(), false) || !NeuronOCL.CalcHiddenGradients(cSSMPlus.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronChimera::updateInputWeights(NeuronOCL)) ReturnFalse; if(!cSSMPlus.UpdateInputWeights(NeuronOCL)) ReturnFalse; CNeuronBaseOCL *current = NULL, *inp = cSSMPlus.AsObject(); for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.UpdateInputWeights(inp.AsObject())) ReturnFalse; inp = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Save(const int file_handle) { if(!CNeuronChimera::Save(file_handle)) ReturnFalse; if(!cSSMPlus.Save(file_handle)) ReturnFalse; if(!cDiscretizationPlus.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Load(const int file_handle) { if(!CNeuronChimera::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSSMPlus.AsObject())) ReturnFalse; if(!cDiscretizationPlus.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronChimeraPlus::SetOpenCL(COpenCLMy * obj) { CNeuronChimera::SetOpenCL(obj); cSSMPlus.SetOpenCL(OpenCL); cDiscretizationPlus.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChimeraPlus::Clear(void) { if(!CNeuronChimera::Clear()) ReturnFalse; if(!cSSMPlus.Clear()) ReturnFalse; CNeuronBaseOCL *current = NULL; for(int i = 0; i < cDiscretizationPlus.Total(); i++) { current = cDiscretizationPlus[i]; if(!current || !current.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHypridDecoder : public CNeuronHidformer { protected: CNeuronChimeraPlus cChimera; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHypridDecoder(void) {}; ~CNeuronHypridDecoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHypridDecoder; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint layers, uint stack_size, uint nactions, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronHidformer::Init(numOutputs, myIndex, open_cl, window_key, window_key, nactions, heads, layers, stack_size, nactions, optimization_type, batch)) ReturnFalse; if(!cChimera.Init(0, 0, OpenCL, window, window_key, units_count, nactions, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cChimera.FeedForward(NeuronOCL)) ReturnFalse; return CNeuronHidformer::feedForward(cChimera.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronHidformer::calcInputGradients(cChimera.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cChimera.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!CNeuronHidformer::updateInputWeights(cChimera.AsObject())) ReturnFalse; //--- if(!cChimera.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Save(const int file_handle) { if(!CNeuronHidformer::Save(file_handle)) ReturnFalse; if(!cChimera.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Load(const int file_handle) { if(!CNeuronHidformer::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cChimera.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHypridDecoder::SetOpenCL(COpenCLMy * obj) { CNeuronHidformer::SetOpenCL(obj); cChimera.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHypridDecoder::Clear(void) { if(!CNeuronHidformer::Clear()) ReturnFalse; if(!cChimera.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAttraos : public CNeuronBaseOCL { protected: CNeuronBaseOCL cOne; CNeuronBaseOCL cX_norm; CNeuronConvOCL cA; CNeuronConvOCL cX_proj; CNeuronBaseOCL cDelta; CNeuronBaseOCL cB; CNeuronBaseOCL cC; CNeuronConvOCL cD; CNeuronBaseOCL cH; CNeuronConvOCL cDelta_proj; CNeuronBaseOCL cDeltaA; CNeuronBaseOCL cDeltaB; CNeuronBaseOCL cDeltaBX; CNeuronBaseOCL cDeltaH; CNeuronBaseOCL cHS; //--- virtual bool PScan(void); virtual bool PScanCalcGradient(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAttraos(void) {}; ~CNeuronAttraos(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronAttraos; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; if(!cOne.Init(0, index, OpenCL, units_count, optimization, iBatch)) ReturnFalse; if(!cOne.getOutput().Fill(1)) ReturnFalse; cOne.SetActivationFunction(None); //--- index++; if(!cA.Init(0, index, OpenCL, 1, 1, window * window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cA.SetActivationFunction(MinusSoftPlus); CBufferFloat *w = cA.GetWeightsConv(); if(!w || !w.Fill(0)) ReturnFalse; //--- index++; if(!cD.Init(0, index, OpenCL, 1, 1, window, units_count, 1, optimization, iBatch)) ReturnFalse; cD.SetActivationFunction(None); w = cD.GetWeightsConv(); if(!w || !w.Fill(1)) ReturnFalse; //--- index++; if(!cX_norm.Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cX_norm.SetActivationFunction(None); index++; if(!cX_proj.Init(0, index, OpenCL, window, window, 4 * window_key, units_count, 1, optimization, iBatch)) ReturnFalse; cX_proj.SetActivationFunction(None); //--- index++; if(!cDelta.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cDelta.SetActivationFunction(None); index++; if(!cB.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); index++; if(!cDelta_proj.Init(0, index, OpenCL, window_key, window_key, window, units_count, 1, optimization, iBatch)) ReturnFalse; cDelta_proj.SetActivationFunction(SoftPlus); //--- index++; if(!cDeltaA.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaA.SetActivationFunction(None); index++; if(!cDeltaB.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaB.SetActivationFunction(None); index++; if(!cDeltaBX.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaBX.SetActivationFunction(None); index++; if(!cDeltaH.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaH.SetActivationFunction(None); index++; if(!cHS.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cHS.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::PScan(void) { if(!OpenCL) ReturnFalse; uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {units, window * window_key}; uint local_work_size[2] = {units, 1}; //--- uint kernel = def_k_PScan; setBuffer(kernel, def_k_psc_A, cDeltaA.getOutputIndex()) setBuffer(kernel, def_k_psc_X, cDeltaBX.getOutputIndex()) setBuffer(kernel, def_k_psc_H, cDeltaH.getOutputIndex()) setBuffer(kernel, def_k_psc_X_out, cHS.getOutputIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::PScanCalcGradient(void) { if(!OpenCL) ReturnFalse; uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {units, window * window_key}; uint local_work_size[2] = {units, 1}; //--- uint kernel = def_k_PScan_CalcHiddenGradient; setBuffer(kernel, def_k_pscgr_A, cDeltaA.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_A, cDeltaA.getGradientIndex()) setBuffer(kernel, def_k_pscgr_X, cDeltaBX.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_X, cDeltaBX.getGradientIndex()) setBuffer(kernel, def_k_pscgr_H, cDeltaH.getOutputIndex()) setBuffer(kernel, def_k_pscgr_grad_H, cDeltaH.getGradientIndex()) setBuffer(kernel, def_k_pscgr_grad_X_out, cHS.getGradientIndex()) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- if(!cA.FeedForward(cOne.AsObject())) // (Units, Window, WindowKey) ReturnFalse; if(!cD.FeedForward(cOne.AsObject())) // (Units, Window)) ReturnFalse; //--- if(!NeuronOCL || !SumAndNormilize(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cX_norm.getOutput(), window, true, 0, 0, 0, 0.5f)) ReturnFalse; if(!cX_proj.FeedForward(cX_norm.AsObject())) // (Units, 4*WindowKey) ReturnFalse; //--- if(!DeConcat(cDelta.getOutput(), cB.getOutput(), cC.getOutput(), cH.getOutput(), cX_proj.getOutput(), window_key, window_key, window_key, window_key, units)) // 4*(Units, WindowKey) ReturnFalse; if(!cDelta_proj.FeedForward(cDelta.AsObject())) // (Units, Window) ReturnFalse; if(!DiagMatMul(cDelta_proj.getOutput(), cA.getOutput(), cDeltaA.getOutput(), window, window_key, units, SoftPlus)) // (Units, Window, WindowKey) ReturnFalse; if(!MatMul(cDelta_proj.getOutput(), cB.getOutput(), cDeltaB.getOutput(), window, 1, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!DiagMatMul(cX_norm.getOutput(), cDeltaB.getOutput(), cDeltaBX.getOutput(), window, window_key, units, None)) // (Units, Window, WindowKey) ReturnFalse; if(!MatMul(cDelta_proj.getOutput(), cH.getOutput(), cDeltaH.getOutput(), window, 1, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!PScan()) ReturnFalse; if(!MatMul(cHS.getOutput(), cC.getOutput(), Output, window, window_key, 1, units)) // (Units, Window, 1) ReturnFalse; if(!ElementMult(cD.getOutput(), cX_norm.getOutput(), PrevOutput)) // (Units, Window)) ReturnFalse; if(!SumAndNormilize(Output, PrevOutput, Output, window, false, 0, 0, 0, 1)) // (Units, Window)) ReturnFalse; if(!SumAndNormilize(Output, NeuronOCL.getOutput(), Output, window, false, 0, 0, 0, 1)) // (Units, Window)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units = cD.GetUnits(); //--- if(!ElementMultGrad(cD.getOutput(), cD.getGradient(), cX_norm.getOutput(), cX_norm.getPrevOutput(), Gradient, cD.Activation(), None)) // (Units, Window)) ReturnFalse; if(!MatMulGrad(cHS.getOutput(), cHS.getGradient(), cC.getOutput(), cC.getGradient(), Gradient, window, window_key, 1, units)) // (Units, Window, 1) ReturnFalse; if(cHS.Activation() != None) { if(!DeActivation(cHS.getOutput(), cHS.getGradient(), cHS.getGradient(), cHS.Activation())) ReturnFalse; } if(cC.Activation() != None) { if(!DeActivation(cC.getOutput(), cC.getGradient(), cC.getGradient(), cC.Activation())) ReturnFalse; } //--- if(!PScanCalcGradient()) ReturnFalse; //--- if(!MatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getGradient(), cH.getOutput(), cH.getGradient(), cDeltaH.getGradient(), window, 1, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!DiagMatMulGrad(cX_norm.getOutput(), cX_norm.getGradient(), cDeltaB.getOutput(), cDeltaB.getGradient(), cDeltaBX.getGradient(), window, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!SumAndNormilize(cX_norm.getGradient(), cX_norm.getPrevOutput(), cX_norm.getPrevOutput(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!MatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getPrevOutput(), cB.getOutput(), cB.getGradient(), cDeltaB.getGradient(), window, 1, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!SumAndNormilize(cDelta_proj.getGradient(), cDelta_proj.getPrevOutput(), cDelta_proj.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!DeActivation(cDeltaA.getOutput(), cDeltaA.getGradient(), cDeltaA.getGradient(), SoftPlus)) ReturnFalse; if(!DiagMatMulGrad(cDelta_proj.getOutput(), cDelta_proj.getPrevOutput(), cA.getOutput(), cA.getGradient(), cDeltaA.getGradient(), window, window_key, units)) // (Units, Window, WindowKey) ReturnFalse; if(!SumAndNormilize(cDelta_proj.getGradient(), cDelta_proj.getPrevOutput(), cDelta_proj.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(cDelta_proj.Activation() != None) { if(!DeActivation(cDelta_proj.getOutput(), cDelta_proj.getGradient(), cDelta_proj.getGradient(), cDelta_proj.Activation())) ReturnFalse; } //--- if(!cDelta.CalcHiddenGradients(cDelta_proj.AsObject())) ReturnFalse; //--- if(!Concat(cDelta.getGradient(), cB.getGradient(), cC.getGradient(), cH.getGradient(), cX_proj.getGradient(), window_key, window_key, window_key, window_key, units)) // 4*(Units, WindowKey) ReturnFalse; //--- if(!cX_norm.CalcHiddenGradients(cX_proj.AsObject())) ReturnFalse; if(!SumAndNormilize(cX_norm.getGradient(), cX_norm.getPrevOutput(), cX_norm.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cX_norm.getGradient(), Gradient, cX_norm.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cX_norm.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { //--- if(!cA.UpdateInputWeights(cOne.AsObject())) // (Units, Window, WindowKey) ReturnFalse; if(!cD.UpdateInputWeights(cOne.AsObject())) // (Units, Window)) ReturnFalse; if(!cX_proj.UpdateInputWeights(NeuronOCL)) // (Units, 4*WindowKey) ReturnFalse; if(!cDelta_proj.UpdateInputWeights(cDelta.AsObject())) // (Units, Window) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cA.Save(file_handle)) ReturnFalse; if(!cD.Save(file_handle)) ReturnFalse; if(!cX_proj.Save(file_handle)) ReturnFalse; if(!cDelta_proj.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttraos::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cX_proj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_proj.AsObject())) ReturnFalse; //--- uint window = cX_proj.GetWindow(); uint window_key = cX_proj.GetFilters() / 4; uint units_count = cD.GetUnits(); //--- if(!cOne.Init(0, 0, OpenCL, units_count, optimization, iBatch)) ReturnFalse; if(!cOne.getOutput().Fill(1)) ReturnFalse; cOne.SetActivationFunction(None); int index = 3; if(!cX_norm.Init(0, index, OpenCL, window * units_count, optimization, iBatch)) ReturnFalse; cX_norm.SetActivationFunction(None); index += 2; if(!cDelta.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cDelta.SetActivationFunction(None); index++; if(!cB.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, window_key * units_count, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); index += 2; if(!cDeltaA.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaA.SetActivationFunction(None); index++; if(!cDeltaB.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaB.SetActivationFunction(None); index++; if(!cDeltaBX.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaBX.SetActivationFunction(None); index++; if(!cDeltaH.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cDeltaH.SetActivationFunction(None); index++; if(!cHS.Init(0, index, OpenCL, window * window_key * units_count, optimization, iBatch)) ReturnFalse; cHS.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAttraos::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cX_proj.SetOpenCL(OpenCL); cDelta.SetOpenCL(OpenCL); cB.SetOpenCL(OpenCL); cC.SetOpenCL(OpenCL); cD.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cDelta_proj.SetOpenCL(OpenCL); cDeltaA.SetOpenCL(OpenCL); cDeltaB.SetOpenCL(OpenCL); cDeltaBX.SetOpenCL(OpenCL); cDeltaH.SetOpenCL(OpenCL); cHS.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTopKGates : public CNeuronSoftMaxOCL { protected: int iK; CBufferFloat cbNoise; CNeuronConvOCL cProjection; CNeuronBaseOCL cGates; //--- virtual bool TopKgates(void); virtual bool TopKgatesGradient(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTopKGates(void) {}; ~CNeuronTopKGates(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint gates, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTopKGates; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetGates(void) const { return cProjection.GetFilters() / 2; } virtual uint GetUnits(void) const { return cProjection.GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::TopKgates(void) { if(!OpenCL) ReturnFalse; uint window = cProjection.GetFilters() / 2; uint units = cProjection.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {window, units}; uint local_work_size[2] = {window, 1}; //--- uint kernel = def_k_TopKgates; setBuffer(kernel, def_k_topK_inputs, cProjection.getOutputIndex()) setBuffer(kernel, def_k_topK_noises, cbNoise.GetIndex()) setBuffer(kernel, def_k_topK_gates, cGates.getOutputIndex()) setArgument(kernel, def_k_topK_k, iK) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::TopKgatesGradient(void) { if(!OpenCL) ReturnFalse; uint window = cProjection.GetFilters() / 2; uint units = cProjection.GetUnits(); //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {window, units}; //--- uint kernel = def_k_TopKgatesGrad; setBuffer(kernel, def_k_topKgr_inputs, cProjection.getOutputIndex()) setBuffer(kernel, def_k_topKgr_grad_inputs, cProjection.getGradientIndex()) setBuffer(kernel, def_k_topKgr_noises, cbNoise.GetIndex()) setBuffer(kernel, def_k_topKgr_gates, cGates.getOutputIndex()) setBuffer(kernel, def_k_topKgr_grad_gates, cGates.getGradientIndex()) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint gates, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, gates * units_count, optimization_type, batch)) ReturnFalse; SetHeads(units_count); if(!cProjection.Init(0, 0, OpenCL, window, window, 2 * gates, units_count, 1, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(None); if(!cbNoise.BufferInit(Neurons(), 0) || !cbNoise.BufferCreate(OpenCL)) ReturnFalse; if(!cGates.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cGates.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; //--- if(bTrain) { double random[]; if(!MathRandomNormal(0, 1, Neurons(), random)) ReturnFalse; if(!cbNoise.AssignArray(random)) ReturnFalse; if(!cbNoise.BufferWrite()) ReturnFalse; } else if(!cbNoise.Fill(0)) ReturnFalse; //--- if(!TopKgates()) ReturnFalse; //--- return CNeuronSoftMaxOCL::feedForward(cGates.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSoftMaxOCL::calcInputGradients(cGates.AsObject())) ReturnFalse; //--- if(!TopKgatesGradient()) ReturnFalse; //--- return NeuronOCL.CalcHiddenGradients(cProjection.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { return cProjection.UpdateInputWeights(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iK) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTopKGates::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iK = FileReadInteger(file_handle); //--- if(!cbNoise.BufferInit(Neurons(), 0) || !cbNoise.BufferCreate(OpenCL)) ReturnFalse; if(!cGates.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cGates.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTopKGates::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cGates.SetOpenCL(OpenCL); cbNoise.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMoE : public CNeuronBaseOCL { protected: CNeuronTopKGates cGates; CNeuronConvOCL cXProj; CLayer cExperts; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMoE(void) {}; ~CNeuronMoE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMoE; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual void TrainMode(bool flag) { bTrain = flag; cGates.TrainMode(bTrain); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count, optimization_type, batch)) ReturnFalse; int index = 0; if(!cGates.Init(0, index, OpenCL, window, units_count, experts, top_k, optimization, iBatch)) ReturnFalse; //--- index++; if(!cXProj.Init(0, index, OpenCL, window, window, window_out, units_count, optimization, iBatch)) ReturnFalse; cXProj.SetActivationFunction(None); //--- cExperts.Clear(); cExperts.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronTransposeRCDOCL *transp = NULL; //--- index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, window_out * experts, units_count, 1, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, units_count, experts, window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_out, window_out, window_out, units_count, experts, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, experts, units_count, window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cGates.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL; int total = cExperts.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *neuron = cExperts[i]; if(!neuron || !neuron.FeedForward(prev)) ReturnFalse; prev = neuron; } if(!MatMul(cGates.getOutput(), prev.getOutput(), getOutput(), 1, cGates.GetGates(), Neurons() / cGates.GetUnits(), cGates.GetUnits())) ReturnFalse; //--- if(!cXProj.FeedForward(NeuronOCL) || !SumAndNormilize(cXProj.getOutput(), Output, Output, cXProj.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL *neuron = cExperts[-1]; if(!MatMulGrad(cGates.getOutput(), cGates.getGradient(), neuron.getOutput(), neuron.getGradient(), getGradient(), 1, cGates.GetGates(), Neurons() / cGates.GetUnits(), cGates.GetUnits())) ReturnFalse; //--- if(cGates.Activation() != None) if(!DeActivation(cGates.getOutput(), cGates.getGradient(), cGates.getGradient(), cGates.Activation())) ReturnFalse; if(neuron.Activation() != None) if(!DeActivation(neuron.getOutput(), neuron.getGradient(), neuron.getGradient(), neuron.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cGates.AsObject())) ReturnFalse; //--- for(int i = cExperts.Total() - 2; i >= 0; i--) { neuron = cExperts[i]; if(!neuron || !neuron.CalcHiddenGradients(cExperts[i + 1])) ReturnFalse; } CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(neuron.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- if(!DeActivation(cXProj.getOutput(), cXProj.getGradient(), Gradient, cXProj.Activation()) || !NeuronOCL.CalcHiddenGradients(cXProj.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cGates.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cXProj.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL *prev = NeuronOCL; int total = cExperts.Total(); for(int i = 0; i < total; i++) { CNeuronBaseOCL *neuron = cExperts[i]; if(!neuron || !neuron.UpdateInputWeights(prev)) ReturnFalse; prev = neuron; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cGates.Save(file_handle)) ReturnFalse; if(!cXProj.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGates.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cXProj.AsObject())) ReturnFalse; if(!cExperts.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMoE::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cGates.SetOpenCL(OpenCL); cXProj.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronChanelMask : public CNeuronBaseOCL { //--- protected: uint iUnits; uint iFFTdimension; CBufferFloat cbFFTReal; CBufferFloat cbFFTImag; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, bool reverse = false); virtual bool Mask(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) { return true; } public: CNeuronChanelMask(void) {}; ~CNeuronChanelMask(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronChanelMask; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window <= 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * units_count, optimization_type, batch)) ReturnFalse; //--- Save constants iUnits = units_count; activation = None; //--- Calculate FFT dimension int power = int(MathLog(window) / M_LN2); if(MathPow(2, power) != window) power++; iFFTdimension = uint(MathPow(2, power)); //--- if(!cbFFTReal.BufferInit(iFFTdimension * iUnits, 0) || !cbFFTReal.BufferCreate(OpenCL)) ReturnFalse; if(!cbFFTImag.BufferInit(iFFTdimension * iUnits, 0) || !cbFFTImag.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {iUnits}; int kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / iUnits)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / iUnits)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Mask(void) { uint global_work_offset[2] = {0, 0 }; uint global_work_size[2] = {iUnits, iUnits}; uint local_work_size[2] = {1, iUnits}; //--- int kernel = def_k_MaskByDistance; setBuffer(kernel, def_k_maskDist_buf_real, cbFFTReal.GetIndex()) setBuffer(kernel, def_k_maskDist_buf_imag, cbFFTImag.GetIndex()) setBuffer(kernel, def_k_maskDist_mask, getOutputIndex()) setArgument(kernel, def_k_maskDist_dimension, int(iFFTdimension)) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!FFT(NeuronOCL.getOutput(), NULL, GetPointer(cbFFTReal), GetPointer(cbFFTImag), false)) ReturnFalse; //--- return Mask(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iFFTdimension)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronChanelMask::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iFFTdimension = (uint)FileReadInteger(file_handle); //--- SetOpenCL(OpenCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronChanelMask::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cbFFTReal.BufferFree(); cbFFTImag.BufferFree(); cbFFTReal.BufferInit(iFFTdimension * iUnits, 0); cbFFTImag.BufferInit(iFFTdimension * iUnits, 0); cbFFTReal.BufferCreate(OpenCL); cbFFTImag.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronDUET : public CNeuronTransposeOCL { protected: uint iWindowKey; uint iHeads; //--- CNeuronTransposeOCL cTranspose; CNeuronMoE cExperts; CNeuronConvOCL cQKV; CNeuronBaseOCL cQ; CNeuronBaseOCL cKV; CNeuronChanelMask cMask; CBufferFloat cbScores; CNeuronBaseOCL cMHAttentionOut; CNeuronConvOCL cPooling; CNeuronBaseOCL cResidual; CNeuronMHFeedForward cFeedForward; //--- virtual bool AttentionOut(void); virtual bool AttentionInsideGradients(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer); public: CNeuronDUET(void) {}; ~CNeuronDUET(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint heads, uint units_out, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronDUET; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- //virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual void TrainMode(bool flag) { bTrain = flag; cExperts.TrainMode(bTrain); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint heads, uint units_out, uint experts, uint top_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, window, units_out, optimization_type, batch)) ReturnFalse; //--- iWindowKey = MathMax(window_key, 1); iHeads = MathMax(heads, 1); //--- int index = 0; if(!cTranspose.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cExperts.Init(0, index, OpenCL, units_count, units_out, window, experts, top_k, optimization, iBatch)) ReturnFalse; index++; if(!cMask.Init(0, index, OpenCL, units_count, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQKV.Init(0, index, OpenCL, units_out, units_out, iHeads * iWindowKey * 3, window, 1, optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cbScores.BufferInit(cMask.Neurons()*iHeads, 0) || !cbScores.BufferCreate(OpenCL)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cPooling.Init(0, index, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, units_out, window, 1, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- index++; if(!cFeedForward.Init(0, index, OpenCL, units_out, 4 * units_out, window, 1, heads, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount/*Q units*/, iCount/*K units*/, iHeads}; uint local_work_size[3] = {1, iCount, 1}; ResetLastError(); int kernel = def_k_MaskAttention; setBuffer(kernel, def_k_maskatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskatt_scores, cbScores.GetIndex()) setBuffer(kernel, def_k_maskatt_out, cMHAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_maskatt_masks, cMask.getOutputIndex()) setArgument(kernel, def_k_maskatt_dimension, (int)iWindowKey) setArgument(kernel, def_k_maskatt_heads_kv, (int)iHeads) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iCount, iWindowKey, iHeads}; ResetLastError(); int kernel = def_k_MaskAttentionGradients; setBuffer(kernel, def_k_maskattgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_maskattgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_maskattgr_scores, cbScores.GetIndex()) setBuffer(kernel, def_k_maskattgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_maskattgr_kunits, (int)iCount) setArgument(kernel, def_k_maskattgr_heads_kv, (int)iHeads) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!cExperts.FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cMask.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- if(!cQKV.FeedForward(cExperts.AsObject())) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), iWindowKey, 2 * iWindowKey, cQKV.GetUnits())) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormilize(cExperts.getOutput(), cPooling.getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward.FeedForward(cResidual.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cFeedForward.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cFeedForward.AsObject())) ReturnFalse; if(!cPooling.CalcHiddenGradients(cFeedForward.AsObject())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), iWindowKey, 2 * iWindowKey, iCount)) ReturnFalse; if(cQKV.Activation() != None) if(!DeActivation(cQKV.getOutput(), cQKV.getGradient(), cQKV.getGradient(), cQKV.Activation())) ReturnFalse; if(!cExperts.CalcHiddenGradients(cQKV.AsObject()) || !DeActivation(cExperts.getOutput(), cExperts.getPrevOutput(), cPooling.getGradient(), cExperts.Activation()) || !SumAndNormilize(cExperts.getGradient(), cExperts.getPrevOutput(), cExperts.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cExperts.AsObject())) ReturnFalse; //--- return prevLayer.CalcHiddenGradients(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cExperts.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- if(!cQKV.UpdateInputWeights(cExperts.AsObject())) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cFeedForward.UpdateInputWeights(cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, iHeads) < INT_VALUE) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; if(!cFeedForward.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronDUET::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cExperts.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward.AsObject())) ReturnFalse; //--- int index = 4; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons() * 2, optimization, iBatch)) ReturnFalse; if(!cbScores.BufferInit(cMask.Neurons()*iHeads, 0) || !cbScores.BufferCreate(OpenCL)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronDUET::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); cQKV.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); cFeedForward.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cbScores.BufferCreate(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiWindowsConvOCL : public CNeuronConvOCL { protected: int aiWindows[]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronMultiWindowsConvOCL(void) { activation = SoftPlus; iWindow = -1; } ~CNeuronMultiWindowsConvOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &windows[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMultiWindowsConvOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); //--- virtual int GetWindow(uint index) { if(index >= aiWindows.Size()) return 0; return aiWindows[index]; } virtual uint GetUnits(void) const override { return CNeuronConvOCL::GetUnits() / aiWindows.Size(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint &windows[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(windows.Size() <= 0 || ArrayCopy(aiWindows, windows, 0, 0, windows.Size()) < int(windows.Size())) ReturnFalse; int window = 0; for(uint i = 0; i < aiWindows.Size(); i++) { window += aiWindows[i] + 1; if(aiWindows[i] <= 0) ReturnFalse; } window--; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count * aiWindows.Size()*variables, 1, ADAM, batch)) ReturnFalse; //--- iVariables = variables; int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {Neurons() / iVariables, iVariables}; ResetLastError(); int kernel = def_k_FeedForwardMultWinConv; setBuffer(kernel, def_k_ffmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmwc_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmwc_windows_in, iWindow) setArgument(kernel, def_k_ffmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmwc_window_out, iWindowOut) setArgument(kernel, def_k_ffmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_ffmwc_activation, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG vector res; if(!getOutputVal(res)) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {NeuronOCL.Neurons() / iVariables, iVariables}; ResetLastError(); int kernel = def_k_CalcHiddenGradientMultWinConv; setBuffer(kernel, def_k_chgmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgmwc_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgmwc_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_chgmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgmwc_windows_in, iWindow) setArgument(kernel, def_k_chgmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgmwc_window_out, iWindowOut) setArgument(kernel, def_k_chgmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_chgmwc_activation, NeuronOCL.Activation()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {WeightsConv.Total(), iVariables}; uint local_work_size[2] = {1, iVariables}; ResetLastError(); int kernel = def_k_UpdateWeightsMultWinConvAdam; setBuffer(kernel, def_k_uwmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmwc_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmwc_windows_in, iWindow) setBuffer(kernel, def_k_uwmwc_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmwc_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_uwmwc_window_out, iWindowOut) setArgument(kernel, def_k_uwmwc_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_uwmwc_l, lr) setArgument(kernel, def_k_uwmwc_b1, b1) setArgument(kernel, def_k_uwmwc_b2, b2) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; uint total = aiWindows.Size(); if(FileWriteInteger(file_handle, (int)total, INT_VALUE) < INT_VALUE) ReturnFalse; for(int i = 0; i < (int)total; i++) if(FileWriteInteger(file_handle, aiWindows[i]) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvOCL::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- ArrayFree(aiWindows); if(iWindow >= 0) OpenCL.BufferFree(iWindow); //--- int total = FileReadInteger(file_handle); if(ArrayResize(aiWindows, total) != total) ReturnFalse; for(int i = 0; i < total; i++) { aiWindows[i] = FileReadInteger(file_handle); if(aiWindows[i] <= 0) ReturnFalse; } int sum = 0; for(int i = 0; i < total; i++) sum += aiWindows[i] + 1; if(iStep != (sum - 1)) ReturnFalse; int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiWindowsConvOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && iWindow >= 0) OpenCL.BufferFree(iWindow); CNeuronConvOCL::SetOpenCL(obj); iWindow = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAdaBN : public CNeuronMoE { public: CNeuronAdaBN(void) {}; ~CNeuronAdaBN(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint &bottlenecks[], uint top_k, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronAdaBN; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaBN::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint & bottlenecks[], uint top_k, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; int index = 0; if(!cGates.Init(0, index, OpenCL, window, units_count * variables, bottlenecks.Size(), top_k, optimization, iBatch)) ReturnFalse; //--- index++; if(!cXProj.Init(0, index, OpenCL, window, window, window, units_count, optimization, iBatch)) ReturnFalse; cXProj.SetActivationFunction(None); //--- cExperts.Clear(); cExperts.SetOpenCL(OpenCL); CNeuronConvOCL *conv = NULL; CNeuronMultiWindowsConvOCL *mwconv = NULL; CNeuronTransposeRCDOCL *transp = NULL; //--- uint bn_size = 0; for(uint i = 0; i < bottlenecks.Size(); i++) bn_size += bottlenecks[i]; //--- index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, bn_size, units_count, variables, optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(LReLU); //--- index++; mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv || !mwconv.Init(0, index, OpenCL, bottlenecks, window_out, units_count, variables, optimization, iBatch) || !cExperts.Add(mwconv)) { DeleteObj(conv); ReturnFalse; } mwconv.SetActivationFunction(LReLU); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, units_count * variables, bottlenecks.Size(), window_out, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window_out, window_out, window, units_count * variables, bottlenecks.Size(), optimization, iBatch) || !cExperts.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); transp = new CNeuronTransposeRCDOCL(); index++; if(!transp || !transp.Init(0, index, OpenCL, bottlenecks.Size(), units_count * variables, window, optimization, iBatch) || !cExperts.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronComplexConvOCL : public CNeuronConvOCL { protected: //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronComplexConvOCL(void) { activation = None; } ~CNeuronComplexConvOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) const { return defNeuronComplexConvOCL; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * window_out * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = (int)window; iStep = MathMax(step, 1); activation = None; iWindowOut = window_out; iVariables = variables; //--- if(CheckPointer(WeightsConv) == POINTER_INVALID) { WeightsConv = new CBufferFloat(); if(CheckPointer(WeightsConv) == POINTER_INVALID) ReturnFalse; } int count = (int)(2 * (iWindow + 1) * iWindowOut * iVariables); if(!WeightsConv.Reserve(count)) ReturnFalse; float k = (float)(1 / sqrt(iWindow + 1)); for(int i = 0; i < count; i++) { if(!WeightsConv.Add((GenerateWeight() * 2 * k - k)*WeightsMultiplier)) ReturnFalse; } if(!WeightsConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(optimization == SGD) { if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) { DeltaWeightsConv = new CBufferFloat(); if(CheckPointer(DeltaWeightsConv) == POINTER_INVALID) ReturnFalse; } if(!DeltaWeightsConv.BufferInit(count, 0.0)) ReturnFalse; if(!DeltaWeightsConv.BufferCreate(OpenCL)) ReturnFalse; } else { if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) { FirstMomentumConv = new CBufferFloat(); if(CheckPointer(FirstMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!FirstMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!FirstMomentumConv.BufferCreate(OpenCL)) ReturnFalse; //--- if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) { SecondMomentumConv = new CBufferFloat(); if(CheckPointer(SecondMomentumConv) == POINTER_INVALID) ReturnFalse; } if(!SecondMomentumConv.BufferInit(count, 0.0)) ReturnFalse; if(!SecondMomentumConv.BufferCreate(OpenCL)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3]; global_work_size[0] = Output.Total() / (2 * iWindowOut * iVariables); global_work_size[1] = iWindowOut; global_work_size[2] = iVariables; int kernel = def_k_FeedForwardComplexConv; setBuffer(kernel, def_k_ffc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffc_matrix_o, Output.GetIndex()) setArgument(kernel, def_k_ffc_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_ffc_step, (int)iStep) setArgument(kernel, def_k_ffc_window_in, (int)iWindow) setArgument(kernel, def_k_ffc_activation - 1, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2]; global_work_size[0] = NeuronOCL.Neurons() / (2 * iVariables); global_work_size[1] = iVariables; int kernel = def_k_CalcHiddenGradientComplexConv; setBuffer(kernel, def_k_chgc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_g, Gradient.GetIndex()) setBuffer(kernel, def_k_chgc_matrix_o, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgc_matrix_ig, NeuronOCL.getGradientIndex()) setArgument(kernel, def_k_chgc_outputs, Neurons() / (2 * iVariables)) setArgument(kernel, def_k_chgc_step, (int)iStep) setArgument(kernel, def_k_chgc_window_in, (int)iWindow) setArgument(kernel, def_k_chgc_window_out, (int)iWindowOut) setArgument(kernel, def_k_chgc_activation, (int)NeuronOCL.Activation()) setArgument(kernel, def_k_chgc_shift_out, (int)0) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexConvOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(NeuronOCL) == POINTER_INVALID) ReturnFalse; uint global_work_offset[2] = { 0, 0 }; uint global_work_size[2] = { WeightsConv.Total() / (2 * iVariables), iVariables }; float lt = 0; int kernel = -1; switch(optimization) { case SGD: kernel = def_k_UpdateWeightsComplexConvMomentum; setBuffer(kernel, def_k_uwcm_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwcm_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwcm_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwcm_matrix_dw, DeltaWeightsConv.GetIndex()) setBuffer(kernel, def_k_uwcm_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_uwcm_learning_rates, lr) setArgument(kernel, def_k_uwcm_momentum, alpha) setBuffer(kernel, def_k_uwcm_window_in, (int)iWindow) setBuffer(kernel, def_k_uwcm_window_out, (int)iWindowOut) setBuffer(kernel, def_k_uwcm_step, (int)iStep) kernelExecute(kernel, global_work_offset, global_work_size) break; case ADAM: kernel = def_k_UpdateWeightsComplexConvAdam; setBuffer(kernel, def_k_uwca_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_g, getGradientIndex()) setBuffer(kernel, def_k_uwca_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwca_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwca_matrix_v, SecondMomentumConv.GetIndex()) lt = (float)(lr * sqrt(1 - pow(b2, t)) / (1 - pow(b1, t))); setArgument(kernel, def_k_uwca_inputs, NeuronOCL.Neurons() / (2 * iVariables)) setArgument(kernel, def_k_uwca_l, lt) setArgument(kernel, def_k_uwca_b1, b1) setArgument(kernel, def_k_uwca_b2, b2) setArgument(kernel, def_k_uwca_window_in, (int)iWindow) setArgument(kernel, def_k_uwca_window_out, (int)iWindowOut) setArgument(kernel, def_k_uwca_step, (int)iStep) //--- kernelExecute(kernel, global_work_offset, global_work_size) t++; break; default: ReturnFalse; break; } return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronComplexMVMHMaskAttention : public CNeuronBaseOCL { protected: uint iWindow; uint iWindowKey; uint iHeads; uint iUnits; uint iVariables; //--- CNeuronComplexConvOCL cQKV; CNeuronBaseOCL cQ; CNeuronBaseOCL cKV; CNeuronConvOCL cMask; CNeuronBaseOCL cMHAttentionOut; CNeuronComplexConvOCL cPooling; CNeuronBaseOCL cResidual; CNeuronComplexConvOCL cFeedForward[2]; //--- virtual bool AttentionOut(void); virtual bool AttentionInsideGradients(void); virtual bool SumAndNormilize(CBufferFloat *tensor1, CBufferFloat *tensor2, CBufferFloat *out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) override; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; public: CNeuronComplexMVMHMaskAttention(void) {}; ~CNeuronComplexMVMHMaskAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronComplexMVMHMaskAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * window * units_count * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(window_key, 1); iUnits = units_count; iHeads = MathMax(heads, 1); iVariables = variables; //--- uint index = 0; if(!cMask.Init(0, index, OpenCL, 2 * iWindow, 2 * iWindow, iVariables * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cMask.SetActivationFunction(SIGMOID); CBufferFloat *temp = cMask.GetWeightsConv(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- index++; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(None); index++; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cPooling.Init(0, index, OpenCL, iWindowKey * iHeads, iWindowKey * iHeads, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, iWindow, iWindow, 4 * iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(LReLU); index++; if(!cFeedForward[1].Init(0, index, OpenCL, 4 * iWindow, 4 * iWindow, iWindow, iUnits, iVariables, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionOut(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables/*Q units*/, iVariables/*K units*/, iHeads * iUnits/*Heads*/}; uint local_work_size[3] = {1, iVariables, 1}; ResetLastError(); int kernel = def_k_MaskAttentionComplex; setBuffer(kernel, def_k_maskattcom_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcom_out, cMHAttentionOut.getOutputIndex()) setBuffer(kernel, def_k_maskattcom_masks, cMask.getOutputIndex()) setArgument(kernel, def_k_maskattcom_dimension, (int)iWindowKey) setArgument(kernel, def_k_maskattcom_heads_kv, (int)(iHeads * iUnits)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::AttentionInsideGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {iVariables, iWindowKey, iHeads * iUnits}; ResetLastError(); int kernel = def_k_MaskAttentionGradientsComplex; setBuffer(kernel, def_k_maskattcomgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_scores, cMask.getPrevOutIndex()) setBuffer(kernel, def_k_maskattcomgr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_maskattcomgr_mask_g, cMask.getGradientIndex()) setBuffer(kernel, def_k_maskattcomgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_maskattcomgr_kunits, (int)iVariables) setArgument(kernel, def_k_maskattcomgr_heads_kv, (int)(iHeads * iUnits)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::SumAndNormilize(CBufferFloat * tensor1, CBufferFloat * tensor2, CBufferFloat * out, int dimension, bool normilize = true, int shift_in1 = 0, int shift_in2 = 0, int shift_out = 0, float multiplyer = 0.5f) { if(CheckPointer(OpenCL) == POINTER_INVALID || CheckPointer(tensor1) == POINTER_INVALID || CheckPointer(tensor2) == POINTER_INVALID || CheckPointer(out) == POINTER_INVALID) ReturnFalse; if(tensor1.GetIndex() < 0) ReturnFalse; if(tensor2.GetIndex() < 0) ReturnFalse; if(out.GetIndex() < 0) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[1]; int size = MathMin(MathMin(tensor1.Total() / 2 - shift_in1, tensor2.Total() / 2 - shift_in2), out.Total() / 2 - shift_out); if(size <= 0) ReturnFalse; global_work_size[0] = size / dimension; int kernel = def_k_MatrixSum; setBuffer(kernel, def_k_sum_matrix1, tensor1.GetIndex()) setBuffer(kernel, def_k_sum_matrix2, tensor2.GetIndex()) setBuffer(kernel, def_k_sum_matrix_out, out.GetIndex()) setArgument(kernel, def_k_sum_dimension, 2 * dimension) setArgument(kernel, def_k_sum_shift_in1, 2 * shift_in1) setArgument(kernel, def_k_sum_shift_in2, 2 * shift_in2) setArgument(kernel, def_k_sum_shift_out, 2 * shift_out) setArgument(kernel, def_k_sum_multiplyer, multiplyer) kernelExecute(kernel, global_work_offset, global_work_size) //--- if(!normilize) return true; //--- kernel = def_k_ComplexNormalize; setBuffer(kernel, def_k_cn_inputs, out.GetIndex()) setBuffer(kernel, def_k_cn_outputs, out.GetIndex()) setBuffer(kernel, def_k_cn_means, PrevOutput.GetIndex()) setBuffer(kernel, def_k_cn_vars, PrevOutput.GetIndex()) setArgument(kernel, def_k_cn_dimension, dimension) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; //cMask.getOutput().BufferRead(); if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; //cQKV.getOutput().BufferRead(); if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; //cPooling.getOutput().BufferRead(); if(!SumAndNormilize(NeuronOCL.getOutput(), cPooling.getOutput(), cResidual.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; //cFeedForward[0].getOutput().BufferRead(); if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; //cFeedForward[1].getOutput().BufferRead(); if(!SumAndNormilize(cResidual.getOutput(), cFeedForward[1].getOutput(), getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; //--- if(!DeActivation(cPooling.getOutput(), cPooling.getGradient(), cResidual.getGradient(), cPooling.Activation()) || !DeActivation(cPooling.getOutput(), cPooling.getPrevOutput(), Gradient, cPooling.Activation()) || !SumAndNormilize(cPooling.getGradient(), cPooling.getPrevOutput(), cPooling.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), 2 * iWindowKey * iHeads, 4 * iWindowKey * iHeads, iUnits * iVariables)) ReturnFalse; //--- if(!DeActivation(cQKV.getOutput(), cQKV.getPrevOutput(), cQKV.getGradient(), cQKV.Activation()) || !prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getPrevOutput(), cResidual.getGradient(), prevLayer.Activation()) || !SumAndNormilize(prevLayer.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cResidual.getGradient(), Gradient, prevLayer.Activation()) || !SumAndNormilize(cResidual.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!DeActivation(cMask.getOutput(), cMask.getGradient(), cMask.getGradient(), cMask.Activation()) || !prevLayer.CalcHiddenGradients(cMask.AsObject()) || !SumAndNormilize(prevLayer.getGradient(), cResidual.getPrevOutput(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!NeuronOCL.SwapOutputs()) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronComplexMVMHMaskAttention *Source = source; if(!cMask.WeightsUpdate(GetPointer(Source.cMask), tau)) ReturnFalse; if(!cQKV.WeightsUpdate(GetPointer(Source.cQKV), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(GetPointer(Source.cPooling), tau)) ReturnFalse; for(int i = 0; i < 2; i++) if(!cFeedForward[i].WeightsUpdate(GetPointer(Source.cFeedForward[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowKey) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cMask.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!cFeedForward[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronComplexMVMHMaskAttention::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; for(uint i = 0; i < cFeedForward.Size(); i++) if(!LoadInsideLayer(file_handle, cFeedForward[i].AsObject())) ReturnFalse; //--- uint index = 2; if(!cQ.Init(0, index, OpenCL, 2 * iWindowKey * iHeads * iVariables * iUnits, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); //--- index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cResidual.Init(0, index, OpenCL, cPooling.Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronComplexMVMHMaskAttention::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cMask.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); for(uint i = 0; i < cFeedForward.Size(); i++) cFeedForward[i].SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCATCH : public CNeuronTransposeOCL { protected: CNeuronTransposeOCL cTranspose; CNeuronBaseOCL caFreqIn[2]; CNeuronBaseOCL cFreqConcat; CNeuronComplexConvOCL caProjection[2]; CNeuronComplexMVMHMaskAttention caChannelFusion[2]; CNeuronComplexConvOCL caLinearHead[2]; CNeuronBaseOCL caFreqOut[2]; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, uint variables, bool reverse = false); virtual bool ComplexNormalize(CNeuronBaseOCL *cInputFreqComplex, CNeuronBaseOCL *cNormFreqComplex, CBufferFloat *cMeans, CBufferFloat *cVariances, int dimension, int variables); virtual bool ComplexNormalizeGradient(CNeuronBaseOCL *cInputFreqComplex, CNeuronBaseOCL *cNormFreqComplex, CBufferFloat *cVariances, int dimension, int variables); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; public: CNeuronCATCH(void) {}; ~CNeuronCATCH(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_step, uint variables, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCATCH; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, uint variables, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; int kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / variables)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / variables)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint time_step, uint variables, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables, time_step, optimization_type, batch)) ReturnFalse; //--- Calculate FFT size int power = int(MathLog(time_step) / M_LN2); if(power <= 0) ReturnFalse; if(MathPow(2, power) != time_step) power++; uint FreqUinits = uint(MathPow(2, power)); //--- if(window <= 0 || step <= 0) ReturnFalse; int Segments = int((FreqUinits - int(window) + step - 1) / step); if(Segments <= 0) ReturnFalse; //--- uint index = 0; if(!cTranspose.Init(0, index, OpenCL, time_step, variables, optimization, iBatch)) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) { index++; if(!caFreqIn[i].Init(0, index, OpenCL, FreqUinits * variables, optimization, iBatch)) ReturnFalse; caFreqIn[i].SetActivationFunction(None); } index++; if(!cFreqConcat.Init(0, index, OpenCL, 2 * FreqUinits * variables, optimization, iBatch)) ReturnFalse; cFreqConcat.SetActivationFunction(None); //--- index++; if(!caProjection[0].Init(0, index, OpenCL, window, step, 2 * window, Segments, variables, optimization, iBatch)) ReturnFalse; caProjection[0].SetActivationFunction(LReLU); index++; if(!caProjection[1].Init(0, index, OpenCL, 2 * window, 2 * window, window_key, Segments, variables, optimization, iBatch)) ReturnFalse; caProjection[1].SetActivationFunction(TANH); //--- index++; if(!caChannelFusion[0].Init(0, index, OpenCL, window_key, window_key, heads, Segments, variables, optimization, iBatch)) ReturnFalse; index++; if(Segments % 2 == 0) { if(!caChannelFusion[1].Init(0, index, OpenCL, 2 * window_key, window_key, heads, Segments / 2, variables, optimization, iBatch)) ReturnFalse; } else if(!caChannelFusion[1].Init(0, index, OpenCL, window_key, window_key, heads, Segments, variables, optimization, iBatch)) ReturnFalse; //--- index++; if(!caLinearHead[0].Init(0, index, OpenCL, window_key, window_key, window, Segments, variables, optimization, iBatch)) ReturnFalse; caLinearHead[0].SetActivationFunction(LReLU); index++; if(!caLinearHead[1].Init(0, index, OpenCL, window * Segments, window * Segments, FreqUinits, variables, 1, optimization, iBatch)) ReturnFalse; caLinearHead[1].SetActivationFunction(None); //--- for(uint i = 0; i < caFreqOut.Size(); i++) { index++; if(!caFreqOut[i].Init(0, index, OpenCL, FreqUinits * variables, optimization, iBatch)) ReturnFalse; caFreqOut[i].SetActivationFunction(None); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!FFT(cTranspose.getOutput(), NULL, caFreqIn[0].getOutput(), caFreqIn[1].getOutput(), iCount, false)) ReturnFalse; if(!Concat(caFreqIn[0].getOutput(), caFreqIn[1].getOutput(), cFreqConcat.getOutput(), 1, 1, caFreqIn[0].Neurons())) ReturnFalse; if(!ComplexNormalize(cFreqConcat.AsObject(), cFreqConcat.AsObject(), cFreqConcat.getPrevOutput(), cFreqConcat.getPrevOutput(), caFreqIn[0].Neurons() / iCount, iCount)) ReturnFalse; //--- CNeuronBaseOCL *neuron = cFreqConcat.AsObject(); //neuron.getOutput().BufferRead(); for(uint i = 0; i < caProjection.Size(); i++) { if(!caProjection[i].FeedForward(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); //neuron.getOutput().BufferRead(); } //--- for(uint i = 0; i < caChannelFusion.Size(); i++) { if(!caChannelFusion[i].FeedForward(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); //neuron.getOutput().BufferRead(); } //--- for(uint i = 0; i < caLinearHead.Size(); i++) { if(!caLinearHead[i].FeedForward(neuron)) ReturnFalse; neuron = caLinearHead[i].AsObject(); //neuron.getOutput().BufferRead(); } //--- if(!DeConcat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), neuron.getOutput(), 1, 1, caFreqOut[0].Neurons())) ReturnFalse; if(!FFT(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caFreqOut[0].getPrevOutput(), caFreqOut[1].getPrevOutput(), iCount, true)) ReturnFalse; //--- if(!DeConcat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caFreqOut[0].getPrevOutput(), iWindow, caFreqOut[0].Neurons() / iCount - iWindow, iCount)) ReturnFalse; //--- if(!SumAndNormilize(caFreqOut[0].getOutput(), cTranspose.getOutput(), caFreqOut[0].getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(caFreqOut[0].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(caFreqOut[1].AsObject())) ReturnFalse; if(!SumAndNormilize(caFreqOut[1].getGradient(), caFreqOut[1].getGradient(), cTranspose.getPrevOutput(), iWindow, false, 0, 0, 0, 0.5f)) ReturnFalse; if((caFreqOut[0].Neurons() - iWindow) > 0) if(!SumAndNormilize(caFreqOut[1].getOutput(), caFreqOut[1].getOutput(), caFreqOut[1].getOutput(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; if(!Concat(caFreqOut[1].getGradient(), caFreqOut[1].getOutput(), caFreqOut[0].getGradient(), iWindow, caFreqOut[0].Neurons() - iWindow, iCount)) ReturnFalse; if(!SumAndNormilize(caFreqOut[1].getPrevOutput(), caFreqOut[1].getPrevOutput(), caFreqOut[1].getGradient(), 1, false, 0, 0, 0, -0.5f)) ReturnFalse; //--- if(!FFT(caFreqOut[0].getGradient(), caFreqOut[1].getGradient(), caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), iCount, false)) ReturnFalse; if(!Concat(caFreqOut[0].getOutput(), caFreqOut[1].getOutput(), caLinearHead[1].getGradient(), 1, 1, caFreqOut[0].Neurons())) ReturnFalse; //--- if(!caLinearHead[0].CalcHiddenGradients(caLinearHead[1].AsObject())) ReturnFalse; //--- CObject *neuron = caLinearHead[0].AsObject(); for(int i = int(caChannelFusion.Size()) - 1; i >= 0; i--) { if(!caChannelFusion[i].CalcHiddenGradients(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); } //--- for(int i = int(caProjection.Size()) - 1; i >= 0; i--) { if(!caProjection[i].CalcHiddenGradients(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); } //--- if(!cFreqConcat.CalcHiddenGradients(neuron)) ReturnFalse; //--- if(!DeConcat(caFreqIn[0].getGradient(), caFreqIn[1].getGradient(), cFreqConcat.getGradient(), 1, 1, caFreqIn[0].Neurons())) ReturnFalse; if(!FFT(caFreqIn[0].getGradient(), caFreqIn[1].getGradient(), caFreqIn[0].getPrevOutput(), caFreqIn[1].getPrevOutput(), iCount, false)) ReturnFalse; if(!DeConcat(cTranspose.getGradient(), caFreqIn[0].getGradient(), caFreqIn[0].getPrevOutput(), iWindow, caFreqIn[0].Neurons() / iCount - iWindow, iCount)) ReturnFalse; //--- if(!SumAndNormilize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1.0f)) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), prevLayer.getGradient(), prevLayer.getGradient(), prevLayer.Activation())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *neuron = cFreqConcat.AsObject(); for(uint i = 0; i < caProjection.Size(); i++) { if(!caProjection[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caProjection[i].AsObject(); } //--- for(uint i = 0; i < caChannelFusion.Size(); i++) { if(!caChannelFusion[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caChannelFusion[i].AsObject(); } //--- for(uint i = 0; i < caLinearHead.Size(); i++) { if(!caLinearHead[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = caLinearHead[i].AsObject(); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronCATCH *Source = source; for(uint i = 0; i < caProjection.Size(); i++) if(!caProjection[i].WeightsUpdate(GetPointer(Source.caProjection[i]), tau)) ReturnFalse; //--- for(uint i = 0; i < caChannelFusion.Size(); i++) if(!caChannelFusion[i].WeightsUpdate(GetPointer(Source.caChannelFusion[i]), tau)) ReturnFalse; //--- for(uint i = 0; i < caLinearHead.Size(); i++) if(!caLinearHead[i].WeightsUpdate(GetPointer(Source.caLinearHead[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) if(!caFreqIn[i].Save(file_handle)) ReturnFalse; if(!cFreqConcat.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caProjection.Size(); i++) if(!caProjection[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caChannelFusion.Size(); i++) if(!caChannelFusion[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caLinearHead.Size(); i++) if(!caLinearHead[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFreqOut.Size(); i++) if(!caFreqOut[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caFreqIn.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqIn[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFreqConcat.AsObject())) ReturnFalse; for(uint i = 0; i < caProjection.Size(); i++) if(!LoadInsideLayer(file_handle, caProjection[i].AsObject())) ReturnFalse; for(uint i = 0; i < caChannelFusion.Size(); i++) if(!LoadInsideLayer(file_handle, caChannelFusion[i].AsObject())) ReturnFalse; for(uint i = 0; i < caLinearHead.Size(); i++) if(!LoadInsideLayer(file_handle, caLinearHead[i].AsObject())) ReturnFalse; for(uint i = 0; i < caFreqOut.Size(); i++) if(!LoadInsideLayer(file_handle, caFreqOut[i].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCATCH::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); for(uint i = 0; i < caFreqIn.Size(); i++) caFreqIn[i].SetOpenCL(OpenCL); cFreqConcat.SetOpenCL(OpenCL); for(uint i = 0; i < caProjection.Size(); i++) caProjection[i].SetOpenCL(OpenCL); for(uint i = 0; i < caChannelFusion.Size(); i++) caChannelFusion[i].SetOpenCL(OpenCL); for(uint i = 0; i < caLinearHead.Size(); i++) caLinearHead[i].SetOpenCL(OpenCL); for(uint i = 0; i < caFreqOut.Size(); i++) caFreqOut[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::ComplexNormalize(CNeuronBaseOCL * cInputFreqComplex, CNeuronBaseOCL * cNormFreqComplex, CBufferFloat * cMeans, CBufferFloat * cVariances, int dimension, int variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; setBuffer(def_k_ComplexNormalize, def_k_cn_inputs, cInputFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_outputs, cNormFreqComplex.getOutputIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_means, cMeans.GetIndex()) setBuffer(def_k_ComplexNormalize, def_k_cn_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalize, def_k_cn_dimension, (int)(dimension)) kernelExecute(def_k_ComplexNormalize, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCATCH::ComplexNormalizeGradient(CNeuronBaseOCL * cInputFreqComplex, CNeuronBaseOCL * cNormFreqComplex, CBufferFloat * cVariances, int dimension, int variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_inputs_gr, cInputFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_outputs_gr, cNormFreqComplex.getGradientIndex()) setBuffer(def_k_ComplexNormalizeGradient, def_k_cng_vars, cVariances.GetIndex()) setArgument(def_k_ComplexNormalizeGradient, def_k_cng_dimension, (int)(dimension)) kernelExecute(def_k_ComplexNormalizeGradient, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSkillsEncoder : public CNeuronSoftMaxOCL { protected: CNeuronCATCH cCrossObservAttention; CNeuronTransposeOCL cTranspose; CNeuronConvOCL cSkillsProjection[2]; CNeuronBaseOCL cPrevSkillsConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; public: CNeuronSkillsEncoder(void) {}; ~CNeuronSkillsEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_step, uint variables, uint skills, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSkillsEncoder; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint time_step, uint variables, uint skills, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, variables * skills, optimization_type, batch)) ReturnFalse; SetHeads(variables); //--- int index = 0; if(!cCrossObservAttention.Init(0, index, OpenCL, time_step, variables, window, step, window_key, heads, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, time_step, variables, optimization, iBatch)) ReturnFalse; //--- uint count = (time_step - (window - step) + (step - 1)) / step; if(count <= 1) { window = time_step; count = 1; } //--- index++; if(!cSkillsProjection[0].Init(0, index, OpenCL, window, step, (window_key + 1) / 2, count, variables, optimization, iBatch)) ReturnFalse; cSkillsProjection[0].SetActivationFunction(SoftPlus); index++; if(!cSkillsProjection[1].Init(0, index, OpenCL, (window_key + 1) / 2 * count, (window_key + 1) / 2 * count, skills, 1, variables, optimization, iBatch)) ReturnFalse; cSkillsProjection[1].SetActivationFunction(None); //--- index++; if(!cPrevSkillsConcat.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cPrevSkillsConcat.SetActivationFunction((ENUM_ACTIVATION)cSkillsProjection[1].Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cCrossObservAttention.FeedForward(NeuronOCL)) ReturnFalse; //cCrossObservAttention.getOutput().BufferRead(); if(!cTranspose.FeedForward(cCrossObservAttention.AsObject())) ReturnFalse; if(!cSkillsProjection[0].FeedForward(cTranspose.AsObject())) ReturnFalse; if(!cSkillsProjection[1].SwapOutputs() || !cSkillsProjection[1].FeedForward(cSkillsProjection[0].AsObject())) ReturnFalse; //--- return CNeuronSoftMaxOCL::feedForward(cSkillsProjection[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; if(!CNeuronSoftMaxOCL::calcInputGradients(cSkillsProjection[1].AsObject())) ReturnFalse; //--- if(!Concat(cSkillsProjection[1].getOutput(), cSkillsProjection[1].getPrevOutput(), cPrevSkillsConcat.getOutput(), cSkillsProjection[1].GetFilters(), cSkillsProjection[1].GetFilters(), iHeads)) ReturnFalse; if(!DiversityLoss(cPrevSkillsConcat.AsObject(), 2 * iHeads, cSkillsProjection[1].GetFilters(), false)) ReturnFalse; if(!DeConcat(cPrevSkillsConcat.getOutput(), cPrevSkillsConcat.getPrevOutput(), cPrevSkillsConcat.getGradient(), cSkillsProjection[1].GetFilters(), cSkillsProjection[1].GetFilters(), iHeads)) ReturnFalse; if(!SumAndNormilize(cSkillsProjection[1].getGradient(), cPrevSkillsConcat.getOutput(), cSkillsProjection[1].getGradient(), cSkillsProjection[1].GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cSkillsProjection[0].CalcHiddenGradients(cSkillsProjection[1].AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cSkillsProjection[0].AsObject())) ReturnFalse; if(!cCrossObservAttention.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return prevLayer.CalcHiddenGradients(cCrossObservAttention.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cCrossObservAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSkillsProjection[0].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; if(!cSkillsProjection[1].UpdateInputWeights(cSkillsProjection[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronSkillsEncoder *Source = source; if(!cCrossObservAttention.WeightsUpdate(GetPointer(Source.cCrossObservAttention), tau)) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!cSkillsProjection[i].WeightsUpdate(GetPointer(Source.cSkillsProjection[i]), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; if(!cCrossObservAttention.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!cSkillsProjection[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSkillsEncoder::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossObservAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < cSkillsProjection.Size(); i++) if(!LoadInsideLayer(file_handle, cSkillsProjection[i].AsObject())) ReturnFalse; //--- if(!cPrevSkillsConcat.Init(0, 4, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cPrevSkillsConcat.SetActivationFunction((ENUM_ACTIVATION)cSkillsProjection[1].Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSkillsEncoder::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); cCrossObservAttention.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); for(uint i = 0; i < cSkillsProjection.Size(); i++) cSkillsProjection[i].SetOpenCL(OpenCL); cPrevSkillsConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHiSSDLowLevelControler: public CNeuronConvOCL { protected: uint iTaskSkills; uint iCommonSkills; //--- CNeuronSkillsEncoder cTaskSpecificSkillsEncoder; CNeuronTransposeOCL cTranspose; CNeuronBaseOCL cObservAndSkillsConcat; CNeuronBatchNormOCL cNormalizarion; CNeuronConvOCL cActionDecoder[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None ) override; public: CNeuronHiSSDLowLevelControler(void) {}; ~CNeuronHiSSDLowLevelControler(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_step, uint variables, uint task_skills, uint common_skills, uint n_actions, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronHiSSDLowLevelControler; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint time_step, uint variables, uint task_skills, uint common_skills, uint n_actions, uint window, uint step, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window_key, window_key, n_actions, 1, variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(SIGMOID); //--- int index = 0; if(!cTaskSpecificSkillsEncoder.Init(0, index, OpenCL, time_step, variables, task_skills, window, step, window_key, heads, optimization, iBatch)) ReturnFalse; cTaskSpecificSkillsEncoder.SetActivationFunction(None); //--- iTaskSkills = task_skills; iCommonSkills = MathMax(common_skills, 1); //--- index++; if(!cTranspose.Init(0, index, OpenCL, time_step, variables, optimization, iBatch)) ReturnFalse; //--- uint window_size = (time_step + iTaskSkills + iCommonSkills); index++; if(!cObservAndSkillsConcat.Init(0, index, OpenCL, window_size * iVariables, optimization, iBatch)) ReturnFalse; cObservAndSkillsConcat.SetActivationFunction(None); index++; if(!cNormalizarion.Init(0, index, OpenCL, cObservAndSkillsConcat.Neurons(), iBatch, optimization)) ReturnFalse; cNormalizarion.SetActivationFunction(None); for(uint i = 0; i < cActionDecoder.Size(); i++) { index++; if(!cActionDecoder[i].Init(0, index, OpenCL, window_size, window_size, window_key, 1, iVariables, optimization, iBatch)) ReturnFalse; cActionDecoder[i].SetActivationFunction(SoftPlus); window_size = window_key; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!SecondInput) ReturnFalse; if(!cTaskSpecificSkillsEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cTranspose.getOutput(), cTaskSpecificSkillsEncoder.getOutput(), SecondInput, cObservAndSkillsConcat.getOutput(), cTranspose.GetCount(), iTaskSkills, iCommonSkills, iVariables)) ReturnFalse; if(!cNormalizarion.FeedForward(cObservAndSkillsConcat.AsObject())) ReturnFalse; CNeuronBaseOCL *neuron = cNormalizarion.AsObject(); for(uint i = 0; i < cActionDecoder.Size(); i++) { if(!cActionDecoder[i].FeedForward(neuron)) ReturnFalse; neuron = cActionDecoder[i].AsObject(); } //--- return CNeuronConvOCL::feedForward(neuron); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- uint total = cActionDecoder.Size(); if(total <= 0) ReturnFalse; CObject *neuron = cActionDecoder[total - 1].AsObject(); //--- if(!CNeuronConvOCL::calcInputGradients(neuron)) ReturnFalse; for(int i = int(total - 2); i >= 0; i--) { if(!cActionDecoder[i].CalcHiddenGradients(neuron)) ReturnFalse; neuron = cActionDecoder[i].AsObject(); } if(!cNormalizarion.CalcHiddenGradients(neuron)) ReturnFalse; if(!cObservAndSkillsConcat.CalcHiddenGradients(cNormalizarion.AsObject())) ReturnFalse; if(!DeConcat(cTranspose.getGradient(), cTaskSpecificSkillsEncoder.getGradient(), SecondGradient, cObservAndSkillsConcat.getGradient(), cTranspose.GetCount(), iTaskSkills, iCommonSkills, iVariables)) ReturnFalse; //--- if(SecondActivation != None) { if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; } if(NeuronOCL.Activation() != None) { if(!DeActivation(cTranspose.getOutput(), cTranspose.getGradient(), cTranspose.getGradient(), NeuronOCL.Activation())) ReturnFalse; } if(cTaskSpecificSkillsEncoder.Activation() != None) { if(!DeActivation(cTaskSpecificSkillsEncoder.getOutput(), cTaskSpecificSkillsEncoder.getGradient(), cTaskSpecificSkillsEncoder.getGradient(), cTaskSpecificSkillsEncoder.Activation())) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(cTaskSpecificSkillsEncoder.AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cTranspose.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, iVariables, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * second) { if(!cTaskSpecificSkillsEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNormalizarion.UpdateInputWeights(cObservAndSkillsConcat.AsObject())) ReturnFalse; CNeuronBaseOCL *neuron = cNormalizarion.AsObject(); for(uint i = 0; i < cActionDecoder.Size(); i++) { if(!cActionDecoder[i].UpdateInputWeights(neuron)) ReturnFalse; neuron = cActionDecoder[i].AsObject(); } //--- return CNeuronConvOCL::updateInputWeights(neuron); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!source || source.Type() != Type()) ReturnFalse; CNeuronHiSSDLowLevelControler *Source = source; if(!cTaskSpecificSkillsEncoder.WeightsUpdate(GetPointer(Source.cTaskSpecificSkillsEncoder), tau)) ReturnFalse; if(!cNormalizarion.WeightsUpdate(GetPointer(Source.cNormalizarion), tau)) ReturnFalse; for(uint i = 0; i < cActionDecoder.Size(); i++) if(!cActionDecoder[i].WeightsUpdate(GetPointer(Source.cActionDecoder[i]), tau)) ReturnFalse; //--- return CNeuronConvOCL::WeightsUpdate(source, tau); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iTaskSkills)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iCommonSkills)) < INT_VALUE) ReturnFalse; //--- if(!cTaskSpecificSkillsEncoder.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cNormalizarion.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cActionDecoder.Size(); i++) if(!cActionDecoder[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHiSSDLowLevelControler::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iTaskSkills = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iCommonSkills = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cTaskSpecificSkillsEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormalizarion.AsObject())) ReturnFalse; for(uint i = 0; i < cActionDecoder.Size(); i++) if(!LoadInsideLayer(file_handle, cActionDecoder[i].AsObject())) ReturnFalse; //--- if(!cObservAndSkillsConcat.Init(0, 2, OpenCL, cNormalizarion.Neurons(), optimization, iBatch)) ReturnFalse; cObservAndSkillsConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHiSSDLowLevelControler::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); //--- cTaskSpecificSkillsEncoder.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cObservAndSkillsConcat.SetOpenCL(OpenCL); cNormalizarion.SetOpenCL(OpenCL); for(uint i = 0; i < cActionDecoder.Size(); i++) cActionDecoder[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCGLSTMOCL : public CNeuronBaseOCL { protected: CNeuronBaseOCL cConcatenateInputs; CNeuronConvOCL cProjection; //--- virtual bool CSLSTM_feedForward(void); virtual bool CSLSTM_CalcHiddenGradient(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCGLSTMOCL(void) {}; ~CNeuronCGLSTMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronCGLSTMOCL; } virtual bool Clear(void) override; virtual CBufferFloat *getLSTMWeights(void) { return cProjection.GetWeightsConv(); } virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { return CNeuronCGLSTMOCL::Init(numOutputs, myIndex, open_cl, numNeurons, numNeurons, 1, optimization_type, batch); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- if(!cConcatenateInputs.Init(0, 0, OpenCL, (count + window)*variables, optimization, iBatch)) ReturnFalse; cConcatenateInputs.SetActivationFunction(None); //--- if(!cProjection.Init(0, 1, OpenCL, count + window, count + window, count * 4, 1, variables, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(None); //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::CSLSTM_feedForward(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {cProjection.GetFilters() / 4, cProjection.GetVariables()}; uint kernel = def_k_CSLSTM_FeedForward; setBuffer(kernel, def_k_cslstmff_concatenated, cProjection.getOutputIndex()) setBuffer(kernel, def_k_cslstmff_memory, cProjection.getPrevOutIndex()) setBuffer(kernel, def_k_cslstmff_output, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::CSLSTM_CalcHiddenGradient(void) { uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {cProjection.GetFilters(), cProjection.GetVariables()}; uint kernel = def_k_CSLSTM_CalcHiddenGradient; setBuffer(kernel, def_k_cslstmhg_concatenated, cProjection.getOutputIndex()) setBuffer(kernel, def_k_cslstmhg_concatenated_grad, cProjection.getGradientIndex()) setBuffer(kernel, def_k_cslstmhg_memory, cProjection.getPrevOutIndex()) setBuffer(kernel, def_k_cslstmhg_output_grad, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- int hidden = (int)cProjection.GetFilters() / 4; int inputs = (int)cProjection.GetWindow() - hidden; int variables = (int)cProjection.GetVariables(); //--- if(!Concat(NeuronOCL.getOutput(), getOutput(), cConcatenateInputs.getOutput(), inputs, hidden, variables)) ReturnFalse; if(!cProjection.FeedForward(cConcatenateInputs.AsObject())) ReturnFalse; //--- return CSLSTM_feedForward(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- int hidden = (int)cProjection.GetFilters() / 4; int inputs = (int)cProjection.GetWindow() - hidden; int variables = (int)cProjection.GetVariables(); //--- if(!CSLSTM_CalcHiddenGradient()) ReturnFalse; if(!cConcatenateInputs.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), getPrevOutput(), cConcatenateInputs.getGradient(), inputs, hidden, variables)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { return cProjection.UpdateInputWeights(cConcatenateInputs.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!cConcatenateInputs.Init(0, 0, OpenCL, cProjection.GetWindow()*cProjection.GetVariables(), optimization, iBatch)) ReturnFalse; cConcatenateInputs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCGLSTMOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cConcatenateInputs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; if(!cProjection.WeightsUpdate(((CNeuronCGLSTMOCL*)source).cProjection.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCGLSTMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cProjection.Clear()) ReturnFalse; if(!cProjection.getPrevOutput().Fill(0)) ReturnFalse; if(!cConcatenateInputs.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHProbAttention : public CResidualConv { protected: uint iWindow; uint iWindowKey; uint iHeads; uint iUnits; uint iTopKQuerys; uint iRandomKeys; int ibScore; //--- CNeuronConvOCL cQKV; CNeuronBaseOCL cQ; CNeuronBaseOCL cKV; CNeuronBaseOCL cRandomK; CNeuronBaseOCL cMHAttentionOut; CNeuronConvOCL cPooling; CNeuronTransposeOCL cTranspose[2]; CNeuronConvOCL cScaling; //--- virtual bool RandomKeys(CBufferFloat* indexes, int random, int units, int heads); virtual bool QueryImportance(void); virtual bool TopKIndexes(void); //--- virtual bool AttentionOut(void); virtual bool AttentionInsideGradients(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; public: CNeuronMHProbAttention(void) {}; ~CNeuronMHProbAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMHProbAttention; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch ) { if(!CResidualConv::Init(numOutputs, myIndex, open_cl, window, window, units_count, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iWindowKey = MathMax(5, window_key); iHeads = MathMax(1, heads); iUnits = units_count; iTopKQuerys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); iRandomKeys = int(MathMin(5 * MathMax(MathLog(iUnits), 1), iUnits)); //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, iWindow, iWindow, 3 * iWindowKey * iHeads, iUnits, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(TANH); index++; if(!cQ.Init(0, index, OpenCL, cQKV.Neurons() / 3, optimization, iBatch)) ReturnFalse; index++; if(!cKV.Init(0, index, OpenCL, 2 * cQ.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cRandomK.Init(0, index, OpenCL, iHeads * MathMax(iRandomKeys, iTopKQuerys), optimization, iBatch)) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, iTopKQuerys * iHeads * iWindowKey, optimization, iBatch)) ReturnFalse; index++; if(!cPooling.Init(0, index, OpenCL, iHeads * iWindowKey, iHeads * iWindowKey, iWindow, iTopKQuerys, optimization, iBatch)) ReturnFalse; cPooling.SetActivationFunction(TANH); index++; if(!cTranspose[0].Init(0, index, OpenCL, iTopKQuerys, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cScaling.Init(0, index, OpenCL, iTopKQuerys, iTopKQuerys, iUnits, iWindow, optimization, iBatch)) ReturnFalse; cScaling.SetActivationFunction(None); index++; if(!cTranspose[1].Init(0, index, OpenCL, iWindow, iUnits, optimization, iBatch)) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::RandomKeys(CBufferFloat * indexes, int random, int units, int heads) { if(!indexes || random > units || indexes.Total() < (random * heads) ) ReturnFalse; //--- matrix ind = matrix::Zeros(random, heads); if(random == units) { for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = (float)r; } } else { double step = double(units) / random; for(int r = 0; r < random; r++) { for(int c = 0; c < heads; c++) ind[r, c] = float(int((r + MathRand() / 32767.0) * step)); } } if(!indexes.AssignArray(ind) || !indexes.BufferWrite()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::QueryImportance(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iUnits, iRandomKeys, iHeads}; uint local_work_size[3] = {1, iRandomKeys, 1}; uint kernel = def_k_ProbAttentionQeuryImp; setBuffer(kernel, def_k_probat_querys, cQ.getOutputIndex()) setBuffer(kernel, def_k_probat_keys_values, cKV.getOutputIndex()) setBuffer(kernel, def_k_probat_index_keys, cRandomK.getOutputIndex()) setBuffer(kernel, def_k_probat_querys_imp, cQ.getPrevOutIndex()) setArgument(kernel, def_k_probat_dimension, int(iWindowKey)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::TopKIndexes(void) { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iHeads}; uint kernel = def_k_TopKImportanceToIndex; setBuffer(kernel, def_k_imptoind_importance, cQ.getPrevOutIndex()) setBuffer(kernel, def_k_imptoind_indexes, cRandomK.getPrevOutIndex()) setArgument(kernel, def_k_imptoind_tok_k, int(iTopKQuerys)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionOut(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint local_work_size[3] = {1, iUnits, 1}; uint kernel = def_k_QIndexAttention; setBuffer(kernel, def_k_indatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indatt_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indatt_scores, ibScore) setBuffer(kernel, def_k_indatt_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_indatt_dimension, int(iWindowKey)) setArgument(kernel, def_k_indatt_heads_kv, int(iHeads)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::AttentionInsideGradients(void) { uint global_work_offset[3] = {0}; uint global_work_size[3] = {iTopKQuerys, iUnits, iHeads}; uint kernel = def_k_QIndexAttentionGradients; setBuffer(kernel, def_k_indattgr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_indattgr_q_g, cQ.getGradientIndex()) setBuffer(kernel, def_k_indattgr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_indattgr_kv_g, cKV.getGradientIndex()) setBuffer(kernel, def_k_indattgr_indexes, cRandomK.getPrevOutIndex()) setBuffer(kernel, def_k_indattgr_scores, ibScore) setBuffer(kernel, def_k_indattgr_gradient, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_indattgr_kunits, int(iUnits)) setArgument(kernel, def_k_indattgr_heads_kv, int(iHeads)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cQ.getOutput(), cKV.getOutput(), cQKV.getOutput(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(!RandomKeys(cRandomK.getOutput(), iRandomKeys, iUnits, iHeads)) ReturnFalse; if(!QueryImportance() || !TopKIndexes()) ReturnFalse; if(!AttentionOut()) ReturnFalse; if(!cPooling.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!cTranspose[0].FeedForward(cPooling.AsObject())) ReturnFalse; if(!cScaling.FeedForward(cTranspose[0].AsObject())) ReturnFalse; if(!cTranspose[1].FeedForward(cScaling.AsObject())) ReturnFalse; if(!SumAndNormilize(cTranspose[1].getOutput(), NeuronOCL.getOutput(), cTranspose[1].getOutput(), iWindow, true, 0, 0, 0, 1)) ReturnFalse; //--- return CResidualConv::feedForward(cTranspose[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!cQ.getGradient().Fill(0)) ReturnFalse; //--- if(!CResidualConv::calcInputGradients(cTranspose[1].AsObject())) ReturnFalse; //--- if(!cScaling.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!cTranspose[0].CalcHiddenGradients(cScaling.AsObject())) ReturnFalse; if(!cPooling.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cPooling.AsObject())) ReturnFalse; //--- if(!AttentionInsideGradients()) ReturnFalse; if(!Concat(cQ.getGradient(), cKV.getGradient(), cQKV.getGradient(), iWindowKey * iHeads, 2 * iWindowKey * iHeads, iUnits)) ReturnFalse; if(cQKV.Activation() != None) if(!DeActivation(cQKV.getOutput(), cQKV.getGradient(), cQKV.getGradient(), cQKV.Activation())) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cTranspose[1].getGradient(), cTranspose[1].getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(cTranspose[1].getGradient(), prevLayer.getGradient(), prevLayer.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cPooling.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cScaling.UpdateInputWeights(cTranspose[0].AsObject())) ReturnFalse; //--- return CResidualConv::updateInputWeights(cTranspose[1].AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Save(const int file_handle) { if(!CResidualConv::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindowKey)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iHeads)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iUnits)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iTopKQuerys)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iRandomKeys)) < INT_VALUE) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cRandomK.Save(file_handle)) ReturnFalse; if(!cMHAttentionOut.Save(file_handle)) ReturnFalse; if(!cPooling.Save(file_handle)) ReturnFalse; if(!cTranspose[0].Save(file_handle)) ReturnFalse; if(!cTranspose[1].Save(file_handle)) ReturnFalse; if(!cScaling.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::Load(const int file_handle) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- if(!CResidualConv::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindowKey = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iTopKQuerys = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iRandomKeys = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRandomK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMHAttentionOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPooling.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose[1].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScaling.AsObject())) ReturnFalse; //--- ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); if(ibScore < 0) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHProbAttention::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && ibScore >= 0) OpenCL.BufferFree(ibScore); //--- CResidualConv::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cRandomK.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cPooling.SetOpenCL(OpenCL); cTranspose[0].SetOpenCL(OpenCL); cTranspose[1].SetOpenCL(OpenCL); cScaling.SetOpenCL(OpenCL); //--- if(!!OpenCL) ibScore = OpenCL.AddBuffer(sizeof(float) * iTopKQuerys * iUnits * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHProbAttention::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CResidualConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHProbAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cPooling.WeightsUpdate(Source.cPooling.AsObject(), tau)) ReturnFalse; if(!cScaling.WeightsUpdate(Source.cScaling.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTSPositionEncoder : public CNeuronBaseOCL { protected: CNeuronConvOCL cProjection; CNeuronBatchNormOCL cNorm; CBufferFloat cPeriods; //--- virtual bool AddPE(CBufferFloat *time); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTSPositionEncoder(void) {}; ~CNeuronTSPositionEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint &periods[], uint freqs, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTSPositionEncoder; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetWindow(void) const { return cProjection.GetWindow(); } virtual uint GetWindowOut(void) const { return cProjection.GetFilters(); } virtual uint GetUnits(void) const { return cProjection.GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint units_count, uint & periods[], uint freqs, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * freqs * periods.Size(), optimization_type, batch)) ReturnFalse; //--- cPeriods.BufferFree(); if(!cPeriods.AssignArray(periods) || !cPeriods.BufferCreate(OpenCL)) ReturnFalse; //--- int index = 0; if(!cProjection.Init(0, index, OpenCL, window, window, 2 * freqs * cPeriods.Total(), units_count, 1, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); index++; if(!cNorm.Init(0, index, OpenCL, cProjection.Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); SetActivationFunction(None); if(!SetGradient(cNorm.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::AddPE(CBufferFloat * time) { if(!time) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {GetUnits(), GetWindowOut() / (2 * cPeriods.Total()), cPeriods.Total()}; uint kernel = def_k_TSPositonEncoder; setBuffer(kernel, def_k_tspe_data, cNorm.getOutputIndex()) setBuffer(kernel, def_k_tspe_time, time.GetIndex()) setBuffer(kernel, def_k_tspe_output, getOutputIndex()) setBuffer(kernel, def_k_tspe_period, cPeriods.GetIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(cProjection.AsObject())) ReturnFalse; //--- return AddPE(SecondInput); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cProjection.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return NeuronOCL.CalcHiddenGradients(cProjection.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(cProjection.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronTSPositionEncoder* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cPeriods.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTSPositionEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cPeriods.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!SetGradient(cNorm.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTSPositionEncoder::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cPeriods.BufferFree(); cPeriods.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CMamba4CastEmbeding : public CNeuronBaseOCL { protected: CNeuronConvOCL cProjection; CNeuronBatchNormOCL cNorm; CNeuronTSPositionEncoder cProjectionWithTE; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CMamba4CastEmbeding(void) {}; ~CMamba4CastEmbeding(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint &periods[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defMamba4CastEmbeding; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint & periods[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(periods.Size() <= 0) ReturnFalse; int freqs = (int(window_out / 2 + 2 * periods.Size()) - 1) / int(2 * periods.Size()); if(freqs <= 0) ReturnFalse; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cProjection.Init(0, index, OpenCL, window, window, window_out - 2 * freqs * periods.Size(), units_count, 1, optimization, iBatch)) ReturnFalse; cProjection.SetActivationFunction(TANH); index++; if(!cNorm.Init(0, index, OpenCL, cProjection.Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; if(!cProjectionWithTE.Init(0, index, OpenCL, window, units_count, periods, freqs, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cNorm.FeedForward(cProjection.AsObject())) ReturnFalse; if(!cProjectionWithTE.FeedForward(NeuronOCL, SecondInput)) ReturnFalse; if(!Concat(cNorm.getOutput(), cProjectionWithTE.getOutput(), Output, cProjection.GetFilters(), cProjectionWithTE.GetWindowOut(), cProjection.GetUnits())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(cNorm.getGradient(), cProjectionWithTE.getGradient(), Gradient, cProjection.GetFilters(), cProjectionWithTE.GetWindowOut(), cProjection.GetUnits())) ReturnFalse; if(!cProjection.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cProjectionWithTE.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, cProjection.GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNorm.UpdateInputWeights(cProjection.AsObject())) ReturnFalse; if(!cProjectionWithTE.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CMamba4CastEmbeding* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; if(!cProjectionWithTE.WeightsUpdate(Source.cProjectionWithTE.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(!cProjectionWithTE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CMamba4CastEmbeding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionWithTE.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CMamba4CastEmbeding::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cProjection.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cProjectionWithTE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiWindowsConvWPadOCL : public CNeuronConvOCL { protected: int aiWindows[]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL); public: CNeuronMultiWindowsConvWPadOCL(void) { activation = SoftPlus; iWindow = -1; } ~CNeuronMultiWindowsConvWPadOCL(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &windows[], uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMultiWindowsConvWPadOCL; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual void SetOpenCL(COpenCLMy *obj); //--- virtual uint GetWindow(void) const { return aiWindows[0]; } virtual uint GetWindowsSize(void) const { return aiWindows.Size(); } virtual uint GetWindowOut(void) const { return iWindowOut; } virtual uint GetUnits(void) const { return Neurons() / (iVariables * GetWindowsSize() * iWindowOut); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint & windows[], uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(windows.Size() <= 0 || ArrayCopy(aiWindows, windows) < int(windows.Size())) ReturnFalse; int window = 0; for(uint i = 0; i < aiWindows.Size(); i++) window += aiWindows[i] + 1; window--; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, step, window_out, units_count * aiWindows.Size() * variables, 1, ADAM, batch)) ReturnFalse; //--- iVariables = variables; int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; #ifdef _DEBUG if(!NeuronOCL.getOutput().BufferRead()) ReturnFalse; #endif //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = { GetUnits(), aiWindows.Size(), iVariables}; //--- uint inputs = NeuronOCL.Neurons() / iVariables; for(uint id = 0; id < global_work_size[0]; id++) { for(uint id_w = 0; id_w < global_work_size[1]; id_w++) { for(uint v = 0; id < global_work_size[2]; id++) { //--- int window_in = aiWindows[id_w]; int mid_win = window_in / 2; int shift_in = int(id * iStep) - mid_win; uint shift_in_var = v * inputs; uint shift_weight = 0; //--- for(uint w = 0; w < id_w; w++) shift_weight += (aiWindows[w] + 1) * iWindowOut; //--- for(uint w_out = 0; w_out < iWindowOut; w_out++) { float sum = WeightsConv[shift_weight + window_in]; //--- for(int w = 0; w < window_in; w++) if((shift_in + w) >= 0 && (shift_in + w) < int(inputs)) sum += NeuronOCL.getOutput()[shift_in_var + shift_in + w] * WeightsConv[shift_weight + w]; //--- uint shift_out = (v * global_work_size[0] + id) * iWindowOut + w_out; Output.Update(shift_out, sum); shift_weight += window_in + 1; } } } } //--- ResetLastError(); int kernel = def_k_FeedForwardMultWinConvWPad; setBuffer(kernel, def_k_ffmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmwconvwpad_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmwconvwpad_windows_in, iWindow) setArgument(kernel, def_k_ffmwconvwpad_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmwconvwpad_window_out, iWindowOut) setArgument(kernel, def_k_ffmwconvwpad_step, (int)iStep) setArgument(kernel, def_k_ffmwconvwpad_activation, (int)activation) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint filters = aiWindows.Size() * iWindowOut; uint global_work_offset[3] = {0}; uint global_work_size[3] = {NeuronOCL.Neurons() / iVariables, (uint)MathMin(filters, OpenCL.GetMaxLocalSize(1)), iVariables }; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); int kernel = def_k_CalcHiddenGradientMultWinConvWPad; setBuffer(kernel, def_k_hgmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_hgmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_hgmwconvwpad_windows_in, iWindow) setArgument(kernel, def_k_hgmwconvwpad_outputs, GetUnits()) setArgument(kernel, def_k_hgmwconvwpad_window_out, (int)iWindowOut) setArgument(kernel, def_k_hgmwconvwpad_step, (int)iStep) setArgument(kernel, def_k_hgmwconvwpad_filters, (int)filters) setArgument(kernel, def_k_hgmwconvwpad_activation, NeuronOCL.Activation()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {WeightsConv.Total(), iVariables}; uint local_work_size[2] = {1, iVariables}; ResetLastError(); int kernel = def_k_UpdateWeightsMultWinConvAdamWPad; setBuffer(kernel, def_k_uwmwconvwpad_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmwconvwpad_windows_in, iWindow) setBuffer(kernel, def_k_uwmwconvwpad_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmwconvwpad_matrix_v, SecondMomentumConv.GetIndex()) setArgument(kernel, def_k_uwmwconvwpad_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmwconvwpad_outputs, GetUnits()) setArgument(kernel, def_k_uwmwconvwpad_window_out, iWindowOut) setArgument(kernel, def_k_uwmwconvwpad_windows_total, (int)aiWindows.Size()) setArgument(kernel, def_k_uwmwconvwpad_l, lr) setArgument(kernel, def_k_uwmwconvwpad_b1, b1) setArgument(kernel, def_k_uwmwconvwpad_b2, b2) setArgument(kernel, def_k_uwmwconvwpad_step, (int)iStep) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiWindows.Size()) < INT_VALUE) ReturnFalse; for(int i = 0; i < (int)aiWindows.Size(); i++) if(FileWriteInteger(file_handle, (int)aiWindows[i]) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiWindowsConvWPadOCL::Load(const int file_handle) { ArrayFree(aiWindows); if(iWindow >= 0) OpenCL.BufferFree(iWindow); //--- if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; iWindow = INVALID_HANDLE; //--- int total = FileReadInteger(file_handle); if(ArrayResize(aiWindows, total) != total) ReturnFalse; for(int i = 0; i < total; i++) aiWindows[i] = FileReadInteger(file_handle); int temp = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); if(temp < 0) ReturnFalse; iWindow = (uint)temp; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiWindowsConvWPadOCL::SetOpenCL(COpenCLMy * obj) { if(!!OpenCL && iWindow >= 0) OpenCL.BufferFree(iWindow); CNeuronConvOCL::SetOpenCL(obj); iWindow = OpenCL.AddBufferFromArray(aiWindows, 0, aiWindows.Size(), CL_MEM_READ_ONLY); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConcatDiff: public CNeuronBaseOCL { protected: uint iUnits; uint iVariables; uint iStep; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronConcatDiff(void) : iUnits(0), iVariables(1), iStep(1) { activation = None; } ~CNeuronConcatDiff(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint step, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronConcatDiff; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint units_count, uint step, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * units_count * variables, optimization_type, batch)) ReturnFalse; if(step <= 0 || step >= units_count) ReturnFalse; //--- iUnits = units_count; iVariables = variables; iStep = step; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!OpenCL || !NeuronOCL || !Output) ReturnFalse; if(getOutputIndex() < 0) ReturnFalse; //--- { uint global_work_offset[2] = {0}; uint global_work_size[2] = {iUnits, iVariables}; const int kernel = def_k_ConcatDiff; setBuffer(kernel, def_k_concdiff_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_concdiff_output, getOutputIndex()) setArgument(kernel, def_k_concdiff_step, iStep) //--- kernelExecute(kernel, global_work_offset, global_work_size) } //--- { uint global_work_offset[1] = {0}; uint global_work_size[1] = {2 * iUnits}; const int kernel = def_k_Normilize; setBuffer(kernel, def_k_norm_buffer, getOutputIndex()) setArgument(kernel, def_k_norm_dimension, iVariables) kernelExecute(kernel, global_work_offset, global_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), getPrevOutput(), getGradient(), iVariables, iVariables, iUnits)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!FileWriteInteger(file_handle, int(iUnits))) ReturnFalse; if(!FileWriteInteger(file_handle, int(iVariables))) ReturnFalse; if(!FileWriteInteger(file_handle, int(iStep))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConcatDiff::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iStep = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMantisPatching : public CNeuronTransposeOCL { protected: CNeuronTransposeOCL cToVarSeq; CNeuronConvOCL cProjecting; CNeuronTransposeVRCOCL cToVarProjSeq; CNeuronConvOCL cPatchingProj; CNeuronProofOCL cProof; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMantisPatching(void) {}; ~CNeuronMantisPatching(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronMantisPatching; } //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables * embedding_size, patchs, optimization_type, batch)) ReturnFalse; int index = 0; if(!cToVarSeq.Init(0, index, OpenCL, count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cProjecting.Init(0, index, OpenCL, 3, 1, embedding_size, count - 2, variables, optimization, iBatch)) ReturnFalse; cProjecting.SetActivationFunction(SoftPlus); index++; if(!cToVarProjSeq.Init(0, index, OpenCL, variables, count - 2, embedding_size, optimization, iBatch)) ReturnFalse; index++; int patch_size = (int(count + patchs) - 3) / int(patchs); if(!cPatchingProj.Init(0, index, OpenCL, patch_size, patch_size, patch_filters, patchs, variables * embedding_size, optimization, iBatch)) ReturnFalse; cPatchingProj.SetActivationFunction(SoftPlus); index++; if(!cProof.Init(0, index, OpenCL, patch_filters, patch_filters, patchs * variables * embedding_size, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cToVarSeq.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjecting.FeedForward(cToVarSeq.AsObject())) ReturnFalse; if(!cToVarProjSeq.FeedForward(cProjecting.AsObject())) ReturnFalse; if(!cPatchingProj.FeedForward(cToVarProjSeq.AsObject())) ReturnFalse; if(!cProof.FeedForward(cPatchingProj.AsObject())) ReturnFalse; if(!CNeuronTransposeOCL::feedForward(cProof.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cProof.AsObject())) ReturnFalse; if(!cPatchingProj.CalcHiddenGradients(cProof.AsObject())) ReturnFalse; if(!cToVarProjSeq.CalcHiddenGradients(cPatchingProj.AsObject())) ReturnFalse; if(!cProjecting.CalcHiddenGradients(cToVarProjSeq.AsObject())) ReturnFalse; if(!cToVarSeq.CalcHiddenGradients(cProjecting.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cToVarSeq.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cProjecting.UpdateInputWeights(cToVarSeq.AsObject())) ReturnFalse; if(!cPatchingProj.UpdateInputWeights(cToVarProjSeq.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; if(!cToVarSeq.Save(file_handle)) ReturnFalse; if(!cProjecting.Save(file_handle)) ReturnFalse; if(!cToVarProjSeq.Save(file_handle)) ReturnFalse; if(!cPatchingProj.Save(file_handle)) ReturnFalse; if(!cProof.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjecting.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarProjSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProof.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMantisPatching::SetOpenCL(COpenCLMy * obj) { CNeuronTransposeOCL::SetOpenCL(obj); cToVarSeq.SetOpenCL(obj); cProjecting.SetOpenCL(obj); cToVarProjSeq.SetOpenCL(obj); cPatchingProj.SetOpenCL(obj); cProof.SetOpenCL(obj); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisPatching::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronTransposeOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMantisPatching *Source = source; if(!cProjecting.WeightsUpdate(Source.cProjecting.AsObject(), tau)) ReturnFalse; if(!cPatchingProj.WeightsUpdate(Source.cPatchingProj.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMantisAttentionUnit : public CNeuronSoftMaxOCL { protected: CNeuronBaseOCL cClassToken[2]; CNeuronMVCrossAttentionMLKV cAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMantisAttentionUnit(void) {}; ~CNeuronMantisAttentionUnit(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint token_size, uint window, uint window_key, uint heads, uint units_count, uint layers, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronMantisAttentionUnit; } //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint token_size, uint window, uint window_key, uint heads, uint units_count, uint layers, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, token_size, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cClassToken[0].Init(token_size, index, OpenCL, 1, optimization, iBatch)) ReturnFalse; if(!cClassToken[0].getOutput().Fill(1)) ReturnFalse; index++; if(!cClassToken[1].Init(0, index, OpenCL, token_size, optimization, iBatch)) ReturnFalse; cClassToken[1].SetActivationFunction(SIGMOID); //--- index++; if(!cAttention.Init(0, index, OpenCL, token_size, window_key, heads * variables, window, heads, 1, units_count, layers, 1, 1, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(bTrain) { if(!cClassToken[1].FeedForward(cClassToken[0].AsObject())) ReturnFalse; } if(!cAttention.FeedForward(cClassToken[1].AsObject(), NeuronOCL.getOutput())) ReturnFalse; if(!CNeuronSoftMaxOCL::feedForward(cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSoftMaxOCL::calcInputGradients(cAttention.AsObject())) ReturnFalse; if(!cClassToken[1].CalcHiddenGradients(cAttention.AsObject(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!cClassToken[0].CalcHiddenGradients(cClassToken[1].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cClassToken[1].UpdateInputWeights(cClassToken[0].AsObject())) ReturnFalse; if(!cAttention.UpdateInputWeights(cClassToken[1].AsObject(), NeuronOCL.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < cClassToken.Size(); i++) if(!cClassToken[i].Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < cClassToken.Size(); i++) if(!LoadInsideLayer(file_handle, cClassToken[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMantisAttentionUnit::SetOpenCL(COpenCLMy * obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); for(uint i = 0; i < cClassToken.Size(); i++) cClassToken[i].SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMantisAttentionUnit::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronSoftMaxOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMantisAttentionUnit *Source = source; for(uint i = 0; i < cClassToken.Size(); i++) if(!cClassToken[i].WeightsUpdate(Source.cClassToken[i].AsObject(), tau)) ReturnFalse; if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTimeFoundPatching : public CNeuronConvOCL { protected: CNeuronTransposeOCL cToVarSeq; CNeuronMultiWindowsConvWPadOCL cProjecting; CNeuronTransposeVRCOCL cToVarProjSeq; CNeuronMultiWindowsConvWPadOCL cPatchingProj; CNeuronMHFeedForward cPatchsFeedForward; CNeuronConvOCL cPatchingAgr; CNeuronProofOCL cProof; CNeuronTransposeOCL cToPatchVarEmb; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTimeFoundPatching(void) {}; ~CNeuronTimeFoundPatching(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronTimeFoundPatching; } //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint patchs, uint variables, uint embedding_size, uint patch_filters, ENUM_OPTIMIZATION optimization_type, uint batch) { uint inside_emb = MathMax((embedding_size + 2) / 6, 8); if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, 3 * inside_emb, 3 * inside_emb, embedding_size, patchs * variables, 1, optimization_type, batch)) ReturnFalse; SetActivationFunction(SoftPlus); int index = 0; if(!cToVarSeq.Init(0, index, OpenCL, count, variables, optimization, iBatch)) ReturnFalse; index++; { uint windows[] = {3, 5, 7}; if(!cProjecting.Init(0, index, OpenCL, windows, 1, inside_emb, count, variables, optimization, iBatch)) ReturnFalse; cProjecting.SetActivationFunction(SoftPlus); } index++; if(!cToVarProjSeq.Init(0, index, OpenCL, variables, count, 3 * inside_emb, optimization, iBatch)) ReturnFalse; index++; { uint step = (count + patchs) / (patchs + 1); uint windows[] = {step, 3 * step / 2, 2 * step}; if(!cPatchingProj.Init(0, index, OpenCL, windows, step, patch_filters, patchs, 3 * inside_emb * variables, optimization, iBatch)) ReturnFalse; cPatchingProj.SetActivationFunction(SoftPlus); } index++; if(!cPatchsFeedForward.Init(0, index, OpenCL, 3 * patch_filters, 6 * patch_filters, patchs, 3 * inside_emb * variables, 3, optimization, iBatch)) ReturnFalse; index++; if(!cPatchingAgr.Init(0, index, OpenCL, 3 * patch_filters, 3 * patch_filters, patch_filters, patchs, 3 * inside_emb * variables, optimization, iBatch)) ReturnFalse; cPatchingAgr.SetActivationFunction(SoftPlus); index++; if(!cProof.Init(0, index, OpenCL, patch_filters, patch_filters, 3 * patchs * inside_emb * variables, optimization, iBatch)) ReturnFalse; index++; if(!cToPatchVarEmb.Init(0, index, OpenCL, variables * 3 * inside_emb, patchs, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cToVarSeq.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjecting.FeedForward(cToVarSeq.AsObject())) ReturnFalse; if(!cToVarProjSeq.FeedForward(cProjecting.AsObject())) ReturnFalse; if(!cPatchingProj.FeedForward(cToVarProjSeq.AsObject())) ReturnFalse; if(!cPatchsFeedForward.FeedForward(cPatchingProj.AsObject())) ReturnFalse; if(!cPatchingAgr.FeedForward(cPatchsFeedForward.AsObject())) ReturnFalse; if(!cProof.FeedForward(cPatchingAgr.AsObject())) ReturnFalse; if(!cToPatchVarEmb.FeedForward(cProof.AsObject())) ReturnFalse; //--- return CNeuronConvOCL::feedForward(cToPatchVarEmb.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cToPatchVarEmb.AsObject())) ReturnFalse; if(!cProof.CalcHiddenGradients(cToPatchVarEmb.AsObject())) ReturnFalse; if(cPatchingAgr.Activation() != None) if(!DeActivation(cPatchingAgr.getOutput(), cPatchingAgr.getGradient(), cPatchingAgr.getGradient(), cPatchingAgr.Activation())) ReturnFalse; if(!cPatchingAgr.CalcHiddenGradients(cProof.AsObject())) ReturnFalse; if(!cPatchsFeedForward.CalcHiddenGradients(cPatchingAgr.AsObject())) ReturnFalse; if(!cPatchingProj.CalcHiddenGradients(cPatchsFeedForward.AsObject())) ReturnFalse; if(!cToVarProjSeq.CalcHiddenGradients(cPatchingProj.AsObject())) ReturnFalse; if(!cProjecting.CalcHiddenGradients(cToVarProjSeq.AsObject())) ReturnFalse; if(!cToVarSeq.CalcHiddenGradients(cProjecting.AsObject())) ReturnFalse; //--- return NeuronOCL.CalcHiddenGradients(cToVarSeq.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cProjecting.UpdateInputWeights(cToVarSeq.AsObject())) ReturnFalse; if(!cPatchingProj.UpdateInputWeights(cToVarProjSeq.AsObject())) ReturnFalse; if(!cPatchsFeedForward.UpdateInputWeights(cPatchingProj.AsObject())) ReturnFalse; if(!cPatchingAgr.UpdateInputWeights(cPatchsFeedForward.AsObject())) ReturnFalse; //--- return CNeuronConvOCL::updateInputWeights(cToPatchVarEmb.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronTimeFoundPatching *Source = source; if(!cProjecting.WeightsUpdate(Source.cProjecting.AsObject(), tau)) ReturnFalse; if(!cPatchingProj.WeightsUpdate(Source.cPatchingProj.AsObject(), tau)) ReturnFalse; if(!cPatchsFeedForward.WeightsUpdate(Source.cPatchsFeedForward.AsObject(), tau)) ReturnFalse; if(!cPatchingAgr.WeightsUpdate(Source.cPatchingAgr.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cToVarSeq.Save(file_handle)) ReturnFalse; if(!cProjecting.Save(file_handle)) ReturnFalse; if(!cToVarProjSeq.Save(file_handle)) ReturnFalse; if(!cPatchingProj.Save(file_handle)) ReturnFalse; if(!cPatchsFeedForward.Save(file_handle)) ReturnFalse; if(!cPatchingAgr.Save(file_handle)) ReturnFalse; if(!cProof.Save(file_handle)) ReturnFalse; if(!cToPatchVarEmb.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundPatching::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjecting.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToVarProjSeq.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingProj.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchsFeedForward.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPatchingAgr.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProof.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cToPatchVarEmb.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTimeFoundPatching::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); cToVarSeq.SetOpenCL(OpenCL); cProjecting.SetOpenCL(OpenCL); cToVarProjSeq.SetOpenCL(OpenCL); cPatchingProj.SetOpenCL(OpenCL); cPatchsFeedForward.SetOpenCL(OpenCL); cPatchingAgr.SetOpenCL(OpenCL); cProof.SetOpenCL(OpenCL); cToPatchVarEmb.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTimeFoundTransformerUnit : public CNeuronCrossDMHAttention { protected: CNeuronMVMHAttentionMLKV cSelfAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return updateInputWeights(NeuronOCL); } public: CNeuronTimeFoundTransformerUnit(void) {}; ~CNeuronTimeFoundTransformerUnit(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTimeFoundTransformerUnit; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint heads, uint heads_kv, uint units_count, uint layers, uint layers_to_one_kv, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossDMHAttention::Init(numOutputs, myIndex, open_cl, window, window_key, variables, window, units_count * variables, heads, layers, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); if(!cSelfAttention.Init(0, 0, OpenCL, window, window_key, heads, heads_kv, units_count, layers, layers_to_one_kv, variables, optimization, iBatch)) ReturnFalse; cSelfAttention.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cSelfAttention.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronCrossDMHAttention::feedForward(NeuronOCL, cSelfAttention.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; CBufferFloat *grad = prevLayer.getGradient(); if(!grad || !grad.Fill(0)) ReturnFalse; //--- if(!CNeuronCrossDMHAttention::calcInputGradients(prevLayer, cSelfAttention.getOutput(), cSelfAttention.getPrevOutput(), (ENUM_ACTIVATION)cSelfAttention.Activation())) ReturnFalse; if(!prevLayer.SetGradient(cSelfAttention.getPrevOutput(), false) || !prevLayer.CalcHiddenGradients(cSelfAttention.AsObject()) || !SumAndNormilize(grad, prevLayer.getGradient(), grad, GetWindow(), false, 0, 0, 0, 1) || !prevLayer.SetGradient(grad, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cSelfAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronCrossDMHAttention::updateInputWeights(NeuronOCL, cSelfAttention.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronCrossDMHAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTimeFoundTransformerUnit *Source = source; if(!cSelfAttention.WeightsUpdate(Source.cSelfAttention.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Save(const int file_handle) { if(!CNeuronCrossDMHAttention::Save(file_handle)) ReturnFalse; if(!cSelfAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeFoundTransformerUnit::Load(const int file_handle) { if(!CNeuronCrossDMHAttention::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSelfAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTimeFoundTransformerUnit::SetOpenCL(COpenCLMy * obj) { CNeuronCrossDMHAttention::SetOpenCL(obj); cSelfAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSwiGLUOCL : public CNeuronBaseOCL { protected: CNeuronConvOCL caProjections[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSwiGLUOCL(void) {}; ~CNeuronSwiGLUOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSwiGLUOCL; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual uint GetWindow(void) const { return caProjections[0].GetWindow(); } virtual uint GetWindowOut(void) const { return caProjections[0].GetFilters();} virtual uint GetVariables(void) const { return caProjections[0].GetVariables(); } virtual uint GetUnits(void) const { return caProjections[0].GetUnits(); } virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window_out * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); for(uint i = 0; i < caProjections.Size(); i++) { if(!caProjections[i].Init(0, i, OpenCL, window, step, window_out, units_count, variables, optimization, iBatch)) ReturnFalse; } caProjections[0].SetActivationFunction(GELU); caProjections[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { for(uint i = 0; i < caProjections.Size(); i++) if(!caProjections[i].FeedForward(NeuronOCL)) ReturnFalse; if(!ElementMult(caProjections[0].getOutput(), caProjections[1].getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!ElementMultGrad(caProjections[0].getOutput(), caProjections[0].getGradient(), caProjections[1].getOutput(), caProjections[1].getGradient(), Gradient, caProjections[0].Activation(), caProjections[1].Activation() )) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(caProjections[0].AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(caProjections[1].AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false) ) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { for(uint i = 0; i < caProjections.Size(); i++) if(!caProjections[i].UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSwiGLUOCL *Source = source; for(uint i = 0; i < caProjections.Size(); i++) if(!caProjections[i].WeightsUpdate(Source.caProjections[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; for(uint i = 0; i < caProjections.Size(); i++) if(!caProjections[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSwiGLUOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; for(uint i = 0; i < caProjections.Size(); i++) if(!LoadInsideLayer(file_handle, caProjections[i].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSwiGLUOCL::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < caProjections.Size(); i++) caProjections[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMaskMultiWinConv : public CNeuronConvOCL { protected: uint iWindowsTotal; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronMaskMultiWinConv(void) {}; ~CNeuronMaskMultiWinConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint windows_total, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMaskMultiWinConv; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint windows_total, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint win = window * windows_total + MathMax(windows_total, 1) - 1; if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, win, win, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; iWindowsTotal = windows_total; iWindow = window; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::feedForward(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput) { if(!OpenCL || !Output || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowOut, iVariables}; const int kernel = def_k_FeedForwardMaskMultWinConv; setBuffer(kernel, def_k_ffmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffmmwc_matrix_o, getOutputIndex()) setArgument(kernel, def_k_ffmmwc_activation, Activation()) setArgument(kernel, def_k_ffmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_ffmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_ffmmwc_windows_total, (int)iWindowsTotal) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::calcInputGradients(CNeuronBaseOCL * NeuronOCL, CBufferFloat * SecondInput, CBufferFloat * SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !SecondInput || !NeuronOCL.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {Neurons() / (iVariables * iWindowOut), iWindowsTotal, iVariables}; const int kernel = def_k_CalcHiddenGradientMaskMultWinConv; setBuffer(kernel, def_k_chgmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgmmwc_masks, SecondInput.GetIndex()) setBuffer(kernel, def_k_chgmmwc_masks_g, SecondGradient.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_chgmmwc_activation, NeuronOCL.Activation()) setArgument(kernel, def_k_chgmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_chgmmwc_window_in, (int)iWindow) setArgument(kernel, def_k_chgmmwc_window_out, (int)iWindowOut) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::updateInputWeights(CNeuronBaseOCL * NeuronOCL, CBufferFloat * second) { if(!OpenCL || !Gradient || !WeightsConv || !NeuronOCL || !NeuronOCL.getOutput() || !second) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {(iWindow + 1)*iWindowsTotal, iWindowOut, iVariables}; const int kernel = def_k_UpdateWeightsMaskMultWinConvAdam; setBuffer(kernel, def_k_uwmmwc_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwmmwc_masks, second.GetIndex()) setBuffer(kernel, def_k_chgmmwc_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_v, SecondMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwmmwc_matrix_og, getGradientIndex()) setArgument(kernel, def_k_uwmmwc_inputs, NeuronOCL.Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_outputs, Neurons() / iVariables) setArgument(kernel, def_k_uwmmwc_windows_total, (int)iWindowsTotal) setArgument(kernel, def_k_uwmmwc_l, lr) setArgument(kernel, def_k_uwmmwc_b1, b1) setArgument(kernel, def_k_uwmmwc_b2, b2) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindowsTotal) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMaskMultiWinConv::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindowsTotal = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTimeMoESparseExperts : public CNeuronBaseOCL { protected: CNeuronSwiGLUOCL cExpertsIn; CNeuronSwiGLUOCL cSharedIn; CNeuronMaskMultiWinConv cExpertsOut; CNeuronConvOCL cSharedOut; CNeuronTopKGates cMasks; CNeuronConvOCL cSharedGates; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTimeMoESparseExperts(void) {}; ~CNeuronTimeMoESparseExperts(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, uint experts, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTimeMoESparseExperts; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual uint GetWindow(void) const { return cSharedIn.GetWindow(); } virtual uint GetVariables(void) const { return cSharedIn.GetVariables(); } virtual uint GetUnits(void) const { return cSharedIn.GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, uint experts, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- int index = 0; if(!cExpertsIn.Init(0, index, OpenCL, window, window, window_out * experts, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cSharedIn.Init(0, index, OpenCL, window, window, window_out, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cExpertsOut.Init(0, index, OpenCL, window_out, experts, window, units_count, variables, optimization, iBatch)) ReturnFalse; cExpertsOut.SetActivationFunction(None); index++; if(!cSharedOut.Init(0, index, OpenCL, window_out, window_out, window, units_count, variables, optimization, iBatch)) ReturnFalse; cSharedOut.SetActivationFunction(None); index++; if(!cMasks.Init(0, index, OpenCL, window, units_count, experts, topK, optimization, iBatch)) ReturnFalse; index++; if(!cSharedGates.Init(0, index, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; cSharedGates.SetActivationFunction(SIGMOID); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cExpertsIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cSharedIn.FeedForward(NeuronOCL)) ReturnFalse; if(!cSharedGates.FeedForward(NeuronOCL)) ReturnFalse; if(!cMasks.FeedForward(NeuronOCL)) ReturnFalse; if(!cExpertsOut.FeedForward(cExpertsIn.AsObject(), cMasks.getOutput())) ReturnFalse; if(!cSharedOut.FeedForward(cSharedIn.AsObject())) ReturnFalse; if(!ElementMult(cSharedOut.getOutput(), cSharedGates.getOutput(), cSharedOut.getPrevOutput())) ReturnFalse; const int window = (int)cSharedGates.GetWindow(); if(!SumAndNormilize(cExpertsOut.getOutput(), cSharedOut.getPrevOutput(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), PrevOutput, Output, window, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cExpertsOut.getOutput(), cExpertsOut.getGradient(), Gradient, cExpertsOut.Activation())) ReturnFalse; if(!ElementMultGrad(cSharedOut.getOutput(), cSharedOut.getGradient(), cSharedGates.getOutput(), cSharedGates.getGradient(), Gradient, cSharedOut.Activation(), cSharedGates.Activation())) ReturnFalse; //--- if(!cExpertsIn.CalcHiddenGradients(cExpertsOut.AsObject(), cMasks.getOutput(), cMasks.getGradient(), (ENUM_ACTIVATION)cMasks.Activation())) ReturnFalse; if(!cSharedIn.CalcHiddenGradients(cSharedOut.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cExpertsIn.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; const int window = (int)cSharedGates.GetWindow(); if(!SumAndNormilize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSharedIn.AsObject())) ReturnFalse; if(!SumAndNormilize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMasks.AsObject())) ReturnFalse; if(!SumAndNormilize(PrevOutput, NeuronOCL.getGradient(), PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSharedGates.AsObject())) ReturnFalse; if(!SumAndNormilize(PrevOutput, NeuronOCL.getGradient(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!cExpertsIn.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSharedIn.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSharedGates.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMasks.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cExpertsOut.UpdateInputWeights(cExpertsIn.AsObject(), cMasks.getOutput())) ReturnFalse; if(!cSharedOut.UpdateInputWeights(cSharedIn.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTimeMoESparseExperts *Source = source; if(!cExpertsIn.WeightsUpdate(Source.cExpertsIn.AsObject(), tau)) ReturnFalse; if(!cSharedIn.WeightsUpdate(Source.cSharedIn.AsObject(), tau)) ReturnFalse; if(!cSharedGates.WeightsUpdate(Source.cSharedGates.AsObject(), tau)) ReturnFalse; if(!cMasks.WeightsUpdate(Source.cMasks.AsObject(), tau)) ReturnFalse; if(!cExpertsOut.WeightsUpdate(Source.cExpertsOut.AsObject(), tau)) ReturnFalse; if(!cSharedOut.WeightsUpdate(Source.cSharedOut.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cExpertsIn.Save(file_handle)) ReturnFalse; if(!cSharedIn.Save(file_handle)) ReturnFalse; if(!cExpertsOut.Save(file_handle)) ReturnFalse; if(!cSharedOut.Save(file_handle)) ReturnFalse; if(!cMasks.Save(file_handle)) ReturnFalse; if(!cSharedGates.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoESparseExperts::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cExpertsIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedIn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cExpertsOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedOut.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMasks.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSharedGates.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTimeMoESparseExperts::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cExpertsIn.SetOpenCL(OpenCL); cSharedIn.SetOpenCL(OpenCL); cExpertsOut.SetOpenCL(OpenCL); cSharedOut.SetOpenCL(OpenCL); cMasks.SetOpenCL(OpenCL); cSharedGates.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTimeMoESparseExperts::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cExpertsIn.TrainMode(bTrain); cSharedIn.TrainMode(bTrain); cExpertsOut.TrainMode(bTrain); cSharedOut.TrainMode(bTrain); cMasks.TrainMode(bTrain); cSharedGates.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTimeMoEAttention : public CNeuronCrossDMHAttention { protected: //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return updateInputWeights(NeuronOCL); } public: CNeuronTimeMoEAttention(void) {}; ~CNeuronTimeMoEAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTimeMoEAttention; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_count, uint window_cross, uint units_cross, uint heads, uint layers, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count, optimization_type, batch)) ReturnFalse; //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronRelativeSelfAttention *attention = NULL; CNeuronRelativeCrossAttention *cross = NULL; CNeuronTimeMoESparseExperts *MoE = NULL; bool use_self = units_count > 0; int layer = 0; for(uint i = 0; i < layers; i++) { if(use_self) { attention = new CNeuronRelativeSelfAttention(); if(!attention || !attention.Init(0, layer, OpenCL, window, window_key, units_count, heads, optimization, iBatch) || !cLayers.Add(attention) ) { DeleteObj(attention); ReturnFalse; } layer++; } cross = new CNeuronRelativeCrossAttention(); if(!cross || !cross.Init(0, layer, OpenCL, window, window_key, units_count, heads, window_cross, units_cross, optimization, iBatch) || !cLayers.Add(cross) ) { DeleteObj(cross); ReturnFalse; } layer++; MoE = new CNeuronTimeMoESparseExperts(); if(!MoE || !MoE.Init(0, layer, OpenCL, window, experts_dimension, units_count, 1, experts, topK, optimization, iBatch) || !cLayers.Add(MoE) ) { DeleteObj(MoE); ReturnFalse; } layer++; } //--- SetOutput(MoE.getOutput(), true); SetGradient(MoE.getGradient(), true); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; return CNeuronCrossDMHAttention::feedForward(NeuronOCL, NeuronOCL.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::calcInputGradients(CNeuronBaseOCL * prevLayer) { if(!prevLayer) ReturnFalse; CBufferFloat *temp = prevLayer.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; if(!CNeuronCrossDMHAttention::calcInputGradients(prevLayer, prevLayer.getOutput(), prevLayer.getPrevOutput(), (ENUM_ACTIVATION) prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(temp, prevLayer.getPrevOutput(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTimeMoEAttention::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- return CNeuronCrossDMHAttention::updateInputWeights(NeuronOCL, NeuronOCL.getOutput()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAdaptConv : public CNeuronConvOCL { protected: CBufferFloat bMainFreq; //--- virtual bool FFT(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *out_re, CBufferFloat *out_im, uint variables, bool reverse = false); virtual bool PeriodsFinding(CBufferFloat *inp_re, CBufferFloat *inp_im, CBufferFloat *main_freq, uint variables); virtual bool AdaptiveConvolution(CNeuronBaseOCL *NeuronOCL, CBufferFloat *main_freq); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAdaptConv(void) {}; ~CNeuronAdaptConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronAdaptConv; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; //--- bMainFreq.BufferFree(); if(!bMainFreq.BufferInit(iVariables, 1) || !bMainFreq.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::FFT(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * out_re, CBufferFloat * out_im, uint variables, bool reverse = false) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; uint kernel = def_k_FFT; setBuffer(kernel, def_k_fft_inputs_re, inp_re.GetIndex()) setBuffer(kernel, def_k_fft_inputs_im, (!!inp_im ? inp_im.GetIndex() : inp_re.GetIndex())) setBuffer(kernel, def_k_fft_outputs_re, out_re.GetIndex()) setBuffer(kernel, def_k_fft_outputs_im, out_im.GetIndex()) setArgument(kernel, def_k_fft_input_window, (int)(inp_re.Total() / variables)) setArgument(kernel, def_k_fft_input_complex, int(!!inp_im)) setArgument(kernel, def_k_fft_output_window, (int)(out_re.Total() / variables)) setArgument(kernel, def_k_fft_reverse, int(reverse)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::PeriodsFinding(CBufferFloat * inp_re, CBufferFloat * inp_im, CBufferFloat * windows, uint variables) { uint global_work_offset[1] = {0}; uint global_work_size[1] = {variables}; uint kernel = def_k_MainFreq; setBuffer(kernel, def_k_mf_freq_r, inp_re.GetIndex()) setBuffer(kernel, def_k_mf_freq_im, inp_im.GetIndex()) setBuffer(kernel, def_k_mf_main_freq, windows.GetIndex()) setArgument(kernel, def_k_mf_dimension, (int)(inp_re.Total() / variables)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::AdaptiveConvolution(CNeuronBaseOCL * NeuronOCL, CBufferFloat * main_freq) { uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {Output.Total() / (iWindowOut * iVariables), iWindowOut, iVariables}; uint kernel = def_k_FeedForwardAdaptConv; setBuffer(kernel, def_k_ffac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ffac_matrix_o, getOutputIndex()) setBuffer(kernel, def_k_ffac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_ffac_main_freq, main_freq.GetIndex()) setArgument(kernel, def_k_ffac_window_in, iWindow) setArgument(kernel, def_k_ffac_inputs, (int)(NeuronOCL.Neurons())) setArgument(kernel, def_k_ffac_activation, (int)(activation)) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!FFT(NeuronOCL.getOutput(), NULL, Output, PrevOutput, iVariables, false)) ReturnFalse; if(!PeriodsFinding(Output, PrevOutput, GetPointer(bMainFreq), iVariables)) ReturnFalse; //--- return AdaptiveConvolution(NeuronOCL, GetPointer(bMainFreq)); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0}; uint global_work_size[] = {NeuronOCL.Neurons() / iVariables, iVariables}; uint kernel = def_k_CalcHiddenGradientAdaptConv; setBuffer(kernel, def_k_chgac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chgac_matrix_ig, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chgac_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_chgac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_chgac_main_freq, bMainFreq.GetIndex()) setArgument(kernel, def_k_chgac_window_in, iWindow) setArgument(kernel, def_k_chgac_window_out, iWindowOut) setArgument(kernel, def_k_chgac_activation, (int)(NeuronOCL.Activation())) setArgument(kernel, def_k_chgac_outputs, (int)(Neurons())) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::updateInputWeights(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow + 1, iWindowOut, iVariables}; uint kernel = def_k_UpdateWeightsAdaptConvAdam; setBuffer(kernel, def_k_uwac_matrix_i, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_uwac_matrix_og, getGradientIndex()) setBuffer(kernel, def_k_uwac_matrix_w, WeightsConv.GetIndex()) setBuffer(kernel, def_k_uwac_matrix_m, FirstMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwac_matrix_v, SecondMomentumConv.GetIndex()) setBuffer(kernel, def_k_uwac_main_freq, bMainFreq.GetIndex()) setArgument(kernel, def_k_uwac_outputs, (int)Neurons()) setArgument(kernel, def_k_uwac_inputs, int(NeuronOCL.Neurons())) setArgument(kernel, def_k_uwac_l, lr) setArgument(kernel, def_k_uwac_b1, b1) setArgument(kernel, def_k_uwac_b2, b2) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!bMainFreq.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptConv::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!bMainFreq.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAdaptConv::SetOpenCL(COpenCLMy * obj) { CNeuronConvOCL::SetOpenCL(obj); bMainFreq.BufferFree(); bMainFreq.BufferFree(); bMainFreq.BufferInit(iVariables, 1); bMainFreq.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRoPE : public CNeuronPositionEncoder { protected: uint iWindow; uint iVariables; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL)override; public: CNeuronRoPE(void) : iWindow(0), iVariables(0) {}; ~CNeuronRoPE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual int Type(void) override const { return defNeuronRoPE; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint count, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(window % 2 > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, count * window * variables, optimization_type, batch)) ReturnFalse; //--- iWindow = window; iVariables = variables; //--- matrix pe = matrix::Zeros(count, iWindow); vector position = vector::Ones(count); position = position.CumSum() - 1; for(uint i = 0; i < iWindow / 2; i++) { vector temp = position / MathPow(10000.0f, 2.0f * i / window); pe.Col(MathCos(temp), i * 2); pe.Col(MathSin(temp), i * 2 + 1); } PositionEncoder.BufferFree(); if(!PositionEncoder.AssignArray(pe)) ReturnFalse; SetActivationFunction(None); //--- return PositionEncoder.BufferCreate(open_cl); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_RoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- uint global_work_offset[] = {0, 0, 0}; uint global_work_size[] = {iWindow / 2, PositionEncoder.Total() / iWindow, iVariables}; uint kernel = def_k_CalcHiddenGradRoPE; setBuffer(kernel, def_k_rope_inputs, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_rope_position_emb, PositionEncoder.GetIndex()) setBuffer(kernel, def_k_rope_outputs, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Save(const int file_handle) { if(!CNeuronPositionEncoder::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iWindow)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRoPE::Load(const int file_handle) { if(!CNeuronPositionEncoder::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iWindow = uint(FileReadInteger(file_handle)); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = uint(FileReadInteger(file_handle)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CParams : public CNeuronBaseOCL { protected: CNeuronBaseOCL cOne; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { return FeedForward(); } ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return UpdateInputWeights(); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return true; } public: CParams(void) {}; ~CParams(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) override; virtual bool Identity(const int rows, const int cols); //--- virtual bool FeedForward(void); virtual bool UpdateInputWeights(void); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defParams; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual CBufferFloat *getWeightsParams(void) { return cOne.getWeights(); } ///< Get pointer of gradient buffer @return Pointer to object //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; if(!cOne.Init(numNeurons, 0, OpenCL, 1, optimization, iBatch)) ReturnFalse; cOne.SetActivationFunction(None); if(!cOne.getOutput().Fill(1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::FeedForward(void) { if(!bTrain) return true; //--- return CNeuronBaseOCL::feedForward(cOne.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::UpdateInputWeights(void) { return CNeuronBaseOCL::updateInputWeights(cOne.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cOne.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cOne.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CParams::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOne.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::WeightsUpdate(CNeuronBaseOCL * source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CParams *Source = source; if(!cOne.WeightsUpdate(Source.cOne.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CParams::Identity(const int rows, const int cols) { if(rows * cols != Neurons()) ReturnFalse; matrix ident = matrix::Identity(rows, cols); if(!ident.Reshape(rows * cols, 1)) ReturnFalse; if(!ident.Resize(rows * cols, 2)) ReturnFalse; if(!ident.Col(vector::Zeros(rows * cols), 1)) ReturnFalse; CBufferFloat *w = cOne.getWeights(); if(!w || !w.AssignArray(ident)) ReturnFalse; w.BufferFree(); if(!w.BufferCreate(OpenCL)) ReturnFalse; //--- return FeedForward(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronK2VAEEncoder : public CNeuronBaseOCL { protected: //--- Koopman CNeuronTimeMoESparseExperts cKoopman; CNeuronBaseOCL cKoopmanPred; CNeuronBaseOCL cKoopmanRest; //--- CNeuronTimeMoEAttention cAuxiliaryNet; //--- Kalman Filter CParams cB; // Control input matrix B CParams cF; // State transition matrix F CParams cH; // Observation matrix H CParams cQ; // Learnable covariance matrices Q CParams cR; // Learnable covariance matrices R CNeuronBaseOCL cP; // Covariance matrices P //--- CNeuronTransposeOCL cFT; CNeuronTransposeOCL cHT; CNeuronTransposeOCL cQT; CNeuronTransposeOCL cRT; CNeuronTransposeOCL cPT; //--- CNeuronBaseOCL cQ_QT; CNeuronBaseOCL cR_RT; CNeuronBaseOCL cXPred; CNeuronBaseOCL cF_P; CNeuronBaseOCL cPPred; //--- CNeuronBaseOCL cP_HT; CNeuronBaseOCL cH_P_HT; matrix mS, mSGrad; CNeuronBaseOCL cSInv; CNeuronBaseOCL cK; CNeuronTransposeOCL cKT; CNeuronBaseOCL cYPred; CNeuronBaseOCL cDeltY; CNeuronBaseOCL cX; CNeuronBaseOCL cK_H; CNeuronBaseOCL cIdifK_H; CNeuronTransposeOCL cIdifK_HT; matrix mP; matrix mPGrad; matrix mNoise; matrix mKL; matrix mGrad; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronK2VAEEncoder(void) {}; ~CNeuronK2VAEEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units_cross, uint heads, uint layers, uint scenarios, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) const { return defNeuronK2VAEEncoder; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window, uint window_key, uint units_cross, uint heads, uint layers, uint scenarios, uint experts, uint experts_dimension, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * scenarios, optimization_type, batch)) ReturnFalse; //--- Koopman int index = 0; if(!cKoopman.Init(0, index, OpenCL, window, 2 * window, units_cross, 1, experts, topK, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanRest.Init(0, index, OpenCL, window * (units_cross - 1), optimization, iBatch)) ReturnFalse; //--- index++; if(!cAuxiliaryNet.Init(0, index, OpenCL, window, window_key, 1, window, units_cross - 1, heads, layers, experts, experts_dimension, topK, optimization, iBatch)) ReturnFalse; //--- Kalman Filter index++; if(!cB.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cB.Identity(window, window)) ReturnFalse; index++; if(!cF.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cF.Identity(window, window)) ReturnFalse; index++; if(!cH.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cH.Identity(window, window)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cQ.Identity(window, window)) ReturnFalse; index++; if(!cR.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cR.Identity(window, window)) ReturnFalse; index++; if(!cP.Init(0, index, OpenCL, window * window, optimization, iBatch) || !cP.getOutput().Fill(matrix::Identity(window, window))) ReturnFalse; //--- index++; if(!cFT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cHT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cQT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cRT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cPT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQ_QT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cR_RT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cXPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cF_P.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cPPred.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cP_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cH_P_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- mS = mSGrad = matrix::Zeros(window, window); //--- index++; if(!cSInv.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cK.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cKT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cYPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cDeltY.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cX.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_HT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- mKL = mP = mPGrad = mS; mNoise = mGrad = matrix::Zeros(scenarios, window); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::feedForward(CNeuronBaseOCL * NeuronOCL) { //--- Koopman if(!cKoopman.FeedForward(NeuronOCL)) ReturnFalse; //--- Pred / Rest if(!DeConcat(cKoopmanPred.getOutput(), cKoopmanRest.getPrevOutput(), cKoopman.getOutput(), cKoopman.GetWindow(), cKoopman.Neurons() - cKoopman.GetWindow(), 1)) ReturnFalse; if(!Different(NeuronOCL.getOutput(), cKoopmanRest.getPrevOutput(), cKoopmanRest.getOutput(), cKoopman.GetWindow())) ReturnFalse; //--- Rest Attention if(!cAuxiliaryNet.FeedForward(cKoopmanRest.AsObject())) ReturnFalse; //--- Kalman Filter if(bTrain) { if(!cB.FeedForward()) ReturnFalse; if(!cF.FeedForward()) ReturnFalse; if(!cH.FeedForward()) ReturnFalse; if(!cQ.FeedForward()) ReturnFalse; if(!cR.FeedForward()) ReturnFalse; if(!cFT.FeedForward(cF.AsObject())) ReturnFalse; if(!cHT.FeedForward(cH.AsObject())) ReturnFalse; if(!cQT.FeedForward(cQ.AsObject())) ReturnFalse; if(!cRT.FeedForward(cR.AsObject())) ReturnFalse; if(!MatMul(cQ.getOutput(), cQT.getOutput(), cQ_QT.getOutput(), cQT.GetCount(), cQT.GetWindow(), cQT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cR.getOutput(), cRT.getOutput(), cR_RT.getOutput(), cRT.GetCount(), cRT.GetWindow(), cRT.GetCount(), 1, true)) ReturnFalse; } //--- Prediction step if(!MatMul(NeuronOCL.getOutput(), cFT.getOutput(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cAuxiliaryNet.getOutput(), cB.getOutput(), cXPred.getPrevOutput(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cXPred.getGradient(), cXPred.getPrevOutput(), cXPred.getOutput(), cFT.GetCount(), false, 0, 0, 0, 1)) ReturnFalse; //--- cP.SwapOutputs(); if(!MatMul(cF.getOutput(), cP.getPrevOutput(), cF_P.getOutput(), cFT.GetCount(), cFT.GetWindow(), cP.Neurons() / cFT.GetWindow(), 1, true)) ReturnFalse; if(!MatMul(cF_P.getOutput(), cFT.getOutput(), cPPred.getOutput(), cFT.GetCount(), cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cPPred.getOutput(), cQ_QT.getOutput(), cPPred.getOutput(), cHT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- Update step if(!MatMul(cPPred.getOutput(), cHT.getOutput(), cP_HT.getOutput(), cPPred.Neurons() / cHT.GetWindow(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cH.getOutput(), cP_HT.getOutput(), cH_P_HT.getOutput(), cHT.GetCount(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cH_P_HT.getOutput(), cR_RT.getOutput(), cR_RT.getPrevOutput(), cRT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(cR_RT.getPrevOutput().GetData(mSGrad) <= 0) ReturnFalse; mS = mSGrad.Inv(); if(mS.Rows() == 0) { double max = MathAbs(mSGrad).Max(); if(max > 1) mSGrad = mSGrad / max; mSGrad = mSGrad + matrix::Identity(mSGrad.Rows(), mSGrad.Cols()); mS = mSGrad.Inv(); if(mS.Rows() == 0) { mSGrad = mSGrad + mSGrad.Transpose(); vector eigvals; matrix eigvecs; if(!mSGrad.Eig(eigvecs, eigvals)) ReturnFalse; if(eigvals.Size() > 0) { if(!eigvals.Clip(1e-6, DBL_MAX)) ReturnFalse; mSGrad = matrix::Zeros(eigvals.Size(), eigvals.Size()); mSGrad.Diag(eigvals); mSGrad = eigvecs.MatMul(mSGrad.MatMul(eigvecs.Transpose())); mSGrad = mSGrad + mSGrad.Transpose(); mS = mSGrad.Inv(); } if(mS.Rows() == 0) { mSGrad.Identity(); mS = mSGrad; } } } cSInv.getOutput().Fill(mS); if(!MatMul(cP_HT.getOutput(), cSInv.getOutput(), cK.getOutput(), cHT.GetCount(), (int)mS.Rows(), (int)mS.Cols(), 1, true)) ReturnFalse; //--- Measurement update if(!MatMul(cXPred.getOutput(), cHT.getOutput(), cYPred.getOutput(), 1, cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!Different(cKoopmanPred.getOutput(), cYPred.getOutput(), cDeltY.getOutput(), 1, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cDeltY.getOutput(), cK.getOutput(), cX.getPrevOutput(), 1, cDeltY.Neurons(), cX.Neurons(), 1, true)) ReturnFalse; if(!SumAndNormilize(cX.getPrevOutput(), cXPred.getOutput(), cX.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Joseph stabilized form for P if(!MatMul(cK.getOutput(), cH.getOutput(), cK_H.getOutput(), cKT.GetCount(), cKT.GetWindow(), cHT.GetWindow(), 1, true)) ReturnFalse; if(!IdentDifferent(cK_H.getOutput(), cIdifK_H.getOutput(), cHT.GetWindow(), 0, 0, 1)) ReturnFalse; if(!cIdifK_HT.FeedForward(cIdifK_H.AsObject())) ReturnFalse; if(!cKT.FeedForward(cK.AsObject())) ReturnFalse; if(!MatMul(cIdifK_H.getOutput(), cPPred.getOutput(), cIdifK_H.getPrevOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetWindow(), 1, true)) ReturnFalse; if(!MatMul(cIdifK_H.getPrevOutput(), cIdifK_HT.getOutput(), cP.getOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cK.getOutput(), cR_RT.getOutput(), cK.getPrevOutput(), cKT.GetCount(), cRT.GetCount(), cRT.GetCount(), 1, true)) ReturnFalse; if(!MatMul(cK.getPrevOutput(), cKT.getOutput(), cKT.getPrevOutput(), cKT.GetCount(), cKT.GetWindow(), cKT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cP.getOutput(), cKT.getPrevOutput(), cP.getOutput(), cPT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!cPT.FeedForward(cP.AsObject())) ReturnFalse; if(!SumAndNormilize(cP.getOutput(), cPT.getOutput(), cP.getOutput(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; if(!SumAndNormilize(cX.getOutput(), cAuxiliaryNet.getOutput(), cX.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Sample Output if(!cP.getOutput().GetData(mPGrad)) ReturnFalse; if(mPGrad.HasNan() > 0) { mPGrad.Identity(); if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } mP = mPGrad.Inv(); if(mP.Rows() == 0) { double max = MathAbs(mPGrad).Max(); if(max > 1) mPGrad = mPGrad / max; mPGrad = mPGrad + matrix::Identity(mPGrad.Rows(), mPGrad.Cols()); mP = mPGrad.Inv(); if(mP.Rows() == 0) { mPGrad = mPGrad + mPGrad.Transpose(); vector eigvals; matrix eigvecs; if(!mPGrad.Eig(eigvecs, eigvals)) ReturnFalse; if(eigvals.Size() > 0) { if(!eigvals.Clip(1e-6, DBL_MAX)) ReturnFalse; mPGrad = matrix::Zeros(eigvals.Size(), eigvals.Size()); mPGrad.Diag(eigvals); mPGrad = eigvecs.MatMul(mPGrad.MatMul(eigvecs.Transpose())); mPGrad = mPGrad + mPGrad.Transpose(); mP = mPGrad.Inv(); } if(mP.Rows() == 0) { mPGrad.Identity(); if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; mP = mPGrad.Inv(); } else if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } else if(!cP.getOutput().Fill(mPGrad)) ReturnFalse; } mNoise.Random(-1, 1); matrix temp = mNoise.MatMul(mP); if(!PrevOutput.Fill(temp)) ReturnFalse; if(!SumVecMatrix(cX.getOutput(), PrevOutput, Output, (int)mNoise.Cols(), 1, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- From Output if(!SumVecMatrixGrad(cX.getGradient(), PrevOutput, Gradient, (int)mNoise.Cols(), 1, 0, 0, 0, 1)) ReturnFalse; if(!PrevOutput.GetData(mGrad)) ReturnFalse; mPGrad = mNoise.Transpose().MatMul(mGrad); mKL = mP.Transpose(); mKL = (mKL * (-1)).MatMul(mPGrad.MatMul(mKL)); //--- Joseph stabilized form for P if(!cPT.getGradient().Fill(mKL)) ReturnFalse; if(!cP.CalcHiddenGradients(cPT.AsObject())) ReturnFalse; if(!SumAndNormilize(cP.getGradient(), cPT.getGradient(), cP.getGradient(), 1, false, 0, 0, 0, 0.5f)) ReturnFalse; //--- if(!MatMulGrad(cK.getPrevOutput(), cKT.getPrevOutput(), cKT.getOutput(), cKT.getGradient(), cP.getGradient(), cKT.GetCount(), cKT.GetWindow(), cKT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cK.getOutput(), cK.getPrevOutput(), cR_RT.getOutput(), cR_RT.getGradient(), cKT.getPrevOutput(), cKT.GetCount(), cRT.GetCount(), cRT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cIdifK_H.getPrevOutput(), cIdifK_HT.getPrevOutput(), cIdifK_HT.getOutput(), cIdifK_HT.getGradient(), cP.getGradient(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cIdifK_H.getOutput(), cIdifK_H.getPrevOutput(), cPPred.getOutput(), cPPred.getGradient(), cIdifK_HT.getPrevOutput(), cIdifK_HT.GetCount(), cIdifK_HT.GetWindow(), cIdifK_HT.GetWindow(), 1, true)) ReturnFalse; //--- if(!cK.CalcHiddenGradients(cKT.AsObject())) ReturnFalse; if(!SumAndNormilize(cK.getGradient(), cK.getPrevOutput(), cK.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Measurement update if(!cIdifK_H.CalcHiddenGradients(cIdifK_HT.AsObject())) ReturnFalse; if(!SumAndNormilize(cIdifK_H.getGradient(), cIdifK_H.getPrevOutput(), cIdifK_H.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!IdentDifferentGrad(cK_H.getGradient(), cIdifK_H.getGradient(), cHT.GetWindow(), 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cK.getOutput(), cK.getPrevOutput(), cH.getOutput(), cH.getGradient(), cK_H.getGradient(), cKT.GetCount(), cKT.GetWindow(), cHT.GetWindow(), 1, true)) ReturnFalse; //--- if(!MatMulGrad(cDeltY.getOutput(), cDeltY.getGradient(), cK.getOutput(), cK.getPrevOutput(), cX.getGradient(), 1, cDeltY.Neurons(), cX.Neurons(), 1, true)) ReturnFalse; if(!SumAndNormilize(cK.getGradient(), cK.getPrevOutput(), cK.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!DifferentGrad(cKoopmanPred.getGradient(), cYPred.getGradient(), cDeltY.getGradient(), 1, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cXPred.getOutput(), cXPred.getGradient(), cHT.getOutput(), cHT.getGradient(), cYPred.getGradient(), 1, cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cXPred.getGradient(), cX.getGradient(), cXPred.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Update step if(!MatMulGrad(cP_HT.getOutput(), cP_HT.getGradient(), cSInv.getOutput(), cSInv.getGradient(), cK.getGradient(), cHT.GetCount(), (int)mS.Rows(), (int)mS.Cols(), 1, true)) ReturnFalse; if(cSInv.getGradient().GetData(mSGrad) <= 0) ReturnFalse; mKL = mS.Transpose(); mKL = (mKL * (-1)).MatMul(mSGrad.MatMul(mKL)); if(cH_P_HT.getGradient().Fill(mKL) <= 0) ReturnFalse; if(!MatMulGrad(cH.getOutput(), cH.getPrevOutput(), cP_HT.getOutput(), cP_HT.getPrevOutput(), cH_P_HT.getGradient(), cHT.GetCount(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cH_P_HT.getGradient(), cR_RT.getGradient(), cR_RT.getGradient(), int(mS.Cols()), false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cH.getGradient(), cH.getPrevOutput(), cH.getPrevOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cP_HT.getGradient(), cP_HT.getPrevOutput(), cP_HT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cPPred.getOutput(), cPPred.getPrevOutput(), cHT.getOutput(), cHT.getPrevOutput(), cP_HT.getGradient(), cPPred.Neurons() / cHT.GetWindow(), cHT.GetWindow(), cHT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cPPred.getGradient(), cPPred.getPrevOutput(), cQ_QT.getGradient(), int(cQT.GetWindow()), false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cHT.getGradient(), cHT.getPrevOutput(), cHT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- Prediction step if(!MatMulGrad(cF_P.getOutput(), cF_P.getGradient(), cFT.getOutput(), cFT.getGradient(), cQ_QT.getGradient(), cFT.GetCount(), cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(cF.getOutput(), cF.getPrevOutput(), cP.getPrevOutput(), cP.getGradient(), cF_P.getGradient(), cFT.GetCount(), cFT.GetWindow(), cP.Neurons() / cFT.GetWindow(), 1, true)) ReturnFalse; if(!MatMulGrad(cAuxiliaryNet.getOutput(), cAuxiliaryNet.getGradient(), cB.getOutput(), cB.getGradient(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cFT.getOutput(), cFT.getPrevOutput(), cXPred.getGradient(), 1, cFT.GetWindow(), cFT.GetCount(), 1, true)) ReturnFalse; if(!SumAndNormilize(cFT.getGradient(), cFT.getPrevOutput(), cFT.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!MatMulGrad(cR.getOutput(), cR.getPrevOutput(), cRT.getOutput(), cRT.getGradient(), cR_RT.getGradient(), cRT.GetCount(), cRT.GetWindow(), cRT.GetCount(), 1, false)) ReturnFalse; if(!MatMulGrad(cQ.getOutput(), cQ.getPrevOutput(), cQT.getOutput(), cQT.getGradient(), cQ_QT.getGradient(), cQT.GetCount(), cQT.GetWindow(), cQT.GetCount(), 1, false)) ReturnFalse; if(!cR.CalcHiddenGradients((CObject*)cRT.AsObject())) ReturnFalse; if(!SumAndNormilize(cR.getGradient(), cR.getPrevOutput(), cR.getGradient(), cRT.GetWindow(), false, 0, 0, 0, 1e-4f)) ReturnFalse; cR.getGradient().BufferRead(); if(!cQ.CalcHiddenGradients(cQT.AsObject())) ReturnFalse; if(!SumAndNormilize(cQ.getGradient(), cQ.getPrevOutput(), cQ.getGradient(), cQT.GetWindow(), false, 0, 0, 0, 1e-4f)) ReturnFalse; cQ.getGradient().BufferRead(); if(!cH.CalcHiddenGradients(cHT.AsObject())) ReturnFalse; if(!SumAndNormilize(cH.getGradient(), cH.getPrevOutput(), cH.getGradient(), cHT.GetWindow(), false, 0, 0, 0, 0.01f)) ReturnFalse; if(!cF.CalcHiddenGradients(cFT.AsObject())) ReturnFalse; if(!SumAndNormilize(cF.getGradient(), cF.getPrevOutput(), cF.getGradient(), cFT.GetWindow(), false, 0, 0, 0, 0.01f)) ReturnFalse; //--- Rest Attention if(!cKoopmanRest.CalcHiddenGradients(cAuxiliaryNet.AsObject())) ReturnFalse; //--- Pred / Rest if(!NeuronOCL.getPrevOutput().Fill(0)) ReturnFalse; if(!DifferentGrad(NeuronOCL.getPrevOutput(), cKoopmanRest.getPrevOutput(), cKoopmanRest.getGradient(), cKoopman.GetWindow())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; } if(!Concat(cKoopmanPred.getGradient(), cKoopmanRest.getPrevOutput(), cKoopman.getGradient(), cKoopman.GetWindow(), cKoopman.Neurons() - cKoopman.GetWindow(), 1)) ReturnFalse; //--- Koopman if(!NeuronOCL.CalcHiddenGradients(cKoopman.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Koopman if(!cKoopman.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cAuxiliaryNet.UpdateInputWeights(cKoopmanRest.AsObject())) ReturnFalse; //--- Kalman Filter if(!cB.UpdateInputWeights()) ReturnFalse; if(!cF.UpdateInputWeights()) ReturnFalse; if(!cH.UpdateInputWeights()) ReturnFalse; if(!cQ.UpdateInputWeights()) ReturnFalse; if(!cR.UpdateInputWeights()) ReturnFalse; //--- if(!SumAndNormilize(cQ.getWeightsParams(), cQ.getWeightsParams(), cQ.getWeightsParams(), 2 * cQT.GetWindow(), true)) ReturnFalse; if(!SumAndNormilize(cR.getWeightsParams(), cR.getWeightsParams(), cR.getWeightsParams(), 2 * cRT.GetWindow(), true)) ReturnFalse; //--- if(!cQ.FeedForward()) ReturnFalse; mKL.Random(-1, 1); mKL = mKL.TriL(); mKL = mKL.MatMul(mKL.Transpose()); float error = 0; if(!cQ.getGradient().Fill(mKL) || !cQ.calcOutputGradients(cQ.getGradient(), error) || !cQ.UpdateInputWeights()) ReturnFalse; //--- if(!cR.FeedForward()) ReturnFalse; mKL.Random(-1, 1); mKL = mKL.TriL(); mKL = mKL.MatMul(mKL.Transpose()); error = 0; if(!cR.getGradient().Fill(mKL) || !cR.calcOutputGradients(cR.getGradient(), error) || !cR.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronK2VAEEncoder *Source = source; //--- Koopman if(!cKoopman.WeightsUpdate(Source.cKoopman.AsObject(), tau)) ReturnFalse; if(!cAuxiliaryNet.WeightsUpdate(Source.cAuxiliaryNet.AsObject(), tau)) ReturnFalse; //--- Kalman Filter if(!cB.WeightsUpdate(Source.cB.AsObject(), tau)) ReturnFalse; if(!cF.WeightsUpdate(Source.cF.AsObject(), tau)) ReturnFalse; if(!cH.WeightsUpdate(Source.cH.AsObject(), tau)) ReturnFalse; if(!cQ.WeightsUpdate(Source.cQ.AsObject(), tau)) ReturnFalse; if(!cR.WeightsUpdate(Source.cR.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- Koopman if(!cKoopman.Save(file_handle)) ReturnFalse; if(!cAuxiliaryNet.Save(file_handle)) ReturnFalse; //--- Kalman Filter if(!cB.Save(file_handle)) ReturnFalse; if(!cF.Save(file_handle)) ReturnFalse; if(!cH.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cR.Save(file_handle)) ReturnFalse; if(!cP.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronK2VAEEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- Koopman if(!LoadInsideLayer(file_handle, cKoopman.AsObject())) ReturnFalse; //--- uint window = cKoopman.GetWindow(); uint units_cross = cKoopman.GetUnits(); int index = 1; if(!cKoopmanPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cKoopmanRest.Init(0, index, OpenCL, window * (units_cross - 1), optimization, iBatch)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAuxiliaryNet.AsObject())) ReturnFalse; //--- Kalman Filter if(!LoadInsideLayer(file_handle, cB.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cF.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cH.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cP.AsObject())) ReturnFalse; //--- index += 7; if(!cFT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cHT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cQT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cRT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; index++; if(!cPT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cQ_QT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cR_RT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cXPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; index++; if(!cF_P.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cPPred.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cP_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cH_P_HT.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- mS = mSGrad = matrix::Zeros(window, window); //--- index++; if(!cSInv.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cK.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; index++; if(!cKT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cYPred.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cDeltY.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cX.Init(0, index, OpenCL, window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_H.Init(0, index, OpenCL, window * window, optimization, iBatch)) ReturnFalse; //--- index++; if(!cIdifK_HT.Init(0, index, OpenCL, window, window, optimization, iBatch)) ReturnFalse; //--- mKL = mP = mPGrad = mS; mNoise = mGrad = matrix::Zeros(Neurons() / window, window); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronK2VAEEncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cKoopman.TrainMode(bTrain); cAuxiliaryNet.TrainMode(bTrain); cB.TrainMode(bTrain); cF.TrainMode(bTrain); cH.TrainMode(bTrain); cQ.TrainMode(bTrain); cR.TrainMode(bTrain); cP.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronK2VAEEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- Koopman cKoopman.SetOpenCL(OpenCL); cKoopmanPred.SetOpenCL(OpenCL); cKoopmanRest.SetOpenCL(OpenCL); //--- cAuxiliaryNet.SetOpenCL(OpenCL); //--- Kalman Filter cB.SetOpenCL(OpenCL); cF.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cR.SetOpenCL(OpenCL); cP.SetOpenCL(OpenCL); cFT.SetOpenCL(OpenCL); cHT.SetOpenCL(OpenCL); cQT.SetOpenCL(OpenCL); cRT.SetOpenCL(OpenCL); cPT.SetOpenCL(OpenCL); cQ_QT.SetOpenCL(OpenCL); cR_RT.SetOpenCL(OpenCL); cXPred.SetOpenCL(OpenCL); cF_P.SetOpenCL(OpenCL); cPPred.SetOpenCL(OpenCL); cP_HT.SetOpenCL(OpenCL); cH_P_HT.SetOpenCL(OpenCL); cSInv.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cKT.SetOpenCL(OpenCL); cYPred.SetOpenCL(OpenCL); cDeltY.SetOpenCL(OpenCL); cX.SetOpenCL(OpenCL); cK_H.SetOpenCL(OpenCL); cIdifK_H.SetOpenCL(OpenCL); cIdifK_HT.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronInterpolationAttention : public CNeuronBaseOCL { protected: //--- uint iCount; uint iDimension; //--- CParams cW; CParams cA; CParams cGL; //--- CNeuronBaseOCL cH; CNeuronBaseOCL cAdj; CNeuronBaseOCL cAttention; //--- virtual bool InterpolationAttention(CNeuronBaseOCL *NeuronOCL); virtual bool InterpolationAttentionGrad(CNeuronBaseOCL *NeuronOCL); virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronInterpolationAttention(void) { activation = None; } ~CNeuronInterpolationAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronInterpolationAttention; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * dimension, optimization_type, batch)) ReturnFalse; //--- iCount = units_count; iDimension = dimension; //--- int index = 0; if(!cW.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cA.Init(0, index, OpenCL, 2 * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cGL.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- index++; if(!cH.Init(0, index, OpenCL, iCount * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAdj.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::InterpolationAttention(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iCount * iDimension)) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iCount}; uint local_work_size[2] = {1, iCount}; const int kernel = def_k_InterpolationAttention; setBuffer(kernel, def_k_ia_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ia_A, cA.getOutputIndex()) setBuffer(kernel, def_k_ia_Adj, cAdj.getOutputIndex()) setBuffer(kernel, def_k_ia_Atten, cAttention.getOutputIndex()) setBuffer(kernel, def_k_ia_GL, cGL.getOutputIndex()) setBuffer(kernel, def_k_ia_H, cH.getOutputIndex()) setBuffer(kernel, def_k_ia_W, cW.getOutputIndex()) setBuffer(kernel, def_k_ia_matrix_out, getOutputIndex()) setArgument(kernel, def_k_ia_dimension, (int)iDimension) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::InterpolationAttentionGrad(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iCount * iDimension)) ReturnFalse; //--- CBufferFloat* temp = cA.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; temp = cGL.getGradient(); if(!temp || !temp.Fill(0)) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = {iCount, iCount}; uint local_work_size[2] = {1, iCount}; const int kernel = def_k_InterpolationAttentionGrad; setBuffer(kernel, def_k_iag_matrix_in, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_iag_matrix_in_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_iag_A, cA.getOutputIndex()) setBuffer(kernel, def_k_iag_A_gr, cA.getGradientIndex()) setBuffer(kernel, def_k_iag_Adj, cAdj.getOutputIndex()) setBuffer(kernel, def_k_iag_Atten, cAttention.getOutputIndex()) setBuffer(kernel, def_k_iag_GL, cGL.getOutputIndex()) setBuffer(kernel, def_k_iag_GL_gr, cGL.getGradientIndex()) setBuffer(kernel, def_k_iag_H, cH.getOutputIndex()) setBuffer(kernel, def_k_iag_H_gr, cH.getGradientIndex()) setBuffer(kernel, def_k_iag_W, cW.getOutputIndex()) setBuffer(kernel, def_k_iag_W_gr, cW.getGradientIndex()) setBuffer(kernel, def_k_iag_matrix_out_gr, getGradientIndex()) setArgument(kernel, def_k_iag_dimension, (int)iDimension) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cW.FeedForward()) ReturnFalse; if(!cA.FeedForward()) ReturnFalse; if(!cGL.FeedForward()) ReturnFalse; } if(!InterpolationAttention(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!InterpolationAttentionGrad(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cW.UpdateInputWeights()) ReturnFalse; if(!cA.UpdateInputWeights()) ReturnFalse; if(!cGL.UpdateInputWeights() || !Normilize(cGL.getWeightsParams(), 2 * iDimension)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronInterpolationAttention *Source = source; if(!cW.WeightsUpdate(Source.cW.AsObject(), tau)) ReturnFalse; if(!cA.WeightsUpdate(Source.cA.AsObject(), tau)) ReturnFalse; if(!cGL.WeightsUpdate(Source.cGL.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iCount) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; //--- if(!cW.Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cGL.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInterpolationAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cW.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGL.AsObject())) ReturnFalse; //--- int index = 3; if(!cH.Init(0, index, OpenCL, iCount * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAdj.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, iDimension * iDimension, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInterpolationAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cW.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cGL.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cAdj.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInterpolationAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cW.TrainMode(bTrain); cA.TrainMode(bTrain); cGL.TrainMode(bTrain); cH.TrainMode(bTrain); cAdj.TrainMode(bTrain); cAttention.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAGCN : public CNeuronBaseOCL { protected: CParams cEa; CNeuronSwiGLUOCL cWx; CNeuronSwiGLUOCL cWe; CNeuronBaseOCL cWconcat_ex; CNeuronConvOCL cEn; CNeuronTransposeOCL cEnT; CNeuronBaseOCL cEnEnT; CNeuronSoftMaxOCL cAadapt; CNeuronBaseOCL cAadaptX; CNeuronBaseOCL cApreX; CNeuronSwiGLUOCL cWadapt; CNeuronSwiGLUOCL cWpre; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronAGCN(void) {activation = None;} ~CNeuronAGCN(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronAGCN; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); //--- virtual uint GetWindow(void) const { return cWx.GetWindow(); } virtual uint GetUnits(void) const { return cWx.GetUnits(); } virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * dimension, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cEa.SetActivationFunction(None); index++; if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; index++; if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; index++; if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cWconcat_ex.SetActivationFunction(None); index++; if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; cEn.SetActivationFunction(None); index++; if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(GELU); index++; if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cAadapt.SetHeads(units_count); index++; if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAadaptX.SetActivationFunction(None); index++; if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cApreX.SetActivationFunction(None); index++; if(!cWadapt.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; cWadapt.SetActivationFunction(None); index++; if(!cWpre.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; cWpre.SetActivationFunction(None); SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!SecondInput || SecondInput.Total() < cAadapt.Neurons()) ReturnFalse; //--- if(bTrain) { if(!cEa.FeedForward()) ReturnFalse; if(!cWe.FeedForward(cEa.AsObject())) ReturnFalse; } //--- if(!cWx.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits())) ReturnFalse; if(!cEn.FeedForward(cWconcat_ex.AsObject())) ReturnFalse; if(!cEnT.FeedForward(cEn.AsObject())) ReturnFalse; if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount())) ReturnFalse; if(cEnEnT.Activation() != None) if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation())) ReturnFalse; if(!cAadapt.FeedForward(cEnEnT.AsObject())) ReturnFalse; if(!IdentSum(cAadapt.getOutput(), cAadapt.getOutput(), cAadapt.Heads())) ReturnFalse; if(!MatMul(SecondInput, NeuronOCL.getOutput(), cApreX.getOutput(), cWpre.GetUnits(), cWpre.GetUnits(), cWpre.GetWindow())) ReturnFalse; if(!MatMul(cAadapt.getOutput(), NeuronOCL.getOutput(), cAadaptX.getOutput(), cWadapt.GetUnits(), cWadapt.GetUnits(), cWadapt.GetWindow())) ReturnFalse; if(!cWpre.FeedForward(cApreX.AsObject())) ReturnFalse; if(!cWadapt.FeedForward(cAadaptX.AsObject())) ReturnFalse; if(!SumAndNormilize(cWadapt.getOutput(), cWpre.getOutput(), Output, cWadapt.GetWindowOut(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient || SecondInput.Total() < cAadapt.Neurons() || SecondGradient.Total() < cAadapt.Neurons()) ReturnFalse; //--- if(!DeActivation(cWadapt.getOutput(), cWadapt.getGradient(), Gradient, cWadapt.Activation())) ReturnFalse; if(!DeActivation(cWpre.getOutput(), cWpre.getGradient(), Gradient, cWpre.Activation())) ReturnFalse; if(!cApreX.CalcHiddenGradients(cWpre.AsObject())) ReturnFalse; if(!cAadaptX.CalcHiddenGradients(cWadapt.AsObject())) ReturnFalse; if(!MatMulGrad(SecondInput, SecondGradient, NeuronOCL.getOutput(), NeuronOCL.getGradient(), cApreX.getGradient(), cWpre.GetUnits(), cWpre.GetUnits(), cWpre.GetWindow())) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- if(!MatMulGrad(cAadapt.getOutput(), cAadapt.getGradient(), NeuronOCL.getOutput(), PrevOutput, cAadaptX.getGradient(), cWadapt.GetUnits(), cWadapt.GetUnits(), cWadapt.GetWindow())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, cWx.GetWindow(), false)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, PrevOutput, NeuronOCL.Activation())) ReturnFalse; if(!cEnEnT.CalcHiddenGradients(cAadapt.AsObject())) ReturnFalse; if(cEnEnT.Activation() != None) if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(), cEnEnT.getGradient(), cEnEnT.Activation())) ReturnFalse; if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(), cEnT.getOutput(), cEnT.getGradient(), cEnEnT.getGradient(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount())) ReturnFalse; if(!cEn.CalcHiddenGradients(cEnT.AsObject())) ReturnFalse; if(!SumAndNormilize(cEn.getGradient(), cEn.getPrevOutput(), cEn.getGradient(), cEnT.GetWindow(), false)) ReturnFalse; if(cEn.Activation() != None) if(!DeActivation(cEn.getOutput(), cEn.getGradient(), cEn.getGradient(), cEn.Activation())) ReturnFalse; if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject())) ReturnFalse; if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits())) ReturnFalse; if(cWx.Activation() != None) if(!DeActivation(cWx.getOutput(), cWx.getGradient(), cWx.getGradient(), cWx.Activation())) ReturnFalse; if(cWe.Activation() != None) if(!DeActivation(cWe.getOutput(), cWe.getGradient(), cWe.getGradient(), cWe.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), cWx.GetWindow(), false)) ReturnFalse; if(!cEa.CalcHiddenGradients(cWe.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cEa.UpdateInputWeights()) ReturnFalse; if(!cWe.UpdateInputWeights(cEa.AsObject())) ReturnFalse; if(!cWx.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject())) ReturnFalse; if(!cWpre.UpdateInputWeights(cApreX.AsObject())) ReturnFalse; if(!cWadapt.UpdateInputWeights(cAadaptX.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronAGCN* Source = source; if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau)) ReturnFalse; if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau)) ReturnFalse; if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau)) ReturnFalse; if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau)) ReturnFalse; if(!cWpre.WeightsUpdate(Source.cWpre.AsObject(), tau)) ReturnFalse; if(!cWadapt.WeightsUpdate(Source.cWadapt.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cEa.Save(file_handle)) ReturnFalse; if(!cWe.Save(file_handle)) ReturnFalse; if(!cWx.Save(file_handle)) ReturnFalse; if(!cEn.Save(file_handle)) ReturnFalse; if(!cWpre.Save(file_handle)) ReturnFalse; if(!cWadapt.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAGCN::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEa.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWe.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWx.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWpre.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWadapt.AsObject())) ReturnFalse; //--- uint units_count = GetUnits(); uint dimension = GetWindow(); //--- int index = 3; if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cWconcat_ex.SetActivationFunction(None); index += 2; if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(GELU); index++; if(!cAadapt.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cAadapt.SetHeads(units_count); index++; if(!cAadaptX.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAadaptX.SetActivationFunction(None); index++; if(!cApreX.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cApreX.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAGCN::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cEa.SetOpenCL(OpenCL); cWe.SetOpenCL(OpenCL); cWx.SetOpenCL(OpenCL); cEn.SetOpenCL(OpenCL); cWpre.SetOpenCL(OpenCL); cWadapt.SetOpenCL(OpenCL); cWconcat_ex.SetOpenCL(OpenCL); cEnT.SetOpenCL(OpenCL); cEnEnT.SetOpenCL(OpenCL); cAadapt.SetOpenCL(OpenCL); cAadaptX.SetOpenCL(OpenCL); cApreX.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAGCN::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cEa.TrainMode(bTrain); cWe.TrainMode(bTrain); cWx.TrainMode(bTrain); cEn.TrainMode(bTrain); cWpre.TrainMode(bTrain); cWadapt.TrainMode(bTrain); cWconcat_ex.TrainMode(bTrain); cEnT.TrainMode(bTrain); cEnEnT.TrainMode(bTrain); cAadapt.TrainMode(bTrain); cAadaptX.TrainMode(bTrain); cApreX.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGinARCell : public CNeuronBaseOCL { protected: CNeuronInterpolationAttention cX_IA; CNeuronAGCN cX_AGNC; CNeuronAGCN cForgetGate; CNeuronAGCN cResetGate; CNeuronBaseOCL cContext; CBufferFloat bTemp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronGinARCell(void) { activation = None;} ~CNeuronGinARCell(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronGinARCell; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * dimension, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cX_IA.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cX_AGNC.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cForgetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cResetGate.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cContext.Init(0, index, OpenCL, units_count * dimension, optimization, iBatch)) ReturnFalse; if(!cContext.getPrevOutput().Fill(0)) ReturnFalse; cContext.SetActivationFunction(None); bTemp.BufferFree(); if(!bTemp.BufferInit(units_count * units_count, 0) || !bTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!cX_IA.FeedForward(NeuronOCL)) ReturnFalse; if(!cX_AGNC.FeedForward(cX_IA.AsObject(), SecondInput)) ReturnFalse; if(!cForgetGate.FeedForward(cX_IA.AsObject(), SecondInput)) ReturnFalse; if(!cResetGate.FeedForward(cX_IA.AsObject(), SecondInput)) ReturnFalse; if(!Activation(cForgetGate.getOutput(), cForgetGate.getOutput(), GELU)) ReturnFalse; if(!Activation(cResetGate.getOutput(), cResetGate.getOutput(), GELU)) ReturnFalse; //--- Context if(!cContext.SwapOutputs()) ReturnFalse; if(!GateElementMult(cContext.getPrevOutput(), cX_AGNC.getOutput(), cForgetGate.getOutput(), cContext.getOutput())) ReturnFalse; //--- Output if(!Activation(cContext.getOutput(), cContext.getPrevOutput(), ELU)) ReturnFalse; if(!GateElementMult(cContext.getPrevOutput(), cX_IA.getOutput(), cResetGate.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- Output if(!GateElementMultGrad(cContext.getPrevOutput(), cContext.getGradient(), cX_IA.getOutput(), cX_IA.getPrevOutput(), cResetGate.getOutput(), cResetGate.getGradient(), Gradient, ELU, cX_IA.Activation(), GELU)) ReturnFalse; //--- Context if(!GateElementMultGrad(cContext.getOutput(), cContext.getPrevOutput(), cX_AGNC.getOutput(), cX_AGNC.getGradient(), cForgetGate.getOutput(), cForgetGate.getGradient(), cContext.getGradient(), None, cX_AGNC.Activation(), GELU)) ReturnFalse; //--- Gradient to Interposition Attention if(!cX_IA.CalcHiddenGradients(cX_AGNC.AsObject(), SecondInput, SecondGradient, SecondActivation) || !SumAndNormilize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!cX_IA.CalcHiddenGradients(cForgetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) || !SumAndNormilize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) || !SumAndNormilize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1)) ReturnFalse; if(!cX_IA.CalcHiddenGradients(cResetGate.AsObject(), SecondInput, GetPointer(bTemp), SecondActivation) || !SumAndNormilize(cX_IA.getGradient(), cX_IA.getPrevOutput(), cX_IA.getPrevOutput(), cForgetGate.GetWindow(), false, 0, 0, 0, 1) || !SumAndNormilize(SecondGradient, GetPointer(bTemp), SecondGradient, cForgetGate.GetUnits(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cX_IA.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cX_IA.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cX_AGNC.UpdateInputWeights(cX_IA.AsObject(), second)) ReturnFalse; if(!cForgetGate.UpdateInputWeights(cX_IA.AsObject(), second)) ReturnFalse; if(!cResetGate.UpdateInputWeights(cX_IA.AsObject(), second)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronGinARCell* Source = source; if(!cX_IA.WeightsUpdate(Source.cX_IA.AsObject(), tau)) ReturnFalse; if(!cX_AGNC.WeightsUpdate(Source.cX_AGNC.AsObject(), tau)) ReturnFalse; if(!cForgetGate.WeightsUpdate(Source.cForgetGate.AsObject(), tau)) ReturnFalse; if(!cResetGate.WeightsUpdate(Source.cResetGate.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cX_IA.Save(file_handle)) ReturnFalse; if(!cX_AGNC.Save(file_handle)) ReturnFalse; if(!cForgetGate.Save(file_handle)) ReturnFalse; if(!cResetGate.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cX_IA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cX_AGNC.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cForgetGate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResetGate.AsObject())) ReturnFalse; //--- int index = 4; if(!cContext.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cContext.getPrevOutput().Fill(0)) ReturnFalse; cContext.SetActivationFunction(None); bTemp.BufferFree(); if(!bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0) || !bTemp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGinARCell::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cX_IA.SetOpenCL(OpenCL); cX_AGNC.SetOpenCL(OpenCL); cForgetGate.SetOpenCL(OpenCL); cResetGate.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); bTemp.BufferFree(); bTemp.BufferInit(uint(MathPow(cForgetGate.GetUnits(), 2)), 0); bTemp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGinARCell::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cX_IA.TrainMode(bTrain); cX_AGNC.TrainMode(bTrain); cForgetGate.TrainMode(bTrain); cResetGate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinARCell::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cContext.getPrevOutput() || !cContext.getPrevOutput().Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGinAR : public CNeuronSwiGLUOCL { protected: CParams cEa; CNeuronSwiGLUOCL cWx; CNeuronSwiGLUOCL cWe; CNeuronBaseOCL cWconcat_ex; CNeuronConvOCL cEn; CNeuronTransposeOCL cEnT; CNeuronBaseOCL cEnEnT; CNeuronSoftMaxOCL cApre; CNeuronGinARCell caCells[4]; CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGinAR(void) {}; ~CNeuronGinAR(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronGinAR; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSwiGLUOCL::Init(numOutputs, myIndex, open_cl, caCells.Size()*dimension, caCells.Size()*dimension, dimension, units_count, 1, optimization_type, batch)) ReturnFalse; int index = 0; if(!cEa.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cEa.SetActivationFunction(None); index++; if(!cWx.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; index++; if(!cWe.Init(0, index, OpenCL, dimension, dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cWconcat_ex.SetActivationFunction(None); index++; if(!cEn.Init(0, index, OpenCL, 2 * dimension, 2 * dimension, dimension, units_count, 1, optimization, iBatch)) ReturnFalse; cEn.SetActivationFunction(None); index++; if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(GELU); index++; if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cApre.SetHeads(units_count); for(uint i = 0; i < caCells.Size(); i++) { index++; if(!caCells[i].Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; } index++; if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Calculate Apre if(bTrain) { if(!cEa.FeedForward()) ReturnFalse; if(!cWe.FeedForward(cEa.AsObject())) ReturnFalse; } //--- if(!cWx.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cWx.getOutput(), cWe.getOutput(), cWconcat_ex.getOutput(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits())) ReturnFalse; if(!cEn.FeedForward(cWconcat_ex.AsObject())) ReturnFalse; if(!cEnT.FeedForward(cEn.AsObject())) ReturnFalse; if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount())) ReturnFalse; if(cEnEnT.Activation() != None) if(!Activation(cEnEnT.getOutput(), cEnEnT.getOutput(), cEnEnT.Activation())) ReturnFalse; if(!cApre.FeedForward(cEnEnT.AsObject())) ReturnFalse; if(!IdentSum(cApre.getOutput(), cApre.getOutput(), cApre.Heads())) ReturnFalse; //--- GimAR Cells CNeuronBaseOCL *temp = NeuronOCL; for(uint i = 0; i < caCells.Size(); i++) { if(!caCells[i].FeedForward(temp, cApre.getOutput())) ReturnFalse; temp = caCells[i].AsObject(); } //--- if(!Concat(caCells[0].getOutput(), caCells[1].getOutput(), caCells[2].getOutput(), caCells[3].getOutput(), cConcat.getOutput(), GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetUnits())) ReturnFalse; //--- return CNeuronSwiGLUOCL::feedForward(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSwiGLUOCL::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(caCells[0].getPrevOutput(), caCells[1].getPrevOutput(), caCells[2].getPrevOutput(), caCells[3].getGradient(), cConcat.getGradient(), GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetWindow() / 4, GetUnits())) ReturnFalse; //--- GimAR Cells cApre.getGradient().Fill(0); for(uint i = caCells.Size() - 1; i > 0; i--) if(!caCells[i - 1].CalcHiddenGradients(caCells[i].AsObject(), cApre.getOutput(), cApre.getPrevOutput(), (ENUM_ACTIVATION)cApre.Activation()) || !SumAndNormilize(caCells[i - 1].getGradient(), caCells[i - 1].getPrevOutput(), caCells[i - 1].getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1) || !SumAndNormilize(cApre.getGradient(), cApre.getPrevOutput(), cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(caCells[0].AsObject(), cApre.getOutput(), cApre.getPrevOutput(), (ENUM_ACTIVATION)cApre.Activation()) || !SumAndNormilize(cApre.getGradient(), cApre.getPrevOutput(), cApre.getGradient(), GetUnits(), false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cEnEnT.CalcHiddenGradients(cApre.AsObject())) ReturnFalse; if(cEnEnT.Activation() != None) if(!DeActivation(cEnEnT.getOutput(), cEnEnT.getGradient(), cEnEnT.getGradient(), cEnEnT.Activation())) ReturnFalse; if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(), cEnT.getOutput(), cEnT.getGradient(), cEnEnT.getGradient(), cEnT.GetCount(), cEnT.GetWindow(), cEnT.GetCount())) ReturnFalse; if(!cEn.CalcHiddenGradients(cEnT.AsObject())) ReturnFalse; if(!SumAndNormilize(cEn.getGradient(), cEn.getPrevOutput(), cEn.getGradient(), cEnT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(cEn.Activation() != None) if(!DeActivation(cEn.getOutput(), cEn.getGradient(), cEn.getGradient(), cEn.Activation())) ReturnFalse; if(!cWconcat_ex.CalcHiddenGradients(cEn.AsObject())) ReturnFalse; if(!DeConcat(cWx.getGradient(), cWe.getGradient(), cWconcat_ex.getGradient(), cWx.GetWindowOut(), cWe.GetWindowOut(), cWx.GetUnits())) ReturnFalse; if(cWx.Activation() != None) if(!DeActivation(cWx.getOutput(), cWx.getGradient(), cWx.getGradient(), cWx.Activation())) ReturnFalse; if(cWe.Activation() != None) if(!DeActivation(cWe.getOutput(), cWe.getGradient(), cWe.getGradient(), cWe.Activation())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cWx.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), temp, NeuronOCL.getGradient(), cWx.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; if(!cEa.CalcHiddenGradients(cWe.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Apre if(!cEa.UpdateInputWeights()) ReturnFalse; if(!cWe.UpdateInputWeights(cEa.AsObject())) ReturnFalse; //--- if(!cWx.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cEn.UpdateInputWeights(cWconcat_ex.AsObject())) ReturnFalse; //--- GimAR Cells CNeuronBaseOCL *temp = NeuronOCL; for(uint i = 0; i < caCells.Size(); i++) { if(!caCells[i].UpdateInputWeights(temp, cApre.getOutput())) ReturnFalse; temp = caCells[i].AsObject(); } //--- return CNeuronSwiGLUOCL::updateInputWeights(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSwiGLUOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGinAR* Source = source; //--- Apre if(!cEa.WeightsUpdate(Source.cEa.AsObject(), tau)) ReturnFalse; if(!cWe.WeightsUpdate(Source.cWe.AsObject(), tau)) ReturnFalse; if(!cWx.WeightsUpdate(Source.cWx.AsObject(), tau)) ReturnFalse; if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau)) ReturnFalse; //--- GimAR Cells for(uint i = 0; i < caCells.Size(); i++) if(!caCells[i].WeightsUpdate(Source.caCells[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::Save(const int file_handle) { if(!CNeuronSwiGLUOCL::Save(file_handle)) ReturnFalse; //--- Apre if(!cEa.Save(file_handle)) ReturnFalse; if(!cWe.Save(file_handle)) ReturnFalse; if(!cWx.Save(file_handle)) ReturnFalse; if(!cEn.Save(file_handle)) ReturnFalse; //--- GimAR Cells for(uint i = 0; i < caCells.Size(); i++) if(!caCells[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::Load(const int file_handle) { if(!CNeuronSwiGLUOCL::Load(file_handle)) ReturnFalse; //--- Apre if(!LoadInsideLayer(file_handle, cEa.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWe.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWx.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEn.AsObject())) ReturnFalse; //--- GimAR Cells for(uint i = 0; i < caCells.Size(); i++) if(!LoadInsideLayer(file_handle, caCells[i].AsObject())) ReturnFalse; //--- uint units_count = GetUnits(); uint dimension = cWx.GetWindow(); uint index = 3; if(!cWconcat_ex.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cWconcat_ex.SetActivationFunction(None); index += 2; if(!cEnT.Init(0, index, OpenCL, units_count, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cEnEnT.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(GELU); index++; if(!cApre.Init(0, index, OpenCL, units_count * units_count, optimization, iBatch)) ReturnFalse; cApre.SetHeads(units_count); index += caCells.Size(); if(!cConcat.Init(0, index, OpenCL, GetWindow()*units_count, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGinAR::SetOpenCL(COpenCLMy *obj) { CNeuronSwiGLUOCL::SetOpenCL(obj); //--- Apre cEa.SetOpenCL(OpenCL); cWe.SetOpenCL(OpenCL); cWx.SetOpenCL(OpenCL); cEn.SetOpenCL(OpenCL); //--- GimAR Cells for(uint i = 0; i < caCells.Size(); i++) caCells[i].SetOpenCL(OpenCL); //--- cWconcat_ex.SetOpenCL(OpenCL); cEnT.SetOpenCL(OpenCL); cEnEnT.SetOpenCL(OpenCL); cApre.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGinAR::TrainMode(bool flag) { CNeuronSwiGLUOCL::TrainMode(flag); //--- Apre cEa.TrainMode(bTrain); cWe.TrainMode(bTrain); cWx.TrainMode(bTrain); cEn.TrainMode(bTrain); //--- GimAR Cells for(uint i = 0; i < caCells.Size(); i++) caCells[i].TrainMode(bTrain); //--- cWconcat_ex.TrainMode(bTrain); cEnT.TrainMode(bTrain); cEnEnT.TrainMode(bTrain); cApre.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGinAR::Clear(void) { if(!CNeuronSwiGLUOCL::Clear()) ReturnFalse; for(uint i = 0; i < caCells.Size(); i++) if(!caCells[i].Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPeriodNorm : public CNeuronBaseOCL { protected: uint iPeriod; uint iVariables; uint iCount; CNeuronBaseOCL cMeanSTDevs; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPeriodNorm(void) : iPeriod(-1), iVariables(1) {}; ~CNeuronPeriodNorm(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronPeriodNorm; } virtual void SetOpenCL(COpenCLMy *obj) override; //--- CNeuronBaseOCL* GetMeanSTDevs(void) { return cMeanSTDevs.AsObject(); } virtual uint GetPeriod(void) const { return iPeriod; } virtual uint GetVariables(void) const { return iVariables; } virtual uint GetUnits(void) const { return iCount; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * period * variables, optimization_type, batch)) ReturnFalse; //--- iPeriod = period; iVariables = variables; iCount = units_count; //--- if(!cMeanSTDevs.Init(0, 0, OpenCL, iCount * iVariables * 2, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, iPeriod, iVariables }; uint local_work_size[3] = { 1, iPeriod, 1 }; //--- uint kernel = def_k_PeriodNorm; setBuffer(kernel, def_k_pn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_pn_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_pn_outputs, getOutputIndex()) setArgument(kernel, def_k_pn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / (iVariables * iPeriod), iCount)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, iPeriod, iVariables }; uint local_work_size[3] = { 1, iPeriod, 1 }; //--- uint kernel = def_k_PeriodNormGrad; setBuffer(kernel, def_k_png_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_png_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_png_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_png_mean_stdevs_gr, cMeanSTDevs.getGradientIndex()) setBuffer(kernel, def_k_png_outputs, getOutputIndex()) setBuffer(kernel, def_k_png_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_png_total_inputs, (int)MathMin(NeuronOCL.Neurons() / (iVariables * iPeriod), iCount)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cMeanSTDevs.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPeriod)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPeriodNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cMeanSTDevs.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iPeriod = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPeriodNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cMeanSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAdaptSpatialNorm : public CNeuronBaseOCL { protected: uint iVariables; uint iCount; //--- CParams cEn; CNeuronTransposeOCL cEnT; CNeuronBaseOCL cEnEnT; CNeuronSoftMaxOCL cAttan; CNeuronBaseOCL cMeanSTDevs; //--- virtual bool AdaptSpatialNorm(CNeuronBaseOCL *NeuronOCL); virtual bool AdaptSpatialNormGrad(CNeuronBaseOCL *NeuronOCL); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAdaptSpatialNorm(void) : iCount(0), iVariables(1) {}; ~CNeuronAdaptSpatialNorm(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronAdaptSpatialNorm; } virtual void SetOpenCL(COpenCLMy *obj) override; //--- CNeuronBaseOCL* GetMeanSTDevs(void) { return cMeanSTDevs.AsObject(); } virtual uint GetVariables(void) const { return iVariables; } virtual uint GetUnits(void) const { return iCount; } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * variables, optimization_type, batch)) ReturnFalse; //--- iVariables = variables; iCount = units_count; //--- uint dimension = (iVariables + 1) / 2; uint index = 0; if(!cEn.Init(0, index, OpenCL, iVariables * dimension, optimization, iBatch)) ReturnFalse; cEn.SetActivationFunction(None); index++; if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch)) ReturnFalse; cEnT.SetActivationFunction(None); index++; if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(None); index++; if(!cAttan.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch)) ReturnFalse; cAttan.SetHeads(iVariables); //--- index++; if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cMeanSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::AdaptSpatialNorm(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[3] = { 0 }; uint global_work_size[] = { iCount, MathMin(iVariables, uint(OpenCL.GetMaxLocalSize(1))), iVariables}; uint local_work_size[] = { 1, global_work_size[1], 1}; //--- uint kernel = def_k_AdaptSpatialNorm; setBuffer(kernel, def_k_asn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_asn_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_asn_attention, cAttan.getOutputIndex()) setBuffer(kernel, def_k_asn_outputs, getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::AdaptSpatialNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL) ReturnFalse; uint global_work_offset[3] = { 0 }; uint global_work_size[] = { MathMax(iVariables, iCount), MathMin(MathMax(iVariables, iCount), uint(OpenCL.GetMaxLocalSize(1))), iVariables}; uint local_work_size[] = { 1, global_work_size[1], 1}; //--- uint kernel = def_k_AdaptSpatialNormGrad; setBuffer(kernel, def_k_asng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_asng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_asng_mean_stdevs, cMeanSTDevs.getOutputIndex()) setBuffer(kernel, def_k_asng_mean_stdevs_gr, cMeanSTDevs.getGradientIndex()) setBuffer(kernel, def_k_asng_attention, cAttan.getOutputIndex()) setBuffer(kernel, def_k_asng_attention_gr, cAttan.getGradientIndex()) setBuffer(kernel, def_k_asng_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_asng_total_inputs, iCount) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cEn.FeedForward()) ReturnFalse; if(!cEnT.FeedForward(cEn.AsObject())) ReturnFalse; if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), iVariables, cEnT.GetWindow(), iVariables, 1, false)) ReturnFalse; if(!cAttan.FeedForward(cEnEnT.AsObject())) ReturnFalse; } //--- return AdaptSpatialNorm(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!AdaptSpatialNormGrad(NeuronOCL)) ReturnFalse; if(!cEnEnT.CalcHiddenGradients(cAttan.AsObject())) ReturnFalse; if(!MatMulGrad(cEn.getOutput(), cEn.getPrevOutput(), cEnT.getOutput(), cEnT.getGradient(), cEnEnT.getGradient(), iVariables, cEnT.GetWindow(), iVariables, 1, false)) ReturnFalse; if(!cEn.CalcHiddenGradients(cEnT.AsObject()) || !SumAndNormilize(cEn.getGradient(), cEn.getPrevOutput(), cEn.getGradient(), cEnT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(cEn.Activation() != None) if(!DeActivation(cEn.getOutput(), cEn.getGradient(), cEn.getGradient(), cEn.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return cEn.UpdateInputWeights(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cEn.Save(file_handle)) ReturnFalse; if(!cAttan.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEn.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttan.AsObject())) ReturnFalse; //--- iVariables = cAttan.Heads(); iCount = Neurons() / iVariables; uint dimension = cEn.Neurons() / iVariables; //--- uint index = 1; if(!cEnT.Init(0, index, OpenCL, iVariables, dimension, optimization, iBatch)) ReturnFalse; cEnT.SetActivationFunction(None); index++; if(!cEnEnT.Init(0, index, OpenCL, iVariables * iVariables, optimization, iBatch)) ReturnFalse; cEnEnT.SetActivationFunction(None); index += 2; if(!cMeanSTDevs.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch)) ReturnFalse; cMeanSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAdaptSpatialNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cEn.SetOpenCL(OpenCL); cEnT.SetOpenCL(OpenCL); cEnEnT.SetOpenCL(OpenCL); cAttan.SetOpenCL(OpenCL); cMeanSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAdaptSpatialNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronAdaptSpatialNorm* Source = source; if(!cEn.WeightsUpdate(Source.cEn.AsObject(), tau)) ReturnFalse; //--- if(!cEn.FeedForward()) ReturnFalse; if(!cEnT.FeedForward(cEn.AsObject())) ReturnFalse; if(!MatMul(cEn.getOutput(), cEnT.getOutput(), cEnEnT.getOutput(), iVariables, cEnT.GetWindow(), iVariables, 1, false)) ReturnFalse; if(!cAttan.FeedForward(cEnEnT.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSCNNEncoder : public CNeuronTransposeOCL { protected: CNeuronPeriodNorm cLongNorm; CNeuronTransposeVRCOCL cSeasonTransp; CNeuronPeriodNorm cSeasonNorm; CNeuronTransposeVRCOCL cUnSeasonTransp; CNeuronPeriodNorm cShortNorm; CNeuronAdaptSpatialNorm cAdaptSpatNorm; CNeuronBaseOCL cConcatenated; CNeuronSwiGLUOCL cProjection; CNeuronTransposeOCL cTranspose; CNeuronConvOCL caFusion[2]; CNeuronBaseOCL cFusionOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSCNNEncoder(void) {}; ~CNeuronSCNNEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSCNNEncoder; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast, variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- uint index = 0; if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, variables, optimization, iBatch)) ReturnFalse; cLongNorm.SetActivationFunction(None); index++; if(!cSeasonTransp.Init(0, index, OpenCL, variables, units_count / season_period, season_period, optimization, iBatch)) ReturnFalse; cSeasonTransp.SetActivationFunction(None); index++; if(!cSeasonNorm.Init(0, index, OpenCL, cSeasonTransp.GetCount(), season_period, variables, optimization, iBatch)) ReturnFalse; cSeasonNorm.SetActivationFunction(None); index++; if(!cUnSeasonTransp.Init(0, index, OpenCL, variables, season_period, cSeasonTransp.GetCount(), optimization, iBatch)) ReturnFalse; index++; if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, variables, optimization, iBatch)) ReturnFalse; cSeasonNorm.SetActivationFunction(None); index++; if(!cAdaptSpatNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch)) ReturnFalse; cAdaptSpatNorm.SetActivationFunction(None); index++; uint concatSize = units_count * variables; //inputs concatSize += cLongNorm.Neurons() + cLongNorm.GetMeanSTDevs().Neurons(); // long term concatSize += cSeasonNorm.Neurons() + cSeasonNorm.GetMeanSTDevs().Neurons(); // seasons concatSize += cShortNorm.Neurons() + cShortNorm.GetMeanSTDevs().Neurons(); // short term concatSize += cAdaptSpatNorm.Neurons() + cAdaptSpatNorm.GetMeanSTDevs().Neurons(); // spatial if(!cConcatenated.Init(0, index, OpenCL, concatSize, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); index++; if(!cProjection.Init(0, index, OpenCL, concatSize / variables, concatSize / variables, units_count + forecast, 1, variables, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, variables, units_count + forecast, optimization, iBatch)) ReturnFalse; index++; if(!caFusion[0].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch)) ReturnFalse; caFusion[0].SetActivationFunction(TANH); index++; if(!caFusion[1].Init(0, index, OpenCL, variables, variables, variables, units_count + forecast, optimization, iBatch)) ReturnFalse; caFusion[1].SetActivationFunction(SIGMOID); index++; if(!cFusionOut.Init(0, index, OpenCL, caFusion[0].Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cLongNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cSeasonTransp.FeedForward(cLongNorm.AsObject())) ReturnFalse; if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject())) ReturnFalse; if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject())) ReturnFalse; if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cAdaptSpatNorm.FeedForward(cShortNorm.AsObject())) ReturnFalse; uint windows[3] = {NeuronOCL.Neurons() / iWindow, cLongNorm.GetPeriod()*cLongNorm.GetUnits(), 2 * cLongNorm.GetUnits() }; if(!Concat(NeuronOCL.getOutput(), cLongNorm.getOutput(), cLongNorm.GetMeanSTDevs().getOutput(), cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[0] = windows[0] + windows[1] + windows[2]; windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits(); windows[2] = 2 * cSeasonNorm.GetUnits(); if(!cConcatenated.SwapOutputs() || !Concat(cConcatenated.getPrevOutput(), cSeasonNorm.getOutput(), cSeasonNorm.GetMeanSTDevs().getOutput(), cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[0] = windows[0] + windows[1] + windows[2]; windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits(); windows[2] = 2 * cShortNorm.GetUnits(); if(!cConcatenated.SwapOutputs() || !Concat(cConcatenated.getPrevOutput(), cShortNorm.getOutput(), cShortNorm.GetMeanSTDevs().getOutput(), cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[0] = windows[0] + windows[1] + windows[2]; windows[1] = cAdaptSpatNorm.GetUnits(); windows[2] = 2 * cAdaptSpatNorm.GetUnits(); if(!cConcatenated.SwapOutputs() || !Concat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getOutput(), cAdaptSpatNorm.GetMeanSTDevs().getOutput(), cConcatenated.getOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; if(!cProjection.FeedForward(cConcatenated.AsObject())) ReturnFalse; if(!cTranspose.FeedForward(cProjection.AsObject())) ReturnFalse; for(uint i = 0; i < caFusion.Size(); i++) if(!caFusion[i].FeedForward(cTranspose.AsObject())) ReturnFalse; if(!ElementMult(caFusion[0].getOutput(), caFusion[1].getOutput(), cFusionOut.getOutput())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cFusionOut.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cFusionOut.AsObject())) ReturnFalse; if(!ElementMultGrad(caFusion[0].getOutput(), caFusion[0].getGradient(), caFusion[1].getOutput(), caFusion[1].getGradient(), cFusionOut.getGradient(), caFusion[0].Activation(), caFusion[1].Activation())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(caFusion[0].AsObject())) ReturnFalse; CBufferFloat* temp = cTranspose.getGradient(); if(!cTranspose.SetGradient(cTranspose.getPrevOutput(), false) || !cTranspose.CalcHiddenGradients(caFusion[1].AsObject()) || !SumAndNormilize(temp, cTranspose.getGradient(), temp, cTranspose.GetCount(), false, 0, 0, 0, 1) || !cTranspose.SetGradient(temp, false)) ReturnFalse; if(!cProjection.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; //--- uint windows[3] = {0}; windows[1] = cAdaptSpatNorm.GetUnits(); windows[2] = 2 * cAdaptSpatNorm.GetUnits(); windows[0] = cConcatenated.Neurons() / iWindow - windows[1] - windows[2]; if(!DeConcat(cConcatenated.getPrevOutput(), cAdaptSpatNorm.getGradient(), cAdaptSpatNorm.GetMeanSTDevs().getGradient(), cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[1] = cShortNorm.GetPeriod() * cShortNorm.GetUnits(); windows[2] = 2 * cShortNorm.GetUnits(); windows[0] = windows[0] - windows[1] - windows[2]; if(!DeConcat(cConcatenated.getGradient(), cShortNorm.getPrevOutput(), cShortNorm.GetMeanSTDevs().getGradient(), cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[1] = cSeasonNorm.GetPeriod() * cSeasonNorm.GetUnits(); windows[2] = 2 * cSeasonNorm.GetUnits(); windows[0] = windows[0] - windows[1] - windows[2]; if(!DeConcat(cConcatenated.getPrevOutput(), cSeasonNorm.getPrevOutput(), cSeasonNorm.GetMeanSTDevs().getGradient(), cConcatenated.getGradient(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; windows[1] = cLongNorm.GetPeriod() * cLongNorm.GetUnits(); windows[2] = 2 * cLongNorm.GetUnits(); windows[0] = windows[0] - windows[1] - windows[2]; if(!DeConcat(NeuronOCL.getPrevOutput(), cLongNorm.getPrevOutput(), cLongNorm.GetMeanSTDevs().getGradient(), cConcatenated.getPrevOutput(), windows[0], windows[1], windows[2], iWindow)) ReturnFalse; //--- if(!cShortNorm.CalcHiddenGradients(cAdaptSpatNorm.AsObject()) || !SumAndNormilize(cShortNorm.getGradient(), cShortNorm.getPrevOutput(), cShortNorm.getGradient(), cShortNorm.GetPeriod(), false, 0, 0, 0, 1)) ReturnFalse; if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject())) ReturnFalse; if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject()) || !SumAndNormilize(cSeasonNorm.getGradient(), cSeasonNorm.getPrevOutput(), cSeasonNorm.getGradient(), cSeasonNorm.GetPeriod(), false, 0, 0, 0, 1)) ReturnFalse; if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject())) ReturnFalse; if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject()) || !SumAndNormilize(cLongNorm.getGradient(), cLongNorm.getPrevOutput(), cLongNorm.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), cLongNorm.GetPeriod(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cAdaptSpatNorm.UpdateInputWeights(cShortNorm.AsObject())) ReturnFalse; if(!cProjection.UpdateInputWeights(cConcatenated.AsObject())) ReturnFalse; for(uint i = 0; i < caFusion.Size(); i++) if(!caFusion[i].UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(!cLongNorm.Save(file_handle)) ReturnFalse; if(!cSeasonTransp.Save(file_handle)) ReturnFalse; if(!cSeasonNorm.Save(file_handle)) ReturnFalse; if(!cUnSeasonTransp.Save(file_handle)) ReturnFalse; if(!cShortNorm.Save(file_handle)) ReturnFalse; if(!cAdaptSpatNorm.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; for(uint i = 0; i < caFusion.Size(); i++) if(!caFusion[i].Save(file_handle)) ReturnFalse; if(!cFusionOut.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cLongNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShortNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAdaptSpatNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; for(uint i = 0; i < caFusion.Size(); i++) if(!LoadInsideLayer(file_handle, caFusion[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFusionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSCNNEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); //--- cLongNorm.SetOpenCL(OpenCL); cSeasonTransp.SetOpenCL(OpenCL); cSeasonNorm.SetOpenCL(OpenCL); cUnSeasonTransp.SetOpenCL(OpenCL); cShortNorm.SetOpenCL(OpenCL); cAdaptSpatNorm.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); for(uint i = 0; i < caFusion.Size(); i++) caFusion[i].SetOpenCL(OpenCL); cFusionOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSCNNEncoder::TrainMode(bool flag) { CNeuronTransposeOCL::TrainMode(flag); //--- cLongNorm.TrainMode(bTrain); cSeasonTransp.TrainMode(bTrain); cSeasonNorm.TrainMode(bTrain); cUnSeasonTransp.TrainMode(bTrain); cShortNorm.TrainMode(bTrain); cAdaptSpatNorm.TrainMode(bTrain); cConcatenated.TrainMode(bTrain); cProjection.TrainMode(bTrain); cTranspose.TrainMode(bTrain); for(uint i = 0; i < caFusion.Size(); i++) caFusion[i].TrainMode(bTrain); cFusionOut.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTransposeOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSCNNEncoder* Source = source; if(!cAdaptSpatNorm.WeightsUpdate(Source.cAdaptSpatNorm.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; for(uint i = 0; i < caFusion.Size(); i++) if(!caFusion[i].WeightsUpdate(Source.caFusion[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSCNN : public CNeuronBaseOCL { protected: CLayer cLayers; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSCNN(void) {}; ~CNeuronSCNN(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSCNN; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronSCNNEncoder* encoder = NULL; CNeuronBaseOCL* residual = NULL; for(uint l = 0; l < layers; l++) { encoder = new CNeuronSCNNEncoder(); if(!encoder) ReturnFalse; if(!encoder.Init(0, l, OpenCL, units_count, variables, forecast, season_period, short_period, optimization, iBatch) || !cLayers.Add(encoder)) ReturnFalse; encoder.SetActivationFunction(None); if((l + 1) == layers) break; //--- residual = new CNeuronBaseOCL(); if(!residual) ReturnFalse; if(!residual.Init(0, l, OpenCL, units_count * variables, optimization, iBatch) || !cLayers.Add(residual)) ReturnFalse; residual.SetActivationFunction(None); } if(!SetGradient(encoder.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!Output.Fill(0)) ReturnFalse; CNeuronBaseOCL* inputs = NeuronOCL; CNeuronSCNNEncoder* current = NULL; CNeuronBaseOCL* residual = NULL; CNeuronBaseOCL* temp = NULL; int layers = cLayers.Total(); //--- for(int l = 0; l < layers; l += 2) { current = cLayers[l]; if(!current || !current.FeedForward(inputs) || !SumAndNormilize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1)) ReturnFalse; if((l + 1) == layers) break; uint variables = current.GetWindow(); uint dimension = inputs.Neurons() / variables; uint forecast = current.GetCount() - dimension; residual = cLayers[l + 1]; if(!residual) ReturnFalse; if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(), dimension, forecast, variables) || !SumAndNormilize(residual.getOutput(), inputs.getOutput(), residual.getOutput(), dimension, true, 0, 0, 0, 1)) ReturnFalse; inputs = residual; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!PrevOutput.Fill(0)) ReturnFalse; //--- CNeuronBaseOCL* inputs = NULL; CNeuronSCNNEncoder* current = cLayers[-1]; CNeuronBaseOCL* residual = NULL; int layers = cLayers.Total() - 2; //--- for(int l = layers; l >= 0; l--) switch(cLayers[l].Type()) { case defNeuronBaseOCL: inputs = cLayers[l]; if(!inputs || !inputs.CalcHiddenGradients(current)) ReturnFalse; if(!!residual) if(!SumAndNormilize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; residual = inputs; break; case defNeuronSCNNEncoder: current = cLayers[l]; if(!residual) break; inputs = cLayers[l + 2]; if(!inputs) ReturnFalse; if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(), residual.Neurons() / current.GetWindow(), current.GetCount() - residual.Neurons() / current.GetWindow(), current.GetWindow()) || !SumAndNormilize(current.getGradient(), inputs.getGradient(), current.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; break; default: ReturnFalse; break; } //--- if(!NeuronOCL.CalcHiddenGradients(current)) ReturnFalse; if(!!residual) if(!SumAndNormilize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- for(int l = 0; l < cLayers.Total(); l++) { current = cLayers[l]; if(!current) ReturnFalse; if(current.Type() == defNeuronSCNNEncoder || current.Type() == defNeuronSSCNNEncoder) if(!current.UpdateInputWeights(inputs)) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSCNN* Source = source; if(!cLayers.WeightsUpdate(Source.cLayers.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cLayers.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSCNN::Load(const int file_handle) { cLayers.Clear(); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; cLayers.SetOpenCL(OpenCL); if(!cLayers.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL* last = cLayers[-1]; if(!SetGradient(last.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSCNN::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cLayers.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSCNN::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cLayers.TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAttentNorm : public CNeuronBaseOCL { protected: uint iPeriod; uint iVariables; uint iCount; //--- CParams cAttention; CNeuronSoftMaxOCL cSoftMax; CNeuronBaseOCL cMeans; CNeuronBaseOCL cSTDevs; //--- virtual bool AttentNorm(CNeuronBaseOCL *NeuronOCL); virtual bool AttentNormGrad(CNeuronBaseOCL *NeuronOCL); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAttentNorm(void) : iPeriod(0), iVariables(0), iCount(0) {}; ~CNeuronAttentNorm(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronAttentNorm; } virtual void SetOpenCL(COpenCLMy *obj) override; //--- CNeuronBaseOCL* GetMeans(void) { return cMeans.AsObject(); } CNeuronBaseOCL* GetSTDevs(void) { return cSTDevs.AsObject(); } virtual uint GetPeriod(void) const { return iPeriod; } virtual uint GetVariables(void) const { return iVariables; } virtual uint GetUnits(void) const { return iCount; } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { activation = None; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint period, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * period * variables, optimization_type, batch)) ReturnFalse; CNeuronBaseOCL::SetActivationFunction(None); //--- iCount = units_count; iVariables = variables; iPeriod = period; //--- if(!cAttention.Init(0, 0, OpenCL, iPeriod * iVariables, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); //--- if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iCount, MathMin(iPeriod, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atn_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMin(iCount, iPeriod), MathMin(iPeriod * iCount, uint(OpenCL.GetMaxLocalSize(1))), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iVariables, iCount * iPeriod)) setArgument(kernel, def_k_atng_segment_size, (int)iPeriod) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cAttention.FeedForward()) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; } //--- return AttentNorm(NeuronOCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!AttentNormGrad(NeuronOCL)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return cAttention.UpdateInputWeights(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAttentNorm* Source = source; return cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iCount)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iVariables)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iPeriod)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAttentNorm::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iCount = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iPeriod = (uint)FileReadInteger(file_handle); //--- if(!cSoftMax.Init(0, 1, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iVariables); if(!cMeans.Init(0, 2, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 3, OpenCL, iCount * iVariables, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSAttentNorm : public CNeuronTransposeOCL { protected: CNeuronTransposeOCL cTranspose; CNeuronBaseOCL cAttention; CNeuronSoftMaxOCL cSoftMax; CNeuronBaseOCL cMeans; CNeuronBaseOCL cSTDevs; CNeuronBaseOCL cNorm; //--- virtual bool AttentNorm(CNeuronBaseOCL *NeuronOCL); virtual bool AttentNormGrad(CNeuronBaseOCL *NeuronOCL); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSAttentNorm(void) {}; ~CNeuronSAttentNorm(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSAttentNorm; } virtual void SetOpenCL(COpenCLMy *obj) override; //--- CNeuronBaseOCL* GetMeans(void) { return cMeans.AsObject(); } CNeuronBaseOCL* GetSTDevs(void) { return cSTDevs.AsObject(); } virtual uint GetVariables(void) const { return iWindow; } virtual uint GetUnits(void) const { return iCount; } //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { activation = None; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count, variables, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNorm(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { 1, MathMin(iWindow, uint(OpenCL.GetMaxLocalSize(1))), iCount}; uint local_work_size[3] = { 1, global_work_size[1], 1}; //--- uint kernel = def_k_AttentNorm; setBuffer(kernel, def_k_atn_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atn_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atn_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atn_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atn_outputs, cNorm.getOutputIndex()) setArgument(kernel, def_k_atn_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iWindow, iCount)) setArgument(kernel, def_k_atn_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::AttentNormGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMax(iCount, iWindow), MathMin(MathMax(iWindow, iCount), uint(OpenCL.GetMaxLocalSize(1))), iCount }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; //--- uint kernel = def_k_AttentNormGrad; setBuffer(kernel, def_k_atng_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_atng_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_atng_means, cMeans.getOutputIndex()) setBuffer(kernel, def_k_atng_means_gr, cMeans.getGradientIndex()) setBuffer(kernel, def_k_atng_stdevs, cSTDevs.getOutputIndex()) setBuffer(kernel, def_k_atng_attention, cSoftMax.getOutputIndex()) setBuffer(kernel, def_k_atng_attention_gr, cSoftMax.getGradientIndex()) setBuffer(kernel, def_k_atng_outputs_gr, cNorm.getGradientIndex()) setArgument(kernel, def_k_atng_total_inputs, (int)MathMin(NeuronOCL.Neurons() / iCount, iWindow)) setArgument(kernel, def_k_atng_segment_size, (int)iWindow) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cTranspose.getOutput(), cAttention.getOutput(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!cSoftMax.FeedForward(cAttention.AsObject())) ReturnFalse; if(!AttentNorm(cTranspose.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!AttentNormGrad(cTranspose.AsObject())) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSoftMax.AsObject())) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), PrevOutput, cTranspose.getOutput(), cTranspose.getPrevOutput(), cAttention.getGradient(), iWindow, iCount, iWindow, 1, false)) ReturnFalse; if(!SumAndNormilize(cTranspose.getGradient(), cTranspose.getPrevOutput(), cTranspose.getGradient(), iWindow, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), getPrevOutput(), NeuronOCL.getGradient(), iCount, false, 0, 0, 0, 1)) ReturnFalse; //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAttentNorm::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!cTranspose.Init(0, 0, OpenCL, iWindow, iCount, optimization, iBatch)) ReturnFalse; if(!cAttention.Init(0, 1, OpenCL, iWindow * iWindow, optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); if(!cSoftMax.Init(0, 2, OpenCL, cAttention.Neurons(), optimization, iBatch)) ReturnFalse; cSoftMax.SetHeads(iWindow); if(!cMeans.Init(0, 3, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cMeans.SetActivationFunction(None); if(!cSTDevs.Init(0, 4, OpenCL, iCount, optimization, iBatch)) ReturnFalse; cSTDevs.SetActivationFunction(None); if(!cNorm.Init(0, 5, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSAttentNorm::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); cTranspose.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cSoftMax.SetOpenCL(OpenCL); cMeans.SetOpenCL(OpenCL); cSTDevs.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPolynomialRegression : public CNeuronBaseOCL { protected: CNeuronSwiGLUOCL cProjection; CNeuronConvOCL cConvolution; CNeuronConvOCL cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPolynomialRegression(void) {}; ~CNeuronPolynomialRegression(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint window, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronPolynomialRegression; } virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetWindowIn(void) const { return cProjection.GetWindow(); } virtual uint GetWindowOut(void) const { return cResidual.GetFilters(); } virtual uint GetUnits(void) const { return cProjection.GetUnits(); } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint window, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units_count * window_out, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cProjection.Init(0, 0, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch)) ReturnFalse; if(!cConvolution.Init(0, 1, OpenCL, window_out, window_out, window_out, units_count, 1, optimization, iBatch)) ReturnFalse; cConvolution.SetActivationFunction(None); if(!cResidual.Init(0, 2, OpenCL, window, window, window_out, units_count, 1, optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- if(!SetGradient(cConvolution.getGradient(), true)) ReturnFalse; if(!cResidual.SetGradient(getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cConvolution.FeedForward(cProjection.AsObject())) ReturnFalse; if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(cConvolution.getOutput(), cResidual.getOutput(), Output, GetWindowOut(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cProjection.CalcHiddenGradients(cConvolution.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cProjection.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cResidual.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, GetWindowIn(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConvolution.UpdateInputWeights(cProjection.AsObject())) ReturnFalse; if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronPolynomialRegression* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cConvolution.WeightsUpdate(Source.cConvolution.AsObject(), tau)) ReturnFalse; if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cProjection.Save(file_handle)) ReturnFalse; if(!cConvolution.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPolynomialRegression::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvolution.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPolynomialRegression::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cProjection.SetOpenCL(OpenCL); cConvolution.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSSCNNEncoder : public CNeuronTransposeOCL { protected: CNeuronPeriodNorm cLongNorm; CNeuronConvOCL cLongExtrapolate; CNeuronTransposeOCL cLongMeanSTDevTransp; CNeuronBaseOCL cLongMeanExtrapolate; CNeuronTransposeVRCOCL cSeasonTransp; CNeuronAttentNorm cSeasonNorm; CNeuronTransposeVRCOCL cUnSeasonTransp; CNeuronConvOCL cSeasonExtrapolate; CNeuronConvOCL cSeasonMeanExtrapolate; CNeuronAttentNorm cShortNorm; CNeuronConvOCL cShortExtrapolate; CNeuronConvOCL cShortMeanExtrapolate; CNeuronSAttentNorm cSpatialNorm; CNeuronConvOCL cSpatialExtrapolate; CNeuronConvOCL cSpatialMeanExtrapolate; CNeuronBaseOCL cConcatenated; CNeuronTransposeOCL cTranspose; CNeuronPolynomialRegression cFusion; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSSCNNEncoder(void) {}; ~CNeuronSSCNNEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSSCNNEncoder; } virtual void TrainMode(bool flag) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, units_count + forecast, variables, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cLongNorm.Init(0, index, OpenCL, 1, units_count, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cLongExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cLongExtrapolate.SetActivationFunction(None); index++; if(!cLongMeanSTDevTransp.Init(0, index, OpenCL, iWindow, 2, optimization, iBatch)) ReturnFalse; if(!cLongMeanSTDevTransp.getGradient().Fill(0)) ReturnFalse; index++; if(!cLongMeanExtrapolate.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cLongExtrapolate.SetActivationFunction(None); if(!cLongMeanExtrapolate.getPrevOutput().Fill(1)) ReturnFalse; index++; if(!cSeasonTransp.Init(0, index, OpenCL, iWindow, units_count / season_period, season_period, optimization, iBatch)) ReturnFalse; index++; if(!cSeasonNorm.Init(0, index, OpenCL, season_period, cSeasonTransp.GetCount(), iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cUnSeasonTransp.Init(0, index, OpenCL, iWindow, season_period, cSeasonTransp.GetCount(), optimization, iBatch)) ReturnFalse; index++; if(!cSeasonExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cSeasonExtrapolate.SetActivationFunction(None); index++; if(!cSeasonMeanExtrapolate.Init(0, index, OpenCL, season_period, season_period, iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cSeasonMeanExtrapolate.SetActivationFunction(None); index++; if(!cShortNorm.Init(0, index, OpenCL, units_count / short_period, short_period, iWindow, optimization, iBatch)) ReturnFalse; index++; if(!cShortExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cShortExtrapolate.SetActivationFunction(None); index++; if(!cShortMeanExtrapolate.Init(0, index, OpenCL, cShortNorm.GetUnits(), cShortNorm.GetUnits(), iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cShortMeanExtrapolate.SetActivationFunction(None); index++; if(!cSpatialNorm.Init(0, index, OpenCL, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cSpatialExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, iWindow, 1, optimization, iBatch)) ReturnFalse; cSpatialExtrapolate.SetActivationFunction(None); index++; if(!cSpatialMeanExtrapolate.Init(0, index, OpenCL, units_count, units_count, iCount, 1, iWindow, optimization, iBatch)) ReturnFalse; cSpatialMeanExtrapolate.SetActivationFunction(None); index++; if(!cConcatenated.Init(0, index, OpenCL, 8 * Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, 8 * iWindow, iCount, optimization, iBatch)) ReturnFalse; index++; if(!cFusion.Init(0, index, OpenCL, iCount, 8 * iWindow, iWindow, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Long if(!cLongNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cLongExtrapolate.FeedForward(cLongNorm.AsObject())) ReturnFalse; if(!cLongMeanSTDevTransp.FeedForward(cLongNorm.GetMeanSTDevs())) ReturnFalse; if(!MatMul(cLongMeanSTDevTransp.getOutput(), cLongMeanExtrapolate.getPrevOutput(), cLongMeanExtrapolate.getOutput(), 1, 1, iCount, iWindow, true)) ReturnFalse; //--- Season if(!cSeasonTransp.FeedForward(cLongNorm.AsObject())) ReturnFalse; if(!cSeasonNorm.FeedForward(cSeasonTransp.AsObject())) ReturnFalse; if(!cUnSeasonTransp.FeedForward(cSeasonNorm.AsObject())) ReturnFalse; if(!cSeasonExtrapolate.FeedForward(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cSeasonMeanExtrapolate.FeedForward(cSeasonNorm.GetMeans())) ReturnFalse; //--- Short if(!cShortNorm.FeedForward(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cShortExtrapolate.FeedForward(cShortNorm.AsObject())) ReturnFalse; if(!cShortMeanExtrapolate.FeedForward(cShortNorm.GetMeans())) ReturnFalse; //--- Spatial if(!cSpatialNorm.FeedForward(cShortNorm.AsObject())) ReturnFalse; if(!cSpatialExtrapolate.FeedForward(cSpatialNorm.AsObject())) ReturnFalse; if(!cSpatialMeanExtrapolate.FeedForward(cSpatialNorm.GetMeans())) ReturnFalse; //--- Concat if(!Concat(cLongExtrapolate.getOutput(), cLongMeanExtrapolate.getOutput(), cSeasonExtrapolate.getOutput(), cSeasonMeanExtrapolate.getOutput(), cConcatenated.getOutput(), iCount, iCount, iCount, iCount, iWindow)) ReturnFalse; if(!Concat(cConcatenated.getOutput(), cShortExtrapolate.getOutput(), cShortMeanExtrapolate.getOutput(), cConcatenated.getPrevOutput(), 4 * iCount, iCount, iCount, iWindow)) ReturnFalse; if(!Concat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getOutput(), cSpatialMeanExtrapolate.getOutput(), cConcatenated.getOutput(), 6 * iCount, iCount, iCount, iWindow)) ReturnFalse; //--- Fusion if(!cTranspose.FeedForward(cConcatenated.AsObject())) ReturnFalse; if(!cFusion.FeedForward(cTranspose.AsObject())) ReturnFalse; //--- return CNeuronTransposeOCL::feedForward(cFusion.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Fusion if(!CNeuronTransposeOCL::calcInputGradients(cFusion.AsObject())) ReturnFalse; if(!cTranspose.CalcHiddenGradients(cFusion.AsObject())) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- DeConcat if(!DeConcat(cConcatenated.getPrevOutput(), cSpatialExtrapolate.getGradient(), cSpatialMeanExtrapolate.getGradient(), cConcatenated.getGradient(), 6 * iCount, iCount, iCount, iWindow)) ReturnFalse; if(!DeConcat(cConcatenated.getGradient(), cShortExtrapolate.getGradient(), cShortMeanExtrapolate.getGradient(), cConcatenated.getPrevOutput(), 4 * iCount, iCount, iCount, iWindow)) ReturnFalse; if(!DeConcat(cLongExtrapolate.getGradient(), cLongMeanExtrapolate.getGradient(), cSeasonExtrapolate.getGradient(), cSeasonMeanExtrapolate.getGradient(), cConcatenated.getGradient(), iCount, iCount, iCount, iCount, iWindow)) ReturnFalse; //--- Spatial if(!cSpatialNorm.GetMeans().CalcHiddenGradients(cSpatialMeanExtrapolate.AsObject())) ReturnFalse; if(!cSpatialNorm.CalcHiddenGradients(cSpatialExtrapolate.AsObject())) ReturnFalse; //--- Short if(!cShortNorm.CalcHiddenGradients(cSpatialNorm.AsObject())) ReturnFalse; CBufferFloat* temp = cShortNorm.getGradient(); if(!cShortNorm.SetGradient(cShortNorm.getPrevOutput(), false) || !cShortNorm.CalcHiddenGradients(cShortExtrapolate.AsObject()) || !SumAndNormilize(temp, cShortNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cShortNorm.SetGradient(temp, false)) ReturnFalse; if(!cShortNorm.GetMeans().CalcHiddenGradients(cShortMeanExtrapolate.AsObject())) ReturnFalse; //--- Season if(!cUnSeasonTransp.CalcHiddenGradients(cShortNorm.AsObject())) ReturnFalse; temp = cUnSeasonTransp.getGradient(); if(!cUnSeasonTransp.SetGradient(cUnSeasonTransp.getPrevOutput(), false) || !cUnSeasonTransp.CalcHiddenGradients(cSeasonExtrapolate.AsObject()) || !SumAndNormilize(temp, cUnSeasonTransp.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cUnSeasonTransp.SetGradient(temp, false)) ReturnFalse; if(!cSeasonNorm.GetMeans().CalcHiddenGradients(cSeasonMeanExtrapolate.AsObject())) ReturnFalse; if(!cSeasonNorm.CalcHiddenGradients(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cSeasonTransp.CalcHiddenGradients(cSeasonNorm.AsObject())) ReturnFalse; //--- Long if(!MatMulGrad(cLongMeanSTDevTransp.getOutput(), cLongMeanSTDevTransp.getGradient(), cLongMeanExtrapolate.getPrevOutput(), cLongMeanExtrapolate.getGradient(), cLongMeanExtrapolate.getGradient(), 1, 1, iCount, iWindow, true)) ReturnFalse; if(!cLongNorm.GetMeanSTDevs().CalcHiddenGradients(cLongMeanSTDevTransp.AsObject())) ReturnFalse; if(!cLongNorm.CalcHiddenGradients(cSeasonTransp.AsObject())) ReturnFalse; temp = cLongNorm.getGradient(); if(!cLongNorm.SetGradient(cLongNorm.getPrevOutput(), false) || !cLongNorm.CalcHiddenGradients(cLongExtrapolate.AsObject()) || !SumAndNormilize(temp, cLongNorm.getGradient(), temp, iWindow, false, 0, 0, 0, 1) || !cLongNorm.SetGradient(temp, false)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cLongNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Long if(!cLongNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cLongExtrapolate.UpdateInputWeights(cLongNorm.AsObject())) ReturnFalse; //--- Season if(!cSeasonNorm.UpdateInputWeights(cSeasonTransp.AsObject())) ReturnFalse; if(!cSeasonExtrapolate.UpdateInputWeights(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cSeasonMeanExtrapolate.UpdateInputWeights(cSeasonNorm.GetMeans())) ReturnFalse; //--- Short if(!cShortNorm.UpdateInputWeights(cUnSeasonTransp.AsObject())) ReturnFalse; if(!cShortExtrapolate.UpdateInputWeights(cShortNorm.AsObject())) ReturnFalse; if(!cShortMeanExtrapolate.UpdateInputWeights(cShortNorm.GetMeans())) ReturnFalse; //--- Spatial if(!cSpatialNorm.UpdateInputWeights(cShortNorm.AsObject())) ReturnFalse; if(!cSpatialExtrapolate.UpdateInputWeights(cSpatialNorm.AsObject())) ReturnFalse; if(!cSpatialMeanExtrapolate.UpdateInputWeights(cSpatialNorm.GetMeans())) ReturnFalse; //--- Fusion if(!cFusion.UpdateInputWeights(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTransposeOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSSCNNEncoder* Source = source; //--- Long if(!cLongNorm.WeightsUpdate(Source.cLongNorm.AsObject(), tau)) ReturnFalse; if(!cLongExtrapolate.WeightsUpdate(Source.cLongExtrapolate.AsObject(), tau)) ReturnFalse; //--- Season if(!cSeasonNorm.WeightsUpdate(Source.cSeasonNorm.AsObject(), tau)) ReturnFalse; if(!cSeasonExtrapolate.WeightsUpdate(Source.cSeasonExtrapolate.AsObject(), tau)) ReturnFalse; if(!cSeasonMeanExtrapolate.WeightsUpdate(Source.cSeasonMeanExtrapolate.AsObject(), tau)) ReturnFalse; //--- Short if(!cShortNorm.WeightsUpdate(Source.cShortNorm.AsObject(), tau)) ReturnFalse; if(!cShortExtrapolate.WeightsUpdate(Source.cShortExtrapolate.AsObject(), tau)) ReturnFalse; if(!cShortMeanExtrapolate.WeightsUpdate(Source.cShortMeanExtrapolate.AsObject(), tau)) ReturnFalse; //--- Spatial if(!cSpatialNorm.WeightsUpdate(Source.cSpatialNorm.AsObject(), tau)) ReturnFalse; if(!cSpatialExtrapolate.WeightsUpdate(Source.cSpatialExtrapolate.AsObject(), tau)) ReturnFalse; if(!cSpatialMeanExtrapolate.WeightsUpdate(Source.cSpatialMeanExtrapolate.AsObject(), tau)) ReturnFalse; //--- Fusion if(!cFusion.WeightsUpdate(Source.cFusion.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(!cLongNorm.Save(file_handle)) ReturnFalse; if(!cLongExtrapolate.Save(file_handle)) ReturnFalse; if(!cLongMeanSTDevTransp.Save(file_handle)) ReturnFalse; if(!cLongMeanExtrapolate.Save(file_handle)) ReturnFalse; if(!cSeasonTransp.Save(file_handle)) ReturnFalse; if(!cSeasonNorm.Save(file_handle)) ReturnFalse; if(!cUnSeasonTransp.Save(file_handle)) ReturnFalse; if(!cSeasonExtrapolate.Save(file_handle)) ReturnFalse; if(!cSeasonMeanExtrapolate.Save(file_handle)) ReturnFalse; if(!cShortNorm.Save(file_handle)) ReturnFalse; if(!cShortExtrapolate.Save(file_handle)) ReturnFalse; if(!cShortMeanExtrapolate.Save(file_handle)) ReturnFalse; if(!cSpatialNorm.Save(file_handle)) ReturnFalse; if(!cSpatialExtrapolate.Save(file_handle)) ReturnFalse; if(!cSpatialMeanExtrapolate.Save(file_handle)) ReturnFalse; if(!cConcatenated.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cFusion.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNNEncoder::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cLongNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLongExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLongMeanSTDevTransp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLongMeanExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonTransp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cUnSeasonTransp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSeasonMeanExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShortNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShortExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cShortMeanExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpatialNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpatialExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpatialMeanExtrapolate.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConcatenated.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFusion.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSCNNEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); //--- cLongNorm.SetOpenCL(OpenCL); cLongExtrapolate.SetOpenCL(OpenCL); cLongMeanSTDevTransp.SetOpenCL(OpenCL); cLongMeanExtrapolate.SetOpenCL(OpenCL); cSeasonTransp.SetOpenCL(OpenCL); cSeasonNorm.SetOpenCL(OpenCL); cUnSeasonTransp.SetOpenCL(OpenCL); cSeasonExtrapolate.SetOpenCL(OpenCL); cSeasonMeanExtrapolate.SetOpenCL(OpenCL); cShortNorm.SetOpenCL(OpenCL); cShortExtrapolate.SetOpenCL(OpenCL); cShortMeanExtrapolate.SetOpenCL(OpenCL); cSpatialNorm.SetOpenCL(OpenCL); cSpatialExtrapolate.SetOpenCL(OpenCL); cSpatialMeanExtrapolate.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); cFusion.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSCNNEncoder::TrainMode(bool flag) { CNeuronTransposeOCL::TrainMode(flag); //--- cLongNorm.TrainMode(bTrain); cLongExtrapolate.TrainMode(bTrain); cLongMeanSTDevTransp.TrainMode(bTrain); cLongMeanExtrapolate.TrainMode(bTrain); cSeasonTransp.TrainMode(bTrain); cSeasonNorm.TrainMode(bTrain); cUnSeasonTransp.TrainMode(bTrain); cSeasonExtrapolate.TrainMode(bTrain); cSeasonMeanExtrapolate.TrainMode(bTrain); cShortNorm.TrainMode(bTrain); cShortExtrapolate.TrainMode(bTrain); cShortMeanExtrapolate.TrainMode(bTrain); cSpatialNorm.TrainMode(bTrain); cSpatialExtrapolate.TrainMode(bTrain); cSpatialMeanExtrapolate.TrainMode(bTrain); cConcatenated.TrainMode(bTrain); cTranspose.TrainMode(bTrain); cFusion.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSSCNN : public CNeuronSCNN { protected: virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSSCNN(void) {}; ~CNeuronSSCNN(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronSSCNN; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units_count, uint variables, uint forecast, uint season_period, uint short_period, uint layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (units_count + forecast)*variables, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- cLayers.Clear(); cLayers.SetOpenCL(OpenCL); CNeuronSSCNNEncoder* encoder = NULL; CNeuronBaseOCL* residual = NULL; for(uint l = 0; l < layers; l++) { encoder = new CNeuronSSCNNEncoder(); if(!encoder) ReturnFalse; if(!encoder.Init(0, 2 * l, OpenCL, units_count, variables, forecast, season_period, short_period, optimization, iBatch) || !cLayers.Add(encoder)) ReturnFalse; encoder.SetActivationFunction(None); if((l + 1) == layers) break; //--- residual = new CNeuronBaseOCL(); if(!residual) ReturnFalse; if(!residual.Init(0, 2 * l + 1, OpenCL, units_count * variables, optimization, iBatch) || !cLayers.Add(residual)) ReturnFalse; residual.SetActivationFunction(None); } if(!SetGradient(encoder.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNN::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!Output.Fill(0)) ReturnFalse; CNeuronBaseOCL* inputs = NeuronOCL; CNeuronSSCNNEncoder* current = NULL; CNeuronBaseOCL* residual = NULL; CNeuronBaseOCL* temp = NULL; int layers = cLayers.Total(); //--- for(int l = 0; l < layers; l += 2) { current = cLayers[l]; if(!current || !current.FeedForward(inputs) || !SumAndNormilize(Output, current.getOutput(), Output, current.GetCount(), false, 0, 0, 0, 1)) ReturnFalse; if((l + 1) == layers) break; uint variables = current.GetWindow(); uint dimension = inputs.Neurons() / variables; uint forecast = current.GetCount() - dimension; residual = cLayers[l + 1]; if(!residual) ReturnFalse; if(!DeConcat(residual.getOutput(), current.getPrevOutput(), current.getOutput(), dimension, forecast, variables) || !SumAndNormilize(residual.getOutput(), inputs.getOutput(), residual.getOutput(), dimension, true, 0, 0, 0, 1)) ReturnFalse; inputs = residual; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSCNN::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!PrevOutput.Fill(0)) ReturnFalse; //--- CNeuronBaseOCL* inputs = NULL; CNeuronSSCNNEncoder* current = cLayers[-1]; CNeuronBaseOCL* residual = NULL; int layers = cLayers.Total() - 2; //--- for(int l = layers; l >= 0; l--) switch(cLayers[l].Type()) { case defNeuronBaseOCL: inputs = cLayers[l]; if(!inputs || !inputs.CalcHiddenGradients(current)) ReturnFalse; if(!!residual) if(!SumAndNormilize(inputs.getGradient(), residual.getGradient(), inputs.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; residual = inputs; break; case defNeuronSSCNNEncoder: current = cLayers[l]; if(!residual) break; inputs = cLayers[l + 2]; if(!inputs) ReturnFalse; if(!Concat(residual.getGradient(), PrevOutput, current.getGradient(), residual.Neurons() / current.GetWindow(), current.GetCount() - residual.Neurons() / current.GetWindow(), current.GetWindow()) || !SumAndNormilize(current.getGradient(), inputs.getGradient(), current.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; break; default: ReturnFalse; break; } //--- if(!NeuronOCL.CalcHiddenGradients(current)) ReturnFalse; if(!!residual) if(!SumAndNormilize(NeuronOCL.getGradient(), residual.getGradient(), NeuronOCL.getGradient(), current.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CCircleParams : public CNeuronBaseOCL { protected: CLayer cOnes; CNeuronBaseOCL *cCurrent; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { return FeedForward(); } ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return UpdateInputWeights(); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return true; } public: CCircleParams(void) : cCurrent(NULL) {}; ~CCircleParams(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint period, ENUM_OPTIMIZATION optimization_type, uint batch); virtual bool Identity(const int rows, const int cols); virtual bool Zeros(void); //--- virtual bool FeedForward(void); virtual bool UpdateInputWeights(void); virtual bool SetPosition(int position); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defCircleParams; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual CBufferFloat *getWeightsParams(void); virtual int GetPeriod(void) const { return cOnes.Total(); } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint period, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; //--- cOnes.Clear(); cOnes.SetOpenCL(OpenCL); for(uint i = 0; i < period; i++) { cCurrent = new CNeuronBaseOCL(); if(!cCurrent) ReturnFalse; if(!cCurrent.Init(Neurons(), i, OpenCL, 1, optimization, iBatch) || !cOnes.Add(cCurrent)) { DeleteObj(cCurrent); ReturnFalse; } cCurrent.SetActivationFunction(None); } //--- if(!SetPosition(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::Identity(const int rows, const int cols) { if(rows * cols != Neurons()) ReturnFalse; matrix ident = matrix::Identity(rows, cols); if(!ident.Reshape(rows * cols, 1)) ReturnFalse; if(!ident.Resize(rows * cols, 2)) ReturnFalse; if(!ident.Col(vector::Zeros(rows * cols), 1)) ReturnFalse; //--- CBufferFloat *w = NULL; for(int i = 0; i < cOnes.Total(); i++) { if(!cOnes[i]) ReturnFalse; w = cOnes[i].getWeights(); if(!w || !w.AssignArray(ident)) ReturnFalse; w.BufferFree(); if(!w.BufferCreate(OpenCL)) ReturnFalse; } //--- return FeedForward(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::Zeros(void) { matrix zeros = matrix::Zeros(Neurons(), 2); //--- CBufferFloat *w = NULL; for(int i = 0; i < cOnes.Total(); i++) { if(!cOnes[i]) ReturnFalse; w = cOnes[i].getWeights(); if(!w || !w.AssignArray(zeros)) ReturnFalse; w.BufferFree(); if(!w.BufferCreate(OpenCL)) ReturnFalse; } //--- return FeedForward(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::SetPosition(int position) { cCurrent = cOnes[position]; if(!cCurrent) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::FeedForward(void) { if(!cCurrent) if(!SetPosition(0)) ReturnFalse; //--- return CNeuronBaseOCL::feedForward(cCurrent); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::UpdateInputWeights(void) { return CNeuronBaseOCL::updateInputWeights(cCurrent); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CCircleParams* Source = source; if(!cOnes.WeightsUpdate(Source.cOnes.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- return cOnes.Save(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCircleParams::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- return cOnes.Load(file_handle); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCircleParams::SetOpenCL(COpenCLMy * obj) { CNeuronBaseOCL::SetOpenCL(obj); cOnes.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CCircleParams::getWeightsParams(void) { if(!cCurrent) if(!SetPosition(0)) return NULL; return cCurrent.getWeights(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTQMHA : public CNeuronCrossAttention { protected: uint iTimeframe; CCircleParams cParams; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); public: CNeuronTQMHA(void) {}; ~CNeuronTQMHA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_in, uint period, uint timeframe, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronTQMHA; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint period, uint timeframe, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronCrossAttention::Init(numOutputs, myIndex, open_cl, window, window_key, heads, units_count, window, units_count, optimization_type, batch)) ReturnFalse; //--- FF[0].SetActivationFunction(GELU); //--- iTimeframe = MathMax(1, timeframe); if(!cParams.Init(0, 0, OpenCL, window * units_count, period, optimization, iBatch)) ReturnFalse; if(!cParams.Zeros()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!SecondInput) ReturnFalse; //--- int pos = int(SecondInput[0]); pos = (pos / int(iTimeframe)) % cParams.GetPeriod(); if(!cParams.SetPosition(pos) || !cParams.FeedForward()) ReturnFalse; //--- if(!Q_Embedding.FeedForward(cParams.AsObject())) ReturnFalse; //--- if(!KV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!attentionOut()) ReturnFalse; //--- if(!W0.FeedForward(GetPointer(MHAttentionOut))) ReturnFalse; //--- if(!SumAndNormilize(W0.getOutput(), NeuronOCL.getOutput(), AttentionOut.getOutput(), iWindow)) ReturnFalse; //--- if(!FF[0].FeedForward(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].FeedForward(GetPointer(FF[0]))) ReturnFalse; //--- if(!SumAndNormilize(FF[1].getOutput(), AttentionOut.getOutput(), Output, iWindow)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; //--- if(!FF[0].CalcHiddenGradients(FF[1].AsObject())) ReturnFalse; if(!AttentionOut.CalcHiddenGradients(FF[0].AsObject())) ReturnFalse; if(!SumAndNormilize(FF[1].getGradient(), AttentionOut.getGradient(), W0.getGradient(), iWindow, false)) ReturnFalse; if(!MHAttentionOut.CalcHiddenGradients(W0.AsObject())) ReturnFalse; if(!AttentionInsideGradients()) ReturnFalse; if(!prevLayer.CalcHiddenGradients(KV_Embedding.AsObject())) ReturnFalse; if(!cParams.CalcHiddenGradients(Q_Embedding.AsObject())) ReturnFalse; //--- if(!DeActivation(prevLayer.getOutput(), W0.getPrevOutput(), W0.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), W0.getPrevOutput(), prevLayer.getGradient(), iWindow_K, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!cParams.UpdateInputWeights()) ReturnFalse; if(!Q_Embedding.UpdateInputWeights(cParams.AsObject())) ReturnFalse; if(!KV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!W0.UpdateInputWeights(GetPointer(MHAttentionOut))) ReturnFalse; if(!FF[0].UpdateInputWeights(GetPointer(AttentionOut))) ReturnFalse; if(!FF[1].UpdateInputWeights(GetPointer(FF[0]))) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronCrossAttention::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTQMHA* Source = source; if(!cParams.WeightsUpdate(Source.cParams.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Save(const int file_handle) { if(!CNeuronCrossAttention::Save(file_handle)) ReturnFalse; //--- if(!cParams.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iTimeframe)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTQMHA::Load(const int file_handle) { if(!CNeuronCrossAttention::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iTimeframe = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTQMHA::SetOpenCL(COpenCLMy *obj) { CNeuronCrossAttention::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CChebPolinom : public CNeuronBaseOCL { protected: uint iDimension; uint iSteps; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CChebPolinom(void) {}; ~CChebPolinom(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint steps, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defChebPolinom; } virtual uint GetDimension(void) const { return iDimension; } virtual uint GetSteps(void) const { return iSteps; } virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CChebPolinom::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint steps, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * dimension * steps, optimization_type, batch)) ReturnFalse; //--- activation = None; iDimension = dimension; iSteps = steps; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CChebPolinom::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension)) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))), iDimension, iDimension }; uint local_work_size[3] = { global_work_size[0], 1, 1 }; //--- uint kernel = def_k_ChebStep; setBuffer(kernel, def_k_cheb_support, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_cheb_outputs, getOutputIndex()) for(int step = MathMin(2, MathMax(int(iSteps) - 1, 0)); step < int(iSteps); step++) { setArgument(kernel, def_k_cheb_step, step + 1) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CChebPolinom::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() != (iDimension * iDimension)) ReturnFalse; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { MathMin(iDimension, uint(OpenCL.GetMaxLocalSize(0))), iDimension, iDimension }; uint local_work_size[3] = { global_work_size[0], 1, 1 }; //--- uint kernel = def_k_ChebStepGrad; setBuffer(kernel, def_k_chebgr_support, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_chebgr_support_g, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_chebgr_outputs, getOutputIndex()) setBuffer(kernel, def_k_chebgr_outputs_g, getGradientIndex()) for(int step = (int(iSteps) - 1); step >= MathMax(MathMin(2, int(iSteps) - 1), 1); step--) { setArgument(kernel, def_k_chebgr_step, step) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CChebPolinom::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(iDimension)) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, int(iSteps)) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CChebPolinom::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iSteps = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetGrapConv : public CNeuronTransposeRCDOCL { protected: CNeuronBaseOCL cX_G; public: CNeuronHimNetGrapConv(void) {}; ~CNeuronHimNetGrapConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint cheb_k, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual bool FeedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); //--- virtual bool Load(const int file_handle) override; //--- virtual int Type(void) const { return defNeuronHimNetGrapConv; } virtual void SetOpenCL(COpenCLMy *obj); virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual uint GetCount(void) override const { return iWindow; } virtual uint GetWindow(void) override const { return GetDimension(); } virtual uint GetChebK(void) const { return iCount; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGrapConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint count, uint window, uint cheb_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronTransposeRCDOCL::Init(numOutputs, myIndex, open_cl, cheb_k, count, window, optimization_type, batch)) ReturnFalse; activation = None; if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cX_G.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGrapConv::FeedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(!NeuronOCL || !Support) ReturnFalse; if(Support.GetDimension() != iWindow || Support.GetSteps() != iCount) ReturnFalse; if(NeuronOCL.Neurons() != (Neurons() / iCount)) ReturnFalse; //--- if(!MatMul(Support.getOutput(), NeuronOCL.getOutput(), cX_G.getOutput(), iWindow, iWindow, GetDimension(), iCount, false)) ReturnFalse; //--- return CNeuronTransposeRCDOCL::feedForward(cX_G.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGrapConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(!NeuronOCL || !Support) ReturnFalse; if(Support.GetDimension() != iWindow || Support.GetSteps() != iCount) ReturnFalse; if(NeuronOCL.Neurons() != (Neurons() / iCount)) ReturnFalse; //--- if(!CNeuronTransposeRCDOCL::calcInputGradients(cX_G.AsObject())) ReturnFalse; if(!MatMulGrad(Support.getOutput(), Support.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), cX_G.getGradient(), iWindow, iWindow, GetDimension(), iCount, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGrapConv::Load(const int file_handle) { if(!CNeuronTransposeRCDOCL::Load(file_handle)) ReturnFalse; if(!cX_G.Init(0, 0, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetGrapConv::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeRCDOCL::SetOpenCL(obj); cX_G.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetGCRU : public CNeuronBaseOCL { protected: CNeuronBaseOCL cInpAndHidden; CNeuronHimNetGrapConv cZ_R; CNeuronConvOCL cZ_R_emb; CNeuronBaseOCL cZe_Re; CNeuronBaseOCL cZ; CNeuronBaseOCL cR; CNeuronBaseOCL cCandidate; CNeuronHimNetGrapConv cHC; CNeuronConvOCL cHC_emb; CNeuronBaseOCL cHCe; CBufferFloat bSupportAccum; public: CNeuronHimNetGCRU(void) {}; ~CNeuronHimNetGCRU(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding); //--- virtual int Type(void) const { return defNeuronHimNetGCRU; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; //--- virtual uint GetCount(void) const { return cZ_R.GetCount(); } virtual uint GetWindowIn(void) const { return cZ_R.GetWindow() - GetWindowOut(); } virtual uint GetWindowOut(void) const { return Neurons() / GetCount(); } virtual uint GetChebK(void) const { return cZ_R.GetChebK(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch)) ReturnFalse; cInpAndHidden.SetActivationFunction(None); index++; if(!cZ_R.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch)) ReturnFalse; cZ_R.SetActivationFunction(None); index++; if(!cZ_R_emb.Init(0, index, OpenCL, embed_dim, embed_dim, 2 * cheb_k * (window + window_out) * window_out, units, 1, optimization, iBatch)) ReturnFalse; cZ_R_emb.SetActivationFunction(None); index++; if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch)) ReturnFalse; cZe_Re.SetActivationFunction(None); index++; if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction(None); index++; if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cR.SetActivationFunction(None); index++; if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; cCandidate.SetActivationFunction(None); index++; if(!cHC.Init(0, index, OpenCL, units, window + window_out, cheb_k, optimization, iBatch)) ReturnFalse; cHC.SetActivationFunction(None); index++; if(!cHC_emb.Init(0, index, OpenCL, embed_dim, embed_dim, (window + window_out) * window_out * cheb_k, units, 1, optimization, iBatch)) ReturnFalse; cHC_emb.SetActivationFunction(None); index++; if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cHCe.SetActivationFunction(None); //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; //--- if(!Output.Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding) { if(!NeuronOCL || !Support || !Embedding) ReturnFalse; if(!SwapOutputs()) ReturnFalse; //--- uint cheb_k = cZ_R.GetChebK(); uint units = cZ_R.GetCount(); uint window_out = Neurons() / units; uint window = cZ_R.GetWindow() - window_out; //--- if(!Concat(NeuronOCL.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window_out, units)) ReturnFalse; if(!cZ_R.FeedForward(cInpAndHidden.AsObject(), Support)) ReturnFalse; if(!cZ_R_emb.FeedForward(Embedding)) ReturnFalse; if(!MatMul(cZ_R.getOutput(), cZ_R_emb.getOutput(), cZe_Re.getOutput(), 1, (window + window_out)*cheb_k, 2 * window_out, units, true)) ReturnFalse; if(!Activation(cZe_Re.getOutput(), cZe_Re.getOutput(), SIGMOID)) ReturnFalse; if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZe_Re.getOutput(), window_out, window_out, units)) ReturnFalse; if(!ElementMult(cZ.getOutput(), PrevOutput, cZ.getPrevOutput())) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cZ.getPrevOutput(), cCandidate.getOutput(), window, window_out, units)) ReturnFalse; if(!cHC.FeedForward(cCandidate.AsObject(), Support)) ReturnFalse; if(!cHC_emb.FeedForward(Embedding)) ReturnFalse; if(!MatMul(cHC.getOutput(), cHC_emb.getOutput(), cHCe.getOutput(), 1, (window + window_out)*cheb_k, window_out, units, true)) ReturnFalse; if(!Activation(cHCe.getOutput(), cHCe.getOutput(), TANH)) ReturnFalse; if(!GateElementMult(PrevOutput, cHCe.getOutput(), cR.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding) { if(!NeuronOCL || !Support || !Embedding) ReturnFalse; //--- uint cheb_k = cZ_R.GetChebK(); uint units = cZ_R.GetCount(); uint window_out = Neurons() / units; uint window = cZ_R.GetWindow() - window_out; //--- if(!GateElementMultGrad(PrevOutput, cR.getPrevOutput(), cHCe.getOutput(), cHCe.getGradient(), cR.getOutput(), cR.getGradient(), Gradient, None, TANH, cR.Activation())) ReturnFalse; if(!MatMulGrad(cHC.getOutput(), cHC.getGradient(), cHC_emb.getOutput(), cHC_emb.getGradient(), cHCe.getGradient(), 1, (window + window_out)*cheb_k, window_out, units, true)) ReturnFalse; if(!cHC.calcInputGradients(cCandidate.AsObject(), Support)) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cZ.getPrevOutput(), cCandidate.getGradient(), window, window_out, units)) ReturnFalse; if(!ElementMultGrad(cZ.getOutput(), cZ.getGradient(), PrevOutput, cCandidate.getPrevOutput(), cZ.getPrevOutput(), None, None)) ReturnFalse; if(!Concat(cZ.getGradient(), cR.getGradient(), cZe_Re.getGradient(), window_out, window_out, units)) ReturnFalse; if(!DeActivation(cZe_Re.getOutput(), cZe_Re.getGradient(), cZe_Re.getGradient(), SIGMOID)) ReturnFalse; if(!MatMulGrad(cZ_R.getOutput(), cZ_R.getGradient(), cZ_R_emb.getOutput(), cZ_R_emb.getGradient(), cZe_Re.getGradient(), 1, (window + window_out)*cheb_k, 2 * window_out, units, true)) ReturnFalse; CBufferFloat* temp = Support.getGradient(); if(!Support.SetGradient(GetPointer(bSupportAccum), false) || !cZ_R.calcInputGradients(cInpAndHidden.AsObject(), Support) || !SumAndNormilize(temp, Support.getGradient(), temp, Support.GetDimension(), false, 0, 0, 0, 1) || !Support.SetGradient(temp, false)) ReturnFalse; if(!DeConcat(cInpAndHidden.getPrevOutput(), cCandidate.getPrevOutput(), cInpAndHidden.getGradient(), window, window_out, units)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cInpAndHidden.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- if(!Embedding.CalcHiddenGradients(cHC_emb.AsObject())) ReturnFalse; temp = Embedding.getGradient(); if(!Embedding.SetGradient(Embedding.getPrevOutput(), false) || !Embedding.CalcHiddenGradients(cZ_R_emb.AsObject()) || !SumAndNormilize(temp, Embedding.getGradient(), temp, cZ_R_emb.GetWindow(), false, 0, 0, 0, 1) || !Embedding.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, CNeuronBaseOCL *Embedding) { if(!Embedding) ReturnFalse; //--- if(!cZ_R_emb.UpdateInputWeights(Embedding)) ReturnFalse; if(!cHC_emb.UpdateInputWeights(Embedding)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronHimNetGCRU* Source = source; if(!cZ_R_emb.WeightsUpdate(Source.cZ_R_emb.AsObject(), tau)) ReturnFalse; if(!cHC_emb.WeightsUpdate(Source.cHC_emb.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cZ_R.Save(file_handle)) ReturnFalse; if(!cZ_R_emb.Save(file_handle)) ReturnFalse; if(!cHC.Save(file_handle)) ReturnFalse; if(!cHC_emb.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetGCRU::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cZ_R.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cZ_R_emb.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHC.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHC_emb.AsObject())) ReturnFalse; //--- uint cheb_k = cZ_R.GetChebK(); uint units = cZ_R.GetCount(); uint window_out = Neurons() / units; uint window = cZ_R.GetWindow() - window_out; //--- int index = 0; if(!cInpAndHidden.Init(0, index, OpenCL, (window + window_out)*units, optimization, iBatch)) ReturnFalse; cInpAndHidden.SetActivationFunction(None); index += 3; if(!cZe_Re.Init(0, index, OpenCL, 2 * window_out * units, optimization, iBatch)) ReturnFalse; index++; if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction(None); index++; if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cR.SetActivationFunction(None); index++; if(!cCandidate.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; cCandidate.SetActivationFunction(None); index += 3; if(!cHCe.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; cHCe.SetActivationFunction(None); //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; //--- if(!Output.Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetGCRU::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cZ_R.SetOpenCL(OpenCL); cZ_R_emb.SetOpenCL(OpenCL); cHC.SetOpenCL(OpenCL); cHC_emb.SetOpenCL(OpenCL); cInpAndHidden.SetOpenCL(OpenCL); cZe_Re.SetOpenCL(OpenCL); cZ.SetOpenCL(OpenCL); cR.SetOpenCL(OpenCL); cCandidate.SetOpenCL(OpenCL); cHCe.SetOpenCL(OpenCL); //--- bSupportAccum.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetTempEncoder : public CNeuronBaseOCL { protected: uint aTimeframes[2]; CCircleParams caEmbeddings[2]; CNeuronBaseOCL cConcatEmbeddings; CLayer cGRCUs; CBufferFloat bSupportAccum; public: CNeuronHimNetTempEncoder(void) {}; ~CNeuronHimNetTempEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, uint period1, uint timeframe1, uint period2, uint timeframe2, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, int label); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); //--- virtual int Type(void) const { return defNeuronHimNetTempEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual bool Clear(void) override; //--- virtual uint GetCount(void) const; virtual uint GetWindowIn(void) const; virtual uint GetWindowOut(void) const; virtual uint GetChebK(void) const; //--- virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, uint period1, uint timeframe1, uint period2, uint timeframe2, ENUM_OPTIMIZATION optimization_type, uint batch) { if(layers <= 0) ReturnFalse; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!caEmbeddings[0].Init(0, index, OpenCL, embed_dim, period1, optimization, iBatch)) ReturnFalse; aTimeframes[0] = MathMax(1, timeframe1); index++; if(!caEmbeddings[1].Init(0, index, OpenCL, embed_dim, period2, optimization, iBatch)) ReturnFalse; aTimeframes[1] = MathMax(1, timeframe2); if(!cConcatEmbeddings.Init(numOutputs, myIndex, open_cl, 2 * embed_dim, optimization_type, batch)) ReturnFalse; //--- cGRCUs.Clear(); cGRCUs.SetOpenCL(OpenCL); index++; CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) ReturnFalse; for(uint i = 1; i < layers; i++) { index++; temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) ReturnFalse; } //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support, int label) { for(uint i = 0; i < caEmbeddings.Size(); i++) { int position = (label / int(aTimeframes[i])) % caEmbeddings[i].GetPeriod(); if(!caEmbeddings[i].SetPosition(position) || !caEmbeddings[i].FeedForward()) ReturnFalse; } if(!Concat(caEmbeddings[0].getOutput(), caEmbeddings[1].getOutput(), cConcatEmbeddings.getOutput(), 1, 1, caEmbeddings[0].Neurons())) ReturnFalse; //--- CNeuronBaseOCL *inputs = NeuronOCL; CNeuronHimNetGCRU *current = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current || !current.feedForward(inputs, Support, cConcatEmbeddings.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(!NeuronOCL || !Support) ReturnFalse; //--- CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL); CNeuronHimNetGCRU* current = cGRCUs[-1]; if(!current || !current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject())) ReturnFalse; if(!DeConcat(caEmbeddings[0].getGradient(), caEmbeddings[1].getGradient(), cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons())) ReturnFalse; if(cGRCUs.Total() > 1) { CBufferFloat *temp = Support.getGradient(); if(!Support.SetGradient(GetPointer(bSupportAccum), false)) ReturnFalse; for(int i = cGRCUs.Total() - 2; i >= 0; i--) { current = cGRCUs[i]; inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL); if(!current || !current.calcInputGradients(inputs, Support, cConcatEmbeddings.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, Support.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!DeConcat(caEmbeddings[0].getPrevOutput(), caEmbeddings[1].getPrevOutput(), cConcatEmbeddings.getGradient(), 1, 1, caEmbeddings[0].Neurons())) ReturnFalse; for(uint i = 0; i < caEmbeddings.Size(); i++) if(!SumAndNormilize(caEmbeddings[i].getGradient(), caEmbeddings[i].getPrevOutput(), caEmbeddings[i].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!Support.SetGradient(temp, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].UpdateInputWeights()) ReturnFalse; //--- CNeuronBaseOCL* inputs = NeuronOCL; CNeuronHimNetGCRU* current = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current.updateInputWeights(inputs, Support, cConcatEmbeddings.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronHimNetTempEncoder* Source = source; for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau)) ReturnFalse; if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) { if(!caEmbeddings[i].Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, int(aTimeframes[i])) < INT_VALUE) ReturnFalse; } if(!cGRCUs.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) { if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; aTimeframes[i] = (uint)FileReadInteger(file_handle); } if(!cGRCUs.Load(file_handle)) ReturnFalse; //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); CNeuronHimNetGCRU* temp = cGRCUs[-1]; uint units = temp.GetCount(); uint cheb_k = temp.GetChebK(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetTempEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); for(uint i = 0; i < caEmbeddings.Size(); i++) caEmbeddings[i].SetOpenCL(OpenCL); cGRCUs.SetOpenCL(OpenCL); //--- bSupportAccum.BufferFree(); bSupportAccum.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::Clear(void) { CNeuronBaseOCL* temp = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { temp = cGRCUs[i]; if(!temp || !temp.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetTempEncoder::GetCount(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetCount(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetTempEncoder::GetChebK(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetChebK(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetTempEncoder::GetWindowIn(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetWindowIn(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetTempEncoder::GetWindowOut(void) const { uint count = GetCount(); if(count <= 0) return 0; //--- return Neurons() / count; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetSpatEncoder : public CNeuronBaseOCL { protected: CParams cEmbedding; CLayer cGRCUs; CBufferFloat bSupportAccum; CBufferFloat bEmbeddingAccum; public: CNeuronHimNetSpatEncoder(void) {}; ~CNeuronHimNetSpatEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support); //--- virtual int Type(void) const { return defNeuronHimNetSpatEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual bool Clear(void) override; //--- virtual uint GetCount(void) const; virtual uint GetWindowIn(void) const; virtual uint GetWindowOut(void) const; virtual uint GetChebK(void) const; //--- virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch) { if(layers <= 0) ReturnFalse; //--- if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cEmbedding.Init(0, index, OpenCL, embed_dim, optimization, iBatch)) ReturnFalse; //--- cGRCUs.Clear(); cGRCUs.SetOpenCL(OpenCL); index++; CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) { DeleteObj(temp); ReturnFalse; } for(uint i = 1; i < layers; i++) { index++; temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; bEmbeddingAccum.BufferFree(); bEmbeddingAccum.Clear(); if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) || !bEmbeddingAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(bTrain) if(!cEmbedding.FeedForward()) ReturnFalse; //--- CNeuronBaseOCL *inputs = NeuronOCL; CNeuronHimNetGCRU *current = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current || !current.feedForward(inputs, Support, cEmbedding.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(!NeuronOCL || !Support) ReturnFalse; //--- CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : NeuronOCL); CNeuronHimNetGCRU* current = cGRCUs[-1]; if(!current || !current.calcInputGradients(inputs, Support, cEmbedding.AsObject())) ReturnFalse; if(cGRCUs.Total() > 1) { CBufferFloat *sup = Support.getGradient(); if(!Support.SetGradient(GetPointer(bSupportAccum), false)) ReturnFalse; CBufferFloat *emb = cEmbedding.getGradient(); if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false)) ReturnFalse; for(int i = cGRCUs.Total() - 2; i >= 0; i--) { current = cGRCUs[i]; inputs = (i > 0 ? cGRCUs[i - 1] : NeuronOCL); if(!current || !current.calcInputGradients(inputs, Support, cEmbedding.AsObject())) ReturnFalse; if(!SumAndNormilize(sup, Support.getGradient(), sup, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!Support.SetGradient(sup, false)) ReturnFalse; if(!cEmbedding.SetGradient(emb, false)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CChebPolinom *Support) { if(!cEmbedding.UpdateInputWeights()) ReturnFalse; //--- CNeuronBaseOCL* inputs = NeuronOCL; CNeuronHimNetGCRU* current = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current.updateInputWeights(inputs, Support, cEmbedding.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronHimNetSpatEncoder* Source = source; if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau)) ReturnFalse; if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cEmbedding.Save(file_handle)) ReturnFalse; if(!cGRCUs.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cEmbedding.AsObject())) ReturnFalse; if(!cGRCUs.Load(file_handle)) ReturnFalse; //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); CNeuronHimNetGCRU* temp = cGRCUs[-1]; uint units = temp.GetCount(); uint cheb_k = temp.GetChebK(); if(!bSupportAccum.BufferInit(units * units * cheb_k, 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; bEmbeddingAccum.BufferFree(); bEmbeddingAccum.Clear(); if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) || !bEmbeddingAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetSpatEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cEmbedding.SetOpenCL(OpenCL); cGRCUs.SetOpenCL(OpenCL); //--- bSupportAccum.BufferFree(); bSupportAccum.BufferCreate(OpenCL); bEmbeddingAccum.BufferFree(); bEmbeddingAccum.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::Clear(void) { CNeuronBaseOCL* temp = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { temp = cGRCUs[i]; if(!temp || !temp.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetSpatEncoder::GetCount(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetCount(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetSpatEncoder::GetChebK(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetChebK(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetSpatEncoder::GetWindowIn(void) const { if(cGRCUs.Total() <= 0) return 0; CNeuronHimNetGCRU* temp = cGRCUs[0]; if(!temp) return 0; //--- return temp.GetWindowIn(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ uint CNeuronHimNetSpatEncoder::GetWindowOut(void) const { uint count = GetCount(); if(count <= 0) return 0; //--- return Neurons() / count; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetSpatEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true) { if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev)) ReturnFalse; //--- CNeuronBaseOCL* temp = cGRCUs[-1]; if(!temp || !temp.SetGradient(Gradient, delete_prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetTempEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true) { if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev)) ReturnFalse; //--- CNeuronBaseOCL* temp = cGRCUs[-1]; if(!temp || !temp.SetGradient(Gradient, delete_prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetEncoder : public CNeuronBaseOCL { protected: CParams cEmbedding; CNeuronTransposeOCL cEmbeddingT; CNeuronBaseOCL cSupport; CNeuronSoftMaxOCL cNormSupport; CChebPolinom cPolinomSupport; CNeuronHimNetTempEncoder cTempEncoder; CNeuronHimNetSpatEncoder cSpatEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHimNetEncoder(void) {}; ~CNeuronHimNetEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, uint period1, uint timeframe1, uint period2, uint timeframe2, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHimNetEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual bool Clear(void) override; virtual bool SetGradient(CBufferFloat *buffer, bool delete_prev = true); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, uint period1, uint timeframe1, uint period2, uint timeframe2, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cEmbedding.Init(0, index, OpenCL, units * embed_dim, optimization, iBatch)) ReturnFalse; SetActivationFunction(TANH); index++; if(!cEmbeddingT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch)) ReturnFalse; index++; if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch)) ReturnFalse; cSupport.SetActivationFunction(None); index++; if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch)) ReturnFalse; cNormSupport.SetHeads(units); cNormSupport.SetActivationFunction(None); index++; if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch)) ReturnFalse; index++; if(!cTempEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, (embed_dim + 1) / 2, period1, timeframe1, period2, timeframe2, optimization, iBatch)) ReturnFalse; index++; if(!cSpatEncoder.Init(0, index, OpenCL, units, window, window_out, cheb_k, layers, embed_dim, optimization, iBatch)) ReturnFalse; //--- if(!SetGradient(cTempEncoder.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!SecondInput) ReturnFalse; //--- if(bTrain) { if(!cEmbedding.FeedForward()) ReturnFalse; if(!cEmbeddingT.FeedForward(cEmbedding.AsObject())) ReturnFalse; if(!MatMul(cEmbedding.getOutput(), cEmbeddingT.getOutput(), cSupport.getOutput(), cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false)) ReturnFalse; if(!cNormSupport.FeedForward(cSupport.AsObject())) ReturnFalse; if(!cPolinomSupport.FeedForward(cNormSupport.AsObject())) ReturnFalse; } //--- if(!cTempEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject(), int(SecondInput[0]))) ReturnFalse; if(!cSpatEncoder.feedForward(NeuronOCL, cPolinomSupport.AsObject())) ReturnFalse; if(!SumAndNormilize(cTempEncoder.getOutput(), cSpatEncoder.getOutput(), Output, cTempEncoder.GetWindowOut(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!cTempEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject())) ReturnFalse; CBufferFloat* temp = cPolinomSupport.getGradient(); CBufferFloat* prev = prevLayer.getGradient(); if(!cPolinomSupport.SetGradient(cPolinomSupport.getPrevOutput(), false) || !prevLayer.SetGradient(prevLayer.getPrevOutput(), false) || !cSpatEncoder.calcInputGradients(prevLayer, cPolinomSupport.AsObject()) || !SumAndNormilize(temp, cPolinomSupport.getGradient(), temp, cSpatEncoder.GetCount(), false, 0, 0, 0, 1) || !SumAndNormilize(prev, prevLayer.getGradient(), prev, cSpatEncoder.GetCount(), false, 0, 0, 0, 1) || !cPolinomSupport.SetGradient(temp, false) || !prevLayer.SetGradient(prev, false)) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), prev, prev, prevLayer.Activation())) ReturnFalse; //--- if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject())) ReturnFalse; if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject())) ReturnFalse; if(!MatMulGrad(cEmbedding.getOutput(), cEmbedding.getPrevOutput(), cEmbeddingT.getOutput(), cEmbeddingT.getGradient(), cSupport.getGradient(), cEmbeddingT.GetCount(), cEmbeddingT.GetWindow(), cEmbeddingT.GetCount(), 1, false)) ReturnFalse; if(!cEmbedding.CalcHiddenGradients(cEmbeddingT.AsObject())) ReturnFalse; if(!SumAndNormilize(cEmbedding.getGradient(), cEmbedding.getPrevOutput(), cEmbedding.getGradient(), cEmbeddingT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(cEmbedding.Activation() != None) if(!DeActivation(cEmbedding.getOutput(), cEmbedding.getGradient(), cEmbedding.getGradient(), cEmbedding.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cEmbedding.UpdateInputWeights()) ReturnFalse; if(!cTempEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject())) ReturnFalse; if(!cSpatEncoder.updateInputWeights(NeuronOCL, cPolinomSupport.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronHimNetEncoder* Source = source; if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau)) ReturnFalse; if(!cTempEncoder.WeightsUpdate(Source.cTempEncoder.AsObject(), tau)) ReturnFalse; if(!cSpatEncoder.WeightsUpdate(Source.cSpatEncoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cEmbedding.Save(file_handle)) ReturnFalse; if(!cEmbeddingT.Save(file_handle)) ReturnFalse; if(!cSupport.Save(file_handle)) ReturnFalse; if(!cNormSupport.Save(file_handle)) ReturnFalse; if(!cPolinomSupport.Save(file_handle)) ReturnFalse; if(!cTempEncoder.Save(file_handle)) ReturnFalse; if(!cSpatEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cEmbedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEmbeddingT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSupport.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormSupport.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTempEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpatEncoder.AsObject())) ReturnFalse; //--- if(!SetGradient(cTempEncoder.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cEmbedding.SetOpenCL(OpenCL); cEmbeddingT.SetOpenCL(OpenCL); cSupport.SetOpenCL(OpenCL); cNormSupport.SetOpenCL(OpenCL); cPolinomSupport.SetOpenCL(OpenCL); cTempEncoder.SetOpenCL(OpenCL); cSpatEncoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetEncoder::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cEmbedding.TrainMode(bTrain); cEmbeddingT.TrainMode(bTrain); cSupport.TrainMode(bTrain); cNormSupport.TrainMode(bTrain); cPolinomSupport.TrainMode(bTrain); cTempEncoder.TrainMode(bTrain); cSpatEncoder.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cEmbedding.Clear()) ReturnFalse; if(!cEmbeddingT.Clear()) ReturnFalse; if(!cSupport.Clear()) ReturnFalse; if(!cNormSupport.Clear()) ReturnFalse; if(!cPolinomSupport.Clear()) ReturnFalse; if(!cTempEncoder.Clear()) ReturnFalse; if(!cSpatEncoder.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetEncoder::SetGradient(CBufferFloat *buffer, bool delete_prev = true) { if(!CNeuronBaseOCL::SetGradient(buffer, delete_prev)) ReturnFalse; //--- if(!cTempEncoder.SetGradient(Gradient, delete_prev)) ReturnFalse; if(!cSpatEncoder.SetGradient(Gradient, delete_prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronHimNetDecoder : public CNeuronBaseOCL { protected: CNeuronConvOCL cProjection; CNeuronTransposeOCL cProjectionT; CNeuronConvOCL cEmbedding; CNeuronBaseOCL cSupport; CNeuronSoftMaxOCL cNormSupport; CChebPolinom cPolinomSupport; CLayer cGRCUs; CBufferFloat bSupportAccum; CBufferFloat bEmbeddingAccum; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronHimNetDecoder(void) {}; ~CNeuronHimNetDecoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronHimNetDecoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint cheb_k, uint layers, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cProjection.Init(0, index, OpenCL, window, window, embed_dim, units, 1, optimization, iBatch)) ReturnFalse; SetActivationFunction(TANH); index++; if(!cProjectionT.Init(0, index, OpenCL, units, embed_dim, optimization, iBatch)) ReturnFalse; index++; if(!cEmbedding.Init(0, index, OpenCL, units, units, 1, 1, embed_dim, optimization, iBatch)) ReturnFalse; SetActivationFunction(SIGMOID); index++; if(!cSupport.Init(0, index, OpenCL, units * units, optimization, iBatch)) ReturnFalse; cSupport.SetActivationFunction(None); index++; if(!cNormSupport.Init(0, index, OpenCL, cSupport.Neurons(), optimization, iBatch)) ReturnFalse; cNormSupport.SetHeads(units); cNormSupport.SetActivationFunction(None); index++; if(!cPolinomSupport.Init(0, index, OpenCL, units, cheb_k, optimization, iBatch)) ReturnFalse; //--- cGRCUs.Clear(); cGRCUs.SetOpenCL(OpenCL); index++; CNeuronHimNetGCRU *temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window, window_out, cheb_k, embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) { DeleteObj(temp); ReturnFalse; } for(uint i = 1; i < layers; i++) { index++; temp = new CNeuronHimNetGCRU(); if(!temp || !temp.Init(0, index, OpenCL, units, window_out, window_out, cheb_k, 2 * embed_dim, optimization, iBatch) || !cGRCUs.Add(temp)) { DeleteObj(temp); ReturnFalse; } } //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; bEmbeddingAccum.BufferFree(); bEmbeddingAccum.Clear(); if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) || !bEmbeddingAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.FeedForward(NeuronOCL)) ReturnFalse; if(!cProjectionT.FeedForward(cProjection.AsObject())) ReturnFalse; if(!MatMul(cProjection.getOutput(), cProjectionT.getOutput(), cSupport.getOutput(), cProjectionT.GetCount(), cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false)) ReturnFalse; if(!cNormSupport.FeedForward(cSupport.AsObject())) ReturnFalse; if(!cPolinomSupport.FeedForward(cNormSupport.AsObject())) ReturnFalse; //--- if(!cEmbedding.FeedForward(cProjectionT.AsObject())) ReturnFalse; //--- CNeuronHimNetGCRU* current = NULL; CNeuronBaseOCL* inputs = NeuronOCL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current || !current.feedForward(NeuronOCL, cPolinomSupport.AsObject(), cEmbedding.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- CNeuronBaseOCL *inputs = (cGRCUs.Total() > 1 ? cGRCUs[-2] : prevLayer); CNeuronHimNetGCRU* current = cGRCUs[-1]; if(!current || !current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject())) ReturnFalse; if(cGRCUs.Total() > 1) { CBufferFloat *sup = cPolinomSupport.getGradient(); if(!cPolinomSupport.SetGradient(GetPointer(bSupportAccum), false)) ReturnFalse; CBufferFloat *emb = cEmbedding.getGradient(); if(!cEmbedding.SetGradient(GetPointer(bEmbeddingAccum), false)) ReturnFalse; for(int i = cGRCUs.Total() - 2; i >= 0; i--) { current = cGRCUs[i]; inputs = (i > 0 ? cGRCUs[i - 1] : prevLayer); if(!current || !current.calcInputGradients(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject())) ReturnFalse; if(!SumAndNormilize(cPolinomSupport.getGradient(), sup, sup, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cEmbedding.getGradient(), emb, emb, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!cPolinomSupport.SetGradient(sup, false)) ReturnFalse; if(!cEmbedding.SetGradient(emb, false)) ReturnFalse; } if(!cNormSupport.CalcHiddenGradients(cPolinomSupport.AsObject())) ReturnFalse; if(!cSupport.CalcHiddenGradients(cNormSupport.AsObject())) ReturnFalse; if(!MatMulGrad(cProjection.getOutput(), cProjection.getPrevOutput(), cProjectionT.getOutput(), cProjectionT.getPrevOutput(), cSupport.getGradient(), cProjectionT.GetCount(), cProjectionT.GetWindow(), cProjectionT.GetCount(), 1, false)) ReturnFalse; if(!cProjectionT.CalcHiddenGradients(cEmbedding.AsObject())) ReturnFalse; if(cProjectionT.Activation() != None) if(!DeActivation(cProjectionT.getOutput(), cProjectionT.getPrevOutput(), cProjectionT.getPrevOutput(), cProjectionT.Activation())) ReturnFalse; if(!SumAndNormilize(cProjectionT.getGradient(), cProjectionT.getPrevOutput(), cProjectionT.getGradient(), cProjectionT.GetCount(), false, 0, 0, 0, 1)) ReturnFalse; if(!cProjection.CalcHiddenGradients(cProjectionT.AsObject())) ReturnFalse; if(cProjection.Activation() != None) if(!DeActivation(cProjection.getOutput(), cProjection.getPrevOutput(), cProjection.getPrevOutput(), cProjection.Activation())) ReturnFalse; if(!SumAndNormilize(cProjection.getGradient(), cProjection.getPrevOutput(), cProjection.getGradient(), cProjectionT.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- CBufferFloat* temp = prevLayer.getGradient(); if(!prevLayer.SetGradient(prevLayer.getPrevOutput(), false) || !prevLayer.CalcHiddenGradients(cProjection.AsObject()) || !SumAndNormilize(temp, prevLayer.getGradient(), temp, 1, false, 0, 0, 0, 1) || !prevLayer.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProjection.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cEmbedding.UpdateInputWeights(cProjectionT.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* inputs = NeuronOCL; CNeuronHimNetGCRU* current = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { current = cGRCUs[i]; if(!current.updateInputWeights(inputs, cPolinomSupport.AsObject(), cEmbedding.AsObject())) ReturnFalse; inputs = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronHimNetDecoder* Source = source; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau)) ReturnFalse; if(!cGRCUs.WeightsUpdate(Source.cGRCUs.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cProjection.Save(file_handle)) ReturnFalse; if(!cProjectionT.Save(file_handle)) ReturnFalse; if(!cEmbedding.Save(file_handle)) ReturnFalse; if(!cSupport.Save(file_handle)) ReturnFalse; if(!cNormSupport.Save(file_handle)) ReturnFalse; if(!cPolinomSupport.Save(file_handle)) ReturnFalse; if(!cGRCUs.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cProjection.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEmbedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSupport.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNormSupport.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cPolinomSupport.AsObject())) ReturnFalse; if(!cGRCUs.Load(file_handle)) ReturnFalse; //--- bSupportAccum.BufferFree(); bSupportAccum.Clear(); CNeuronHimNetGCRU* temp = cGRCUs[-1]; if(!bSupportAccum.BufferInit(cPolinomSupport.Neurons(), 0) || !bSupportAccum.BufferCreate(OpenCL)) ReturnFalse; bEmbeddingAccum.BufferFree(); bEmbeddingAccum.Clear(); if(!bEmbeddingAccum.BufferInit(cEmbedding.Neurons(), 0) || !bEmbeddingAccum.BufferCreate(OpenCL)) ReturnFalse; //--- SetActivationFunction((ENUM_ACTIVATION)temp.Activation()); if(!SetOutput(temp.getOutput(), true) || !SetGradient(temp.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronHimNetDecoder::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cProjection.SetOpenCL(OpenCL); cProjectionT.SetOpenCL(OpenCL); cEmbedding.SetOpenCL(OpenCL); cSupport.SetOpenCL(OpenCL); cNormSupport.SetOpenCL(OpenCL); cPolinomSupport.SetOpenCL(OpenCL); cGRCUs.SetOpenCL(OpenCL); //--- bSupportAccum.BufferFree(); bSupportAccum.BufferCreate(OpenCL); bEmbeddingAccum.BufferFree(); bEmbeddingAccum.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronHimNetDecoder::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cProjection.Clear()) ReturnFalse; if(!cProjectionT.Clear()) ReturnFalse; if(!cEmbedding.Clear()) ReturnFalse; if(!cSupport.Clear()) ReturnFalse; if(!cNormSupport.Clear()) ReturnFalse; if(!cPolinomSupport.Clear()) ReturnFalse; CNeuronBaseOCL* temp = NULL; for(int i = 0; i < cGRCUs.Total(); i++) { temp = cGRCUs[i]; if(!temp || !temp.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSNSMHAttention : public CNeuronConvOCL { float fSparse; //--- CNeuronBaseOCL cNeighbors; CNeuronBaseOCL cRamdomCandidates; CNeuronConvOCL cProjection[2]; CNeuronBaseOCL cScores; //--- virtual bool SignificantNeighborsSampling(CNeuronBaseOCL *NeuronOCL); virtual bool SparseMHScores(void); virtual bool SparseMHScoresGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSNSMHAttention(void) {}; ~CNeuronSNSMHAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSNSMHAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual CBufferFloat* GetIndexes(void) { return cNeighbors.getOutput(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!sparse >= 1 || sparse < 0) ReturnFalse; fSparse = sparse; //--- if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, heads, heads, 1, units * m_units, 1, optimization_type, batch)) ReturnFalse; int index = 0; if(!cNeighbors.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; CBufferFloat* temp = cNeighbors.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cRamdomCandidates.Init(0, index, OpenCL, units * m_units, optimization, iBatch)) ReturnFalse; temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, (float)(units - 1))) ReturnFalse; index++; if(!cProjection[0].Init(index, 0, OpenCL, window, window, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(SoftPlus); index++; if(!cProjection[1].Init(index, 0, OpenCL, 2 * heads, 2 * heads, 2 * heads, units, 1, optimization, iBatch)) ReturnFalse; cProjection[0].SetActivationFunction(TANH); index++; if(!cScores.Init(0, index, OpenCL, units * m_units * heads, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SignificantNeighborsSampling(CNeuronBaseOCL *NeuronOCL) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint window = cProjection[0].GetWindow(); //--- if(!NeuronOCL || NeuronOCL.Neurons() != (units * window)) ReturnFalse; if(!cNeighbors.SwapOutputs()) ReturnFalse; CBufferFloat *temp = cRamdomCandidates.getOutput(); if(!temp || !temp.Random(0, float(units))) ReturnFalse; //--- uint global_work_offset[2] = { 0 }; uint global_work_size[2] = { units, m_units }; uint local_work_size[2] = { 1, m_units }; //--- uint kernel = def_k_SignificantNeighborsSampling; setBuffer(kernel, def_k_sns_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_sns_random_cands, cRamdomCandidates.getOutputIndex()) setBuffer(kernel, def_k_sns_candidates, cNeighbors.getPrevOutIndex()) setBuffer(kernel, def_k_sns_neighbors, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_sns_dimension, window) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScores(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScores; setBuffer(kernel, def_k_smhs_data, cProjection[1].getOutputIndex()) setBuffer(kernel, def_k_smhs_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_indexes, cNeighbors.getOutputIndex()) setArgument(kernel, def_k_smhs_sparse, fSparse) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::SparseMHScoresGrad(void) { uint units = cProjection[0].GetUnits(); uint m_units = GetUnits() / units; uint heads = iWindow; //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, m_units, heads }; uint local_work_size[3] = { 1, m_units, 1 }; //--- uint kernel = def_k_SparseMHScoresGrad; setBuffer(kernel, def_k_smhs_gr_data_gr, cProjection[1].getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_scores, cScores.getOutputIndex()) setBuffer(kernel, def_k_smhs_gr_scores_gr, cScores.getGradientIndex()) setBuffer(kernel, def_k_smhs_gr_indexes, cNeighbors.getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!SignificantNeighborsSampling(NeuronOCL)) ReturnFalse; CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].FeedForward(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!SparseMHScores()) ReturnFalse; if(!CNeuronConvOCL::feedForward(cScores.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cScores.AsObject())) ReturnFalse; if(!SparseMHScoresGrad()) ReturnFalse; //--- int total = (int)cProjection.Size(); CNeuronBaseOCL* inputs = NULL; for(int i = total - 1; i >= 0; i--) { inputs = (i > 0 ? cProjection[i - 1].AsObject() : prevLayer); if(!inputs.CalcHiddenGradients(cProjection[i].AsObject())) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* inputs = NeuronOCL; for(uint i = 0; i < cProjection.Size(); i++) { if(!cProjection[i].UpdateInputWeights(inputs)) ReturnFalse; inputs = cProjection[i].AsObject(); } if(!CNeuronConvOCL::updateInputWeights(cScores.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSNSMHAttention* Source = source; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].WeightsUpdate(Source.cProjection[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteFloat(file_handle, fSparse) < sizeof(float)) ReturnFalse; //--- if(!cNeighbors.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!cProjection[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSNSMHAttention::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; fSparse = FileReadFloat(file_handle); //--- if(!LoadInsideLayer(file_handle, cNeighbors.AsObject())) ReturnFalse; for(uint i = 0; i < cProjection.Size(); i++) if(!LoadInsideLayer(file_handle, cProjection[i].AsObject())) ReturnFalse; //--- int index = 1; if(!cRamdomCandidates.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; index += int(cProjection.Size()); if(!cScores.Init(0, index, OpenCL, Neurons() * iWindow, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSNSMHAttention::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cNeighbors.SetOpenCL(OpenCL); cRamdomCandidates.SetOpenCL(OpenCL); for(uint i = 0; i < cProjection.Size(); i++) cProjection[i].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFastGConv : public CNeuronBaseOCL { protected: CNeuronBaseOCL cInpAndHidden; CNeuronBaseOCL cNormAttention; CNeuronBaseOCL cInvDiag; CNeuronBaseOCL cAX; CNeuronBaseOCL cAXplusX; CNeuronBaseOCL cNormAXplusX; CNeuronConvOCL cZ_R; CNeuronBaseOCL cZ; CNeuronBaseOCL cR; CNeuronBaseOCL cCandidate; CNeuronConvOCL cHC; //--- virtual bool RandomWalk(CBufferFloat* data, CBufferFloat* normal, CBufferFloat* inv_diag, const int rows, const int cols ); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { ReturnFalse; }; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None ) override { ReturnFalse; } //--- virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronFastGConv(void) {}; ~CNeuronFastGConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool FeedForward(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent); virtual bool CalcInputGradients(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent); //--- virtual int Type(void) const { return defNeuronFastGConv; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; //--- virtual uint GetCount(void) const { return (uint)cInvDiag.Neurons(); } virtual uint GetSparseDimension(void) const { return (uint)cNormAttention.Neurons() / GetCount(); } virtual uint GetWindow(void) const { return (uint)Neurons() / GetCount(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cInpAndHidden.Init(0, index, OpenCL, 2 * units * window, optimization, iBatch)) ReturnFalse; index++; if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch)) ReturnFalse; index++; if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cZ_R.Init(0, index, OpenCL, 2 * window, 2 * window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cZ_R.SetActivationFunction(SIGMOID); index++; if(!cZ.Init(0, index, OpenCL, window * units, optimization, iBatch)) ReturnFalse; index++; if(!cR.Init(0, index, OpenCL, window * units, optimization, iBatch)) ReturnFalse; index++; if(!cCandidate.Init(0, index, OpenCL, 2 * window * units, optimization, iBatch)) ReturnFalse; index++; if(!cHC.Init(0, index, OpenCL, 2 * window, 2 * window, window, units, 1, optimization, iBatch)) ReturnFalse; cHC.SetActivationFunction(TANH); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::RandomWalk(CBufferFloat *data, CBufferFloat *normal, CBufferFloat *inv_diag, const int rows, const int cols) { if(!OpenCL) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {(uint)rows, MathMin((uint)cols, (uint)OpenCL.GetMaxLocalSize(1))}; uint local_work_size[2] = {1, global_work_size[1]}; uint kernel = def_k_RandomWalk; setBuffer(kernel, def_k_rw_data, data.GetIndex()) setBuffer(kernel, def_k_rw_norm, normal.GetIndex()) setBuffer(kernel, def_k_rw_inv_diag, inv_diag.GetIndex()) setArgument(kernel, def_k_rw_total_cols, cols) //--- kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::FeedForward(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent) { if(!SourceData || !SparseAttent) ReturnFalse; //--- const uint units = GetCount(); const uint window = GetWindow(); const uint sparse = GetSparseDimension(); //--- if(!RandomWalk(SparseAttent.getOutput(), cNormAttention.getOutput(), cInvDiag.getOutput(), window, sparse)) ReturnFalse; if(!SwapOutputs()) ReturnFalse; if(!Concat(SourceData.getOutput(), PrevOutput, cInpAndHidden.getOutput(), window, window, units)) ReturnFalse; //--- if(!SparseMatMul(SparseAttent.GetIndexes(), cNormAttention.getOutput(), cInpAndHidden.getOutput(), cAX.getOutput(), units, sparse, units, 2 * window)) ReturnFalse; if(!SumAndNormilize(cAX.getOutput(), cInpAndHidden.getOutput(), cAXplusX.getOutput(), 2 * window, false, 0, 0, 0, 1)) ReturnFalse; if(!DiagMatMul(cInvDiag.getOutput(), cAXplusX.getOutput(), cNormAXplusX.getOutput(), units, 2 * window, 1, None)) ReturnFalse; if(!cZ_R.FeedForward(cNormAXplusX.AsObject())) ReturnFalse; if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZ_R.getOutput(), window, window, units)) ReturnFalse; if(!ElementMult(cR.getOutput(), PrevOutput, cR.getPrevOutput())) ReturnFalse; if(!Concat(SourceData.getOutput(), cR.getPrevOutput(), cCandidate.getOutput(), window, window, units)) ReturnFalse; if(!cHC.FeedForward(cCandidate.AsObject())) ReturnFalse; if(!GateElementMult(PrevOutput, cHC.getOutput(), cZ.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::CalcInputGradients(CNeuronBaseOCL *SourceData, CNeuronSNSMHAttention *SparseAttent) { if(!SourceData || !SparseAttent) ReturnFalse; //--- const uint units = GetCount(); const uint window = GetWindow(); const uint sparse = GetSparseDimension(); //--- if(!GateElementMultGrad(PrevOutput, cInpAndHidden.getGradient(), cHC.getOutput(), cHC.getGradient(), cZ.getOutput(), cZ.getGradient(), Gradient, None, cHC.Activation(), cZ_R.Activation())) ReturnFalse; if(!cCandidate.CalcHiddenGradients(cHC.AsObject())) ReturnFalse; if(!DeConcat(SourceData.getGradient(), cR.getPrevOutput(), cCandidate.getGradient(), window, window, units)) ReturnFalse; if(!ElementMultGrad(cR.getOutput(), cR.getGradient(), PrevOutput, cInpAndHidden.getGradient(), cR.getPrevOutput(), cZ_R.Activation(), None)) ReturnFalse; if(!Concat(cZ.getGradient(), cR.getGradient(), cZ_R.getGradient(), window, window, units)) ReturnFalse; if(!cNormAXplusX.CalcHiddenGradients(cZ_R.AsObject())) ReturnFalse; if(!DiagMatMulGrad(cInvDiag.getOutput(), cInvDiag.getGradient(), cAXplusX.getOutput(), cAX.getGradient(), cNormAXplusX.getGradient(), units, 2 * window, 1)) ReturnFalse; if(!SparseMatMulGrad(SparseAttent.GetIndexes(), cNormAttention.getOutput(), cNormAttention.getGradient(), cInpAndHidden.getOutput(), cInpAndHidden.getGradient(), cAX.getGradient(), units, sparse, units, 2 * window)) ReturnFalse; if(!SumAndNormilize(cAX.getGradient(), cInpAndHidden.getGradient(), cInpAndHidden.getGradient(), 2 * window, false, 0, 0, 0, 1)) ReturnFalse; if(!DeConcat(cAXplusX.getGradient(), cAXplusX.getPrevOutput(), cInpAndHidden.getGradient(), window, window, units)) ReturnFalse; if(!SumAndNormilize(SourceData.getGradient(), cAXplusX.getGradient(), SourceData.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; if(SourceData.Activation() != None) if(!DeActivation(SourceData.getOutput(), SourceData.getGradient(), SourceData.getGradient(), SourceData.Activation())) ReturnFalse; if(!DiagMatMul(cInvDiag.getOutput(), cNormAttention.getGradient(), SparseAttent.getGradient(), units, sparse, 1, None)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- if(!cZ_R.UpdateInputWeights(cNormAXplusX.AsObject())) ReturnFalse; if(!cHC.UpdateInputWeights(cCandidate.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFastGConv *Source = source; if(!cZ_R.WeightsUpdate(Source.cZ_R.AsObject(), tau)) ReturnFalse; if(!cHC.WeightsUpdate(Source.cHC.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cZ_R.Save(file_handle)) ReturnFalse; if(!cHC.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)GetSparseDimension()) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFastGConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cZ_R.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHC.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; uint sparse_dimension = FileReadInteger(file_handle); uint units = cZ_R.GetUnits(); uint window_out = Neurons() / units; uint window = cZ_R.GetWindow() - window_out; //--- int index = 0; if(!cInpAndHidden.Init(0, index, OpenCL, units * (window + window_out), optimization, iBatch)) ReturnFalse; index++; if(!cNormAttention.Init(0, index, OpenCL, units * sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cInvDiag.Init(0, index, OpenCL, units, optimization, iBatch)) ReturnFalse; index++; if(!cAX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cNormAXplusX.Init(0, index, OpenCL, cInpAndHidden.Neurons(), optimization, iBatch)) ReturnFalse; index += 2; if(!cZ.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; index++; if(!cR.Init(0, index, OpenCL, window_out * units, optimization, iBatch)) ReturnFalse; index++; if(!cCandidate.Init(0, index, OpenCL, (window + window_out) * units, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFastGConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cZ_R.SetOpenCL(OpenCL); cHC.SetOpenCL(OpenCL); cInpAndHidden.SetOpenCL(OpenCL); cNormAttention.SetOpenCL(OpenCL); cInvDiag.SetOpenCL(OpenCL); cAX.SetOpenCL(OpenCL); cAXplusX.SetOpenCL(OpenCL); cNormAXplusX.SetOpenCL(OpenCL); cZ.SetOpenCL(OpenCL); cR.SetOpenCL(OpenCL); cCandidate.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSAGDFN : public CNeuronTransposeOCL { protected: CNeuronTransposeOCL cTranspose; CLayer cEmbedding; CNeuronSNSMHAttention cAttention; CLayer cGCRU; CLayer cProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSAGDFN(void) {}; ~CNeuronSAGDFN(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps, uint variables, uint embedding_dim, uint emb_layers, uint sparse_dimension, uint heads, float sparse, uint gcru_layers, uint forecast, uint forec_layers, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSAGDFN; } //--- methods for working with files virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps, uint variables, uint embedding_dim, uint emb_layers, uint sparse_dimension, uint heads, float sparse, uint gcru_layers, uint forecast, uint forec_layers, ENUM_OPTIMIZATION optimization_type, uint batch) { if(emb_layers <= 0 || gcru_layers <= 0 || forec_layers <= 0) ReturnFalse; //--- if(!CNeuronTransposeOCL::Init(numOutputs, myIndex, open_cl, variables, forecast, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cTranspose.Init(0, index, OpenCL, time_steps, variables, optimization, iBatch)) ReturnFalse; //--- Embedding cEmbedding.Clear(); cEmbedding.SetOpenCL(OpenCL); index++; CNeuronConvOCL *conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps, time_steps, embedding_dim, variables, 1, optimization, iBatch) || !cEmbedding.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); for(uint i = 1; i < emb_layers; i++) { index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, embedding_dim, embedding_dim, embedding_dim, variables, 1, optimization, iBatch) || !cEmbedding.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); } CNeuronBatchNormOCL *norm = new CNeuronBatchNormOCL(); index++; if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cEmbedding.Add(norm)) { DeleteObj(norm); ReturnFalse; } norm.SetActivationFunction(None); //--- GCRUs cGCRU.Clear(); cGCRU.SetOpenCL(OpenCL); index++; if(!cAttention.Init(0, index, OpenCL, variables, embedding_dim, heads, sparse_dimension, sparse, optimization, iBatch)) ReturnFalse; CNeuronFastGConv *gcru = NULL; for(uint i = 0; i < gcru_layers; i++) { index++; gcru = new CNeuronFastGConv(); if(!gcru || !gcru.Init(0, index, OpenCL, variables, embedding_dim, sparse_dimension, optimization, iBatch) || !cGCRU.Add(gcru)) { DeleteObj(gcru); ReturnFalse; } } //--- Forecast cProjection.Clear(); cProjection.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, embedding_dim, embedding_dim, forecast, variables, 1, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); for(uint i = 1; i < forec_layers; i++) { index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, forecast, forecast, forecast, variables, 1, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); } norm = new CNeuronBatchNormOCL(); index++; if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cProjection.Add(norm)) { DeleteObj(norm); ReturnFalse; } norm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cTranspose.FeedForward(NeuronOCL)) ReturnFalse; CNeuronBaseOCL *inputs = cTranspose.AsObject(); CNeuronBaseOCL *current = NULL; //--- Embedding for(int i = 0; i < cEmbedding.Total(); i++) { current = cEmbedding[i]; if(!current || !current.FeedForward(inputs)) ReturnFalse; inputs = current; } //--- GCRUs if(!cAttention.FeedForward(inputs)) ReturnFalse; CNeuronFastGConv *gcru = NULL; for(int i = 0; i < cGCRU.Total(); i++) { gcru = cGCRU[i]; if(!gcru || !gcru.FeedForward(inputs, cAttention.AsObject())) ReturnFalse; inputs = gcru; } //--- Forecast for(int i = 0; i < cProjection.Total(); i++) { current = cProjection[i]; if(!current || !current.FeedForward(inputs)) ReturnFalse; inputs = current; } //--- result return CNeuronTransposeOCL::feedForward(inputs); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronTransposeOCL::calcInputGradients(cProjection[-1])) ReturnFalse; //--- Forecast CNeuronBaseOCL *inputs = NULL; for(int i = cProjection.Total() - 1; i >= 0; i--) { inputs = (i > 0 ? cProjection[i - 1] : cGCRU[-1]); if(!inputs || !inputs.CalcHiddenGradients(cProjection[i])) ReturnFalse; } //--- GCRU CNeuronFastGConv *gcru = cGCRU[-1]; int layers = cGCRU.Total(); inputs = (layers > 1 ? cGCRU[-2] : cEmbedding[-1]); if(!gcru || !gcru.CalcInputGradients(inputs, cAttention.AsObject())) ReturnFalse; if(layers > 1) { CBufferFloat *temp = cAttention.getGradient(); if(!cAttention.SetGradient(cAttention.getPrevOutput(), false)) ReturnFalse; for(int i = layers - 2; i >= 0; i--) { inputs = (i > 0 ? cGCRU[i - 1] : cEmbedding[-1]); gcru = cGCRU[i]; if(!gcru || !gcru.CalcInputGradients(inputs, cAttention.AsObject()) || !SumAndNormilize(temp, cAttention.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; } if(!cAttention.SetGradient(temp, false)) ReturnFalse; } CBufferFloat *temp = inputs.getGradient(); if(!inputs.SetGradient(inputs.getPrevOutput(), false) || !inputs.CalcHiddenGradients(cAttention.AsObject()) || !SumAndNormilize(temp, inputs.getGradient(), temp, 1, false, 0, 0, 0, 1) || !inputs.SetGradient(temp, false)) ReturnFalse; //--- Embedding for(int i = cEmbedding.Total() - 2; i >= 0; i--) { inputs = cEmbedding[i]; if(!inputs || !inputs.CalcHiddenGradients(cEmbedding[i + 1])) ReturnFalse; } //--- if(!cTranspose.CalcHiddenGradients(inputs)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *inputs = cTranspose.AsObject(); CNeuronBaseOCL *current = NULL; //--- Embedding for(int i = 0; i < cEmbedding.Total(); i++) { current = cEmbedding[i]; if(!current || !current.UpdateInputWeights(inputs)) ReturnFalse; inputs = current; } //--- GCRUs if(!cAttention.UpdateInputWeights(inputs)) ReturnFalse; for(int i = 0; i < cGCRU.Total(); i++) { current = cGCRU[i]; if(!current || !current.UpdateInputWeights(inputs)) ReturnFalse; inputs = current; } //--- Forecast for(int i = 0; i < cProjection.Total(); i++) { current = cProjection[i]; if(!current || !current.UpdateInputWeights(inputs)) ReturnFalse; inputs = current; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTransposeOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSAGDFN *Source = source; //--- Embedding if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau)) ReturnFalse; //--- GCRUs if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau)) ReturnFalse; if(!cGCRU.WeightsUpdate(Source.cGCRU.AsObject(), tau)) ReturnFalse; //--- Forecast if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::Save(const int file_handle) { if(!CNeuronTransposeOCL::Save(file_handle)) ReturnFalse; //--- if(!cTranspose.Save(file_handle)) ReturnFalse; //--- Embedding if(!cEmbedding.Save(file_handle)) ReturnFalse; //--- GCRUs if(!cGCRU.Save(file_handle) || !cAttention.Save(file_handle)) ReturnFalse; //--- Forecast if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSAGDFN::Load(const int file_handle) { if(!CNeuronTransposeOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; //--- Embedding if(!cEmbedding.Load(file_handle)) ReturnFalse; //--- GCRUs if(!cGCRU.Load(file_handle) || !LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- Forecast if(!cProjection.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSAGDFN::SetOpenCL(COpenCLMy *obj) { CNeuronTransposeOCL::SetOpenCL(obj); //--- cTranspose.SetOpenCL(OpenCL); cEmbedding.SetOpenCL(OpenCL); cGCRU.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpatialEmbedding : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; uint iEmbeddingDim; CParams cEmbedding; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpatialEmbedding(void) {}; ~CNeuronSpatialEmbedding(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSpatialEmbedding; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint embed_dim, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + embed_dim)*units, optimization_type, batch)) ReturnFalse; activation = None; if(!cEmbedding.Init(0, 0, OpenCL, embed_dim * units, optimization, iBatch)) ReturnFalse; cEmbedding.SetActivationFunction(TANH); //--- iUnits = units; iWindow = window; iEmbeddingDim = embed_dim; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(bTrain) if(!cEmbedding.FeedForward()) ReturnFalse; //--- if(!Concat(NeuronOCL.getOutput(), cEmbedding.getOutput(), Output, iWindow, iEmbeddingDim, iUnits)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cEmbedding.getGradient(), Gradient, iWindow, iEmbeddingDim, iUnits)) ReturnFalse; Deactivation(NeuronOCL) Deactivation(cEmbedding) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cEmbedding.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpatialEmbedding *Source = source; if(!cEmbedding.WeightsUpdate(Source.cEmbedding.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cEmbedding.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iEmbeddingDim) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpatialEmbedding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cEmbedding.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iEmbeddingDim = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpatialEmbedding::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cEmbedding.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTempEmbedding : public CNeuronBaseOCL { protected: uint iWindow; uint iUnits; uint aiEmbeddingDim[2]; uint aiFrames[2]; uint aiPeriod[2]; CParams caEmbeddings[2]; //--- virtual bool ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); virtual bool ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;} virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;} virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None ) override; public: CNeuronTempEmbedding(void) {}; ~CNeuronTempEmbedding(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint embed_dim1, uint period1, uint frame1, uint embed_dim2, uint period2, uint frame2, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronTempEmbedding; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint embed_dim1, uint period1, uint frame1, uint embed_dim2, uint period2, uint frame2, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, (window + embed_dim1 + embed_dim2)*units, optimization_type, batch)) ReturnFalse; //--- if(!caEmbeddings[0].Init(0, 0, OpenCL, embed_dim1 * period1, optimization, iBatch)) ReturnFalse; if(!caEmbeddings[1].Init(0, 1, OpenCL, embed_dim2 * period2, optimization, iBatch)) ReturnFalse; //--- iWindow = window; iUnits = units; aiEmbeddingDim[0] = embed_dim1; aiEmbeddingDim[1] = embed_dim2; aiFrames[0] = MathMax(1, frame1); aiFrames[1] = MathMax(1, frame2); aiPeriod[0] = period1; aiPeriod[1] = period2; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::ConcatByLabel(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < int(iUnits * iWindow) || SecondInput.Total() < (int)iUnits) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {iUnits, MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]), 3 }; uint kernel = def_k_ConcatByLabel; setBuffer(kernel, def_k_cbl_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex()) setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getOutputIndex()) setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getOutputIndex()) setBuffer(kernel, def_k_cbl_output, getOutputIndex()) setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow) setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0]) setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1]) setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0]) setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1]) setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0]) setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1]) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::ConcatByLabelGrad(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!OpenCL || !NeuronOCL || !SecondInput) ReturnFalse; if(NeuronOCL.Neurons() < int(iUnits * iWindow) || SecondInput.Total() < (int)iUnits) ReturnFalse; //--- uint global_work_offset[3] = {0, 0, 0}; uint global_work_size[3] = {MathMax(iUnits, aiPeriod[ArrayMaximum(aiPeriod)]), MathMax(iWindow, aiEmbeddingDim[ArrayMaximum(aiEmbeddingDim)]), 3 }; uint kernel = def_k_ConcatByLabelGrad; setBuffer(kernel, def_k_cbl_data, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_cbl_label, SecondInput.GetIndex()) setBuffer(kernel, def_k_cbl_embedding1, caEmbeddings[0].getGradientIndex()) setBuffer(kernel, def_k_cbl_embedding2, caEmbeddings[1].getGradientIndex()) setBuffer(kernel, def_k_cbl_output, getGradientIndex()) setArgument(kernel, def_k_cbl_dimension_data, (int)iWindow) setArgument(kernel, def_k_cbl_dimension_emb1, (int)aiEmbeddingDim[0]) setArgument(kernel, def_k_cbl_dimension_emb2, (int)aiEmbeddingDim[1]) setArgument(kernel, def_k_cbl_period1, (int)aiPeriod[0]) setArgument(kernel, def_k_cbl_period2, (int)aiPeriod[1]) setArgument(kernel, def_k_cbl_frame1, (int)aiFrames[0]) setArgument(kernel, def_k_cbl_frame2, (int)aiFrames[1]) setArgument(kernel, def_k_cbl_gr_units, (int)iUnits) //--- kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(bTrain) for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].FeedForward()) ReturnFalse; //--- return ConcatByLabel(NeuronOCL, SecondInput); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!ConcatByLabelGrad(NeuronOCL, SecondInput)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTempEmbedding* Source = source; for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].WeightsUpdate(Source.caEmbeddings[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) if(!caEmbeddings[i].Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iWindow) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; for(uint i = 0; i < caEmbeddings.Size(); i++) { if(FileWriteInteger(file_handle, (int)aiEmbeddingDim[i]) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiFrames[i]) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)aiPeriod[i]) < INT_VALUE) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTempEmbedding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caEmbeddings.Size(); i++) if(!LoadInsideLayer(file_handle, caEmbeddings[i].AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iWindow = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); for(uint i = 0; i < caEmbeddings.Size(); i++) { if(FileIsEnding(file_handle)) ReturnFalse; aiEmbeddingDim[i] = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; aiFrames[i] = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; aiPeriod[i] = (uint)FileReadInteger(file_handle); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTempEmbedding::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- for(uint i = 0; i < caEmbeddings.Size(); i++) caEmbeddings[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGlobalLocalAttention : public CNeuronMHFeedForward { protected: CNeuronSNSMHAttention cMask; CNeuronConvOCL cQ; CNeuronConvOCL cKV; CNeuronBaseOCL cScore; CNeuronBaseOCL cMHAttention; CNeuronConvOCL cW0; CNeuronBaseOCL cResidual; //--- virtual bool GlobalLocalAttention(void); virtual bool GlobalLocalAttentionGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGlobalLocalAttention(void) {}; ~CNeuronGlobalLocalAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint dimension_k, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronGlobalLocalAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint dimension_k, uint heads, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHFeedForward::Init(numOutputs, myIndex, open_cl, window, 2 * window, units, 1, heads, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cMask.Init(0, index, OpenCL, units, window, heads, m_units, sparse, optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, window, window, 2 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, window, window, 4 * dimension_k * heads, units, 1, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, 2 * dimension_k * heads, 2 * dimension_k * heads, window, units, 1, optimization, iBatch)) ReturnFalse; cW0.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttention(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttention; setBuffer(kernel, def_k_glatt_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_out, cMHAttention.getOutputIndex()) setArgument(kernel, def_k_glatt_dimension, dimension_k) setArgument(kernel, def_k_glatt_total_kv, units) setArgument(kernel, def_k_glatt_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::GlobalLocalAttentionGrad(void) { uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { units, units, 2 * heads }; uint local_work_size[3] = { 1, units, 1 }; //--- uint kernel = def_k_GlobalLocalAttentionGrad; setBuffer(kernel, def_k_glatt_gr_q, cQ.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_q_gr, cQ.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_kv, cKV.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_kv_gr, cKV.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_scores, cScore.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask, cMask.getOutputIndex()) setBuffer(kernel, def_k_glatt_gr_mask_gr, cMask.getGradientIndex()) setBuffer(kernel, def_k_glatt_gr_label, cMask.GetIndexes().GetIndex()) setBuffer(kernel, def_k_glatt_gr_out_gr, cMHAttention.getGradientIndex()) setArgument(kernel, def_k_glatt_gr_dimension, dimension_k) setArgument(kernel, def_k_glatt_gr_total_q, units) setArgument(kernel, def_k_glatt_gr_total_kv, units) setArgument(kernel, def_k_glatt_gr_total_mask, m_units) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cMask.FeedForward(NeuronOCL)) ReturnFalse; if(!cQ.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV.FeedForward(NeuronOCL)) ReturnFalse; if(!GlobalLocalAttention()) ReturnFalse; if(!cW0.FeedForward(cMHAttention.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cResidual.getOutput(), cW0.GetFilters(), true, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return CNeuronMHFeedForward::feedForward(cResidual.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMHFeedForward::calcInputGradients(cResidual.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cResidual.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttention.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!GlobalLocalAttentionGrad()) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQ.AsObject())) ReturnFalse; if(!DeActivation(cResidual.getOutput(), cResidual.getGradient(), cResidual.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cKV.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cMask.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cW0.GetFilters(), false, 0, 0, 0, cW0.GetUnits())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cMask.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cQ.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return CNeuronMHFeedForward::updateInputWeights(cResidual.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGlobalLocalAttention* Source = source; if(!cMask.WeightsUpdate(Source.cMask.AsObject(), tau)) ReturnFalse; if(!cQ.WeightsUpdate(Source.cQ.AsObject(), tau)) ReturnFalse; if(!cKV.WeightsUpdate(Source.cKV.AsObject(), tau)) ReturnFalse; if(!cW0.WeightsUpdate(Source.cW0.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Save(const int file_handle) { if(!CNeuronMHFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cMask.Save(file_handle)) ReturnFalse; if(!cQ.Save(file_handle)) ReturnFalse; if(!cKV.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobalLocalAttention::Load(const int file_handle) { if(!CNeuronMHFeedForward::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cMask.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cQ.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- uint units = cQ.GetUnits(); uint m_units = cMask.Neurons() / units; uint heads = cMask.GetWindow(); uint dimension_k = cQ.GetFilters() / (2 * heads); //--- int index = 3; if(!cScore.Init(0, index, OpenCL, (units + m_units)*units * heads, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, 2 * dimension_k * heads * units, optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); index += 2; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGlobalLocalAttention::SetOpenCL(COpenCLMy *obj) { CNeuronMHFeedForward::SetOpenCL(obj); //--- cMask.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronExtralonger : public CNeuronMHAttentionPooling { protected: CLayer cProjectionT; CLayer cTimeModule; CLayer cSpatialModule; CLayer cMixModule; CNeuronBaseOCL cConcatResults; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;} virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override {ReturnFalse;} virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None ) override; public: CNeuronExtralonger(void) {}; ~CNeuronExtralonger(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint heads, uint dimension_k, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronExtralonger; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint heads, uint dimension_k, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronMLMHAttentionOCL *att = NULL; CNeuronGlobalLocalAttention *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, layers, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; index++; att = new CNeuronMLMHAttentionOCL(); if(!att || !att.Init(0, index, OpenCL, dimension + emb_dimension, dimension_k, heads, time_steps_in, att_layers, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobalLocalAttention(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, variables, dimension_k, heads, m_units, sparse, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.FeedForward(prev, SecondInput)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Concatenate if(!Concat(cTimeModule[-1].getOutput(), cMixModule[-1].getOutput(), cSpatialModule[-1].getOutput(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- return CNeuronMHAttentionPooling::feedForward(cConcatResults.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcatResults.AsObject())) ReturnFalse; //--- DeConcatenate if(!cTimeModule[-1] || !cMixModule[-1] || !cSpatialModule[-1] || !DeConcat(cTimeModule[-1].getGradient(), cMixModule[-1].getGradient(), cSpatialModule[-1].getGradient(), cConcatResults.getOutput(), iWindow, iWindow, iWindow, iUnits)) ReturnFalse; //--- CNeuronBaseOCL *next = NULL; CNeuronBaseOCL *current = NULL; //--- Spatial Module for(int i = cSpatialModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cSpatialModule[i - 1] : NeuronOCL); next = cSpatialModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Mix Module for(int i = cMixModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cMixModule[i - 1] : cProjectionT[-1]); next = cMixModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Time Module CBufferFloat *temp = current.getGradient(); if(!current.SetGradient(current.getPrevOutput(), false)) ReturnFalse; for(int i = cTimeModule.Total() - 1; i >= 0; i--) { current = (i > 0 ? cTimeModule[i - 1] : cProjectionT[-1]); next = cTimeModule[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormilize(temp, current.getGradient(), temp, 1, false, 0, 0, 0, 1) || !current.SetGradient(temp, false)) ReturnFalse; //--- Time projection temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; for(int i = cProjectionT.Total() - 1; i >= 0; i--) { current = (i > 0 ? cProjectionT[i - 1] : NeuronOCL); next = cProjectionT[i]; if(!current || !current.CalcHiddenGradients(next, SecondInput, SecondGradient, SecondActivation)) ReturnFalse; } if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- Time projection for(int i = 0; i < cProjectionT.Total(); i++) { current = cProjectionT[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Time Module for(int i = 0; i < cTimeModule.Total(); i++) { current = cTimeModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Mix Module prev = cProjectionT[-1]; for(int i = 0; i < cMixModule.Total(); i++) { current = cMixModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Spatial Module prev = NeuronOCL; for(int i = 0; i < cSpatialModule.Total(); i++) { current = cSpatialModule[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronMHAttentionPooling::updateInputWeights(cConcatResults.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronExtralonger *Source = source; if(!cProjectionT.WeightsUpdate(Source.cProjectionT.AsObject(), tau)) ReturnFalse; if(!cTimeModule.WeightsUpdate(Source.cTimeModule.AsObject(), tau)) ReturnFalse; if(!cMixModule.WeightsUpdate(Source.cMixModule.AsObject(), tau)) ReturnFalse; if(!cSpatialModule.WeightsUpdate(Source.cSpatialModule.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cProjectionT.Save(file_handle)) ReturnFalse; if(!cTimeModule.Save(file_handle)) ReturnFalse; if(!cSpatialModule.Save(file_handle)) ReturnFalse; if(!cMixModule.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralonger::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!cProjectionT.Load(file_handle)) ReturnFalse; if(!cTimeModule.Load(file_handle)) ReturnFalse; if(!cSpatialModule.Load(file_handle)) ReturnFalse; if(!cMixModule.Load(file_handle)) ReturnFalse; //--- int index = cProjectionT.Total() + cTimeModule.Total() + cSpatialModule.Total() + cMixModule.Total(); if(!cConcatResults.Init(0, index, OpenCL, iHeads * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronExtralonger::SetOpenCL(COpenCLMy *obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cProjectionT.SetOpenCL(OpenCL); cTimeModule.SetOpenCL(OpenCL); cSpatialModule.SetOpenCL(OpenCL); cMixModule.SetOpenCL(OpenCL); cConcatResults.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGraphons : public CNeuronBaseOCL { protected: CLayer acProbability; CLayer acDataEmb; CParams cExpertsEmb; CNeuronBaseOCL cGraphs; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGraphons(void) {}; ~CNeuronGraphons(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronGraphons; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * units, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cGraphs.Init(0, index, OpenCL, Neurons()*experts, optimization, iBatch)) ReturnFalse; cGraphs.SetActivationFunction(SIGMOID); //--- CNeuronBaseOCL *neuron = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronDropoutOCL *dout = NULL; CNeuronSoftMaxOCL *softmax = NULL; //--- Probability acProbability.Clear(); acProbability.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(experts, index, OpenCL, window, window, experts, units, 1, optimization, iBatch) || !acProbability.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !acProbability.Add(neuron)) { DeleteObj(neuron); ReturnFalse; } index++; dout = new CNeuronDropoutOCL(); if(!dout || !dout.Init(0, index, OpenCL, experts, dropout, optimization, iBatch) || !acProbability.Add(dout)) { DeleteObj(dout); ReturnFalse; } index++; softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, experts, optimization, iBatch) || !acProbability.Add(softmax)) { DeleteObj(softmax); ReturnFalse; } softmax.SetHeads(1); //--- Data Embedding acDataEmb.Clear(); acDataEmb.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, 2 * emb_dimension, units, 1, optimization, iBatch) || !acDataEmb.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 2 * emb_dimension, 2 * emb_dimension, emb_dimension, units, 1, optimization, iBatch) || !acDataEmb.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units, emb_dimension, optimization, iBatch) || !acDataEmb.Add(transp)) { DeleteObj(transp); ReturnFalse; } transp.SetActivationFunction((ENUM_ACTIVATION)conv.Activation()); //--- index++; if(!cExpertsEmb.Init(0, index, OpenCL, units * emb_dimension * experts, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Data Embedding prev = NeuronOCL; for(int i = 0; i < acDataEmb.Total(); i++) { current = acDataEmb[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- Experts if(bTrain) if(!cExpertsEmb.FeedForward()) ReturnFalse; //--- Graphs uint units = (uint)MathSqrt((double)Neurons()); uint emb_dim = acDataEmb[-1].Neurons() / units; uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons(); if(!MatMul(cExpertsEmb.getOutput(), current.getOutput(), cGraphs.getOutput(), units, emb_dim, units, experts, false)) ReturnFalse; if(cGraphs.Activation() != None) if(!Activation(cGraphs.getOutput(), cGraphs.getOutput(), cGraphs.Activation())) ReturnFalse; //--- Result if(!MatMul(acProbability[-1].getOutput(), cGraphs.getOutput(), Output, 1, experts, units * units, 1, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!acDataEmb[-1] || !acProbability[-1]) ReturnFalse; //--- uint units = (uint)MathSqrt((double)Neurons()); uint emb_dim = acDataEmb[-1].Neurons() / units; uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons(); //--- if(!MatMulGrad(acProbability[-1].getOutput(), acProbability[-1].getGradient(), cGraphs.getOutput(), cGraphs.getGradient(), Gradient, 1, experts, units * units, 1, false)) ReturnFalse; //--- Graphs if(cGraphs.Activation() != None) if(!DeActivation(cGraphs.getOutput(), cGraphs.getGradient(), cGraphs.getGradient(), cGraphs.Activation())) ReturnFalse; if(!MatMulGrad(cExpertsEmb.getOutput(), cExpertsEmb.getGradient(), acDataEmb[-1].getOutput(), acDataEmb[-1].getGradient(), cGraphs.getGradient(), units, emb_dim, units, experts, false)) ReturnFalse; //--- CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = acProbability.Total() - 1; i >= 0; i--) { next = acProbability[i]; current = (i == 0 ? NeuronOCL : acProbability[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- Data Embedding CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; for(int i = acDataEmb.Total() - 1; i >= 0; i--) { next = acDataEmb[i]; current = (i == 0 ? NeuronOCL : acDataEmb[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Data Embedding prev = NeuronOCL; for(int i = 0; i < acDataEmb.Total(); i++) { current = acDataEmb[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Experts if(!cExpertsEmb.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGraphons *Source = source; if(!acProbability.WeightsUpdate(Source.acProbability.AsObject(), tau)) ReturnFalse; if(!acDataEmb.WeightsUpdate(Source.acDataEmb.AsObject(), tau)) ReturnFalse; if(!cExpertsEmb.WeightsUpdate(Source.cExpertsEmb.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!acProbability.Save(file_handle)) ReturnFalse; if(!acDataEmb.Save(file_handle)) ReturnFalse; if(!cExpertsEmb.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphons::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!acProbability.Load(file_handle)) ReturnFalse; if(!acDataEmb.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cExpertsEmb.AsObject())) ReturnFalse; //--- uint experts = cExpertsEmb.Neurons() / acDataEmb[-1].Neurons(); if(!cGraphs.Init(0, 0, OpenCL, Neurons()*experts, optimization, iBatch)) ReturnFalse; cGraphs.SetActivationFunction(SIGMOID); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphons::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); acProbability.SetOpenCL(OpenCL); acDataEmb.SetOpenCL(OpenCL); cExpertsEmb.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphons::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); acProbability.TrainMode(bTrain); acDataEmb.TrainMode(bTrain); cExpertsEmb.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGraphAttention : public CNeuronBaseOCL { protected: CNeuronConvOCL cValue; CNeuronGraphons cGraphs; CNeuronSoftMaxOCL cScores; CNeuronBaseOCL cAttention; CNeuronBaseOCL cResidual; CNeuronConvOCL cFeedForward[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGraphAttention(void) {}; ~CNeuronGraphAttention(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronGraphAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint emb_dimension, uint experts, float dropout, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units * units, optimization, iBatch)) ReturnFalse; cScores.SetHeads(units); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!MatMul(cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormilize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!MatMulGrad(cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cValue.GetUnits(), cValue.GetUnits(), cValue.GetFilters(), 1, false)) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 2; if(!cScores.Init(0, index, OpenCL, cGraphs.Neurons(), optimization, iBatch)) ReturnFalse; cScores.SetHeads(cValue.GetUnits()); index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSparseSoftMax : public CNeuronSoftMaxOCL { protected: uint iDimensionIn; CBufferFloat cIndexes; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSparseSoftMax(void) {}; ~CNeuronSparseSoftMax(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint dimension_in, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual int Type(void) override const { return defNeuronSparseSoftMax; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual CBufferFloat* GetIndexes(void) { return GetPointer(cIndexes); } virtual uint DimensionOut(void) const { return uint(Neurons() / iHeads); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseSoftMax::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint dimension_in, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension_in < dimension_out) ReturnFalse; if(!CNeuronSoftMaxOCL::Init(numOutputs, myIndex, open_cl, units * dimension_out, optimization_type, batch)) ReturnFalse; //--- SetHeads(units); iDimensionIn = dimension_in; if(!cIndexes.BufferInit(Neurons(), -1) || !cIndexes.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseSoftMax::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iHeads * iDimensionIn)) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { iHeads, iDimensionIn }; uint local_work_size[2] = { 1, global_work_size[1] }; uint kernel = def_k_SparseSoftMax; setBuffer(kernel, def_k_ssoftmax_data, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ssoftmax_outputs, getOutputIndex()) setBuffer(kernel, def_k_ssoftmax_indexes, cIndexes.GetIndex()) setArgument(kernel, def_k_ssoftmax_out_dimension, int(DimensionOut())) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseSoftMax::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!OpenCL || !NeuronOCL || NeuronOCL.Neurons() < int(iHeads * iDimensionIn)) ReturnFalse; uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = { iHeads, iDimensionIn }; uint local_work_size[2] = { 1, global_work_size[1] }; uint kernel = def_k_SparseSoftMaxGrad; setBuffer(kernel, def_k_ssoftmax_gr_data_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_ssoftmax_gr_outputs, getOutputIndex()) setBuffer(kernel, def_k_ssoftmax_gr_outputs_gr, getGradientIndex()) setBuffer(kernel, def_k_ssoftmax_gr_indexes, cIndexes.GetIndex()) setArgument(kernel, def_k_ssoftmax_gr_out_dimension, int(DimensionOut())) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseSoftMax::Save(const int file_handle) { if(!CNeuronSoftMaxOCL::Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iDimensionIn) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseSoftMax::Load(const int file_handle) { if(!CNeuronSoftMaxOCL::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iDimensionIn = (uint)FileReadInteger(file_handle); //--- cIndexes.BufferFree(); if(!cIndexes.BufferInit(Neurons(), -1) || !cIndexes.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseSoftMax::SetOpenCL(COpenCLMy *obj) { CNeuronSoftMaxOCL::SetOpenCL(obj); cIndexes.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSparseGraphAttention : public CNeuronBaseOCL { protected: CNeuronConvOCL cValue; CNeuronGraphons cGraphs; CNeuronSparseSoftMax cScores; CNeuronBaseOCL cAttention; CNeuronBaseOCL cResidual; CNeuronConvOCL cFeedForward[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSparseGraphAttention(void) {}; ~CNeuronSparseGraphAttention(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSparseGraphAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units, units, sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); index++; if(!cFeedForward[0].Init(0, index, OpenCL, window, window, 2 * window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[0].SetActivationFunction(SoftPlus); index++; if(!cFeedForward[1].Init(0, index, OpenCL, cFeedForward[0].GetFilters(), cFeedForward[0].GetFilters(), window, units, 1, optimization, iBatch)) ReturnFalse; cFeedForward[1].SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!SparseMatMul(cScores.GetIndexes(), cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cAttention.getOutput(), cResidual.getOutput(), cValue.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!cFeedForward[0].FeedForward(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].FeedForward(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormilize(cFeedForward[1].getOutput(), cResidual.getOutput(), Output, cFeedForward[1].GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!DeActivation(cFeedForward[1].getOutput(), cFeedForward[1].getGradient(), Gradient, cFeedForward[1].Activation())) ReturnFalse; if(!cFeedForward[0].CalcHiddenGradients(cFeedForward[1].AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cFeedForward[0].AsObject())) ReturnFalse; if(!SumAndNormilize(Gradient, cResidual.getGradient(), cAttention.getOutput(), cValue.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!SparseMatMulGrad(cScores.GetIndexes(), cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getGradient(), cAttention.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), cResidual.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), cValue.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeedForward[0].UpdateInputWeights(cResidual.AsObject())) ReturnFalse; if(!cFeedForward[1].UpdateInputWeights(cFeedForward[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSparseGraphAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; if(!cFeedForward[0].WeightsUpdate(Source.cFeedForward[0].AsObject(), tau)) ReturnFalse; if(!cFeedForward[1].WeightsUpdate(Source.cFeedForward[1].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; if(!cFeedForward[0].Save(file_handle)) ReturnFalse; if(!cFeedForward[1].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseGraphAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScores.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[0].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFeedForward[1].AsObject())) ReturnFalse; //--- int index = 3; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cFeedForward[0].SetOpenCL(OpenCL); cFeedForward[1].SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseGraphAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cFeedForward[0].TrainMode(bTrain); cFeedForward[1].TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGlobLocGraphAtt : public CNeuronMHAttentionPooling { protected: CNeuronGraphAttention cGlobal; CNeuronSparseGraphAttention cLocal; CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGlobLocGraphAtt(void) {}; ~CNeuronGlobLocGraphAtt(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronGlobLocGraphAtt; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, window, units, 2, optimization_type, batch)) ReturnFalse; //--- int index = 0; if(!cGlobal.Init(0, index, OpenCL, iUnits, iWindow, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cLocal.Init(0, index, OpenCL, iUnits, iWindow, experts, dropout, emb_dimension, sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cConcat.Init(0, index, OpenCL, 2 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cGlobal.FeedForward(NeuronOCL)) ReturnFalse; if(!cLocal.FeedForward(NeuronOCL)) ReturnFalse; if(!Concat(cGlobal.getOutput(), cLocal.getOutput(), cConcat.getOutput(), iWindow, iWindow, iUnits)) ReturnFalse; //--- return CNeuronMHAttentionPooling::feedForward(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMHAttentionPooling::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(cGlobal.getGradient(), cLocal.getGradient(), cConcat.getGradient(), iWindow, iWindow, iUnits)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGlobal.AsObject())) ReturnFalse; CBufferFloat *temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cLocal.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, PrevOutput, temp, iWindow, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cGlobal.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cLocal.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return CNeuronMHAttentionPooling::updateInputWeights(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMHAttentionPooling::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGlobLocGraphAtt * Source = source; if(!cGlobal.WeightsUpdate(Source.cGlobal.AsObject(), tau)) ReturnFalse; if(!cLocal.WeightsUpdate(Source.cLocal.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::Save(const int file_handle) { if(!CNeuronMHAttentionPooling::Save(file_handle)) ReturnFalse; //--- if(!cGlobal.Save(file_handle)) ReturnFalse; if(!cLocal.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGlobLocGraphAtt::Load(const int file_handle) { if(!CNeuronMHAttentionPooling::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cGlobal.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLocal.AsObject())) ReturnFalse; //--- if(!cConcat.Init(0, 2, OpenCL, 2 * iWindow * iUnits, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGlobLocGraphAtt::SetOpenCL(COpenCLMy *obj) { CNeuronMHAttentionPooling::SetOpenCL(obj); //--- cGlobal.SetOpenCL(OpenCL); cLocal.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGlobLocGraphAtt::TrainMode(bool flag) { CNeuronMHAttentionPooling::TrainMode(flag); //--- cGlobal.TrainMode(flag); cLocal.TrainMode(flag); cConcat.TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronExtralongerGraph : public CNeuronExtralonger { public: CNeuronExtralongerGraph(void) {}; ~CNeuronExtralongerGraph(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint experts, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronExtralongerGraph; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronExtralongerGraph::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint time_steps_in, uint time_steps_out, uint variables, uint dimension, uint emb_dimension, uint period1, uint frame1, uint period2, uint frame2, uint layers, uint experts, uint m_units, float sparse, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMHAttentionPooling::Init(numOutputs, myIndex, open_cl, variables, time_steps_out, 3, optimization_type, batch)) ReturnFalse; //--- CNeuronBatchNormOCL *norm = NULL; CNeuronConvOCL *conv = NULL; CNeuronTransposeOCL *transp = NULL; CNeuronLearnabledPE *lnoise = NULL; CNeuronSpatialEmbedding *semb = NULL; CNeuronTempEmbedding *temb = NULL; CNeuronGraphAttention *att = NULL; CNeuronGlobLocGraphAtt *glatt = NULL; //--- Time projection cProjectionT.Clear(); cProjectionT.SetOpenCL(OpenCL); int index = 0; lnoise = new CNeuronLearnabledPE(); if(!lnoise || !lnoise.Init(0, index, OpenCL, time_steps_in * variables, optimization, iBatch) || !cProjectionT.Add(lnoise)) { DeleteObj(lnoise); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, variables, variables, dimension, time_steps_in, 1, optimization, iBatch) || !cProjectionT.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; temb = new CNeuronTempEmbedding(); uint half_emb = (emb_dimension + 1) / 2; if(!temb || !temb.Init(0, index, OpenCL, time_steps_in, dimension, half_emb, period1, frame1, emb_dimension - half_emb, period2, frame2, optimization, iBatch) || !cProjectionT.Add(temb)) { DeleteObj(temb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, temb.Neurons(), iBatch, optimization) || !cProjectionT.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Time Module cTimeModule.Clear(); cTimeModule.SetOpenCL(OpenCL); for(uint i = 0; i < layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cTimeModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_in, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cTimeModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cTimeModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cTimeModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- Mix Module cMixModule.Clear(); cMixModule.SetOpenCL(OpenCL); uint att_layers = (layers + 1) / 2; for(uint i = 0; i < att_layers; i++) { index++; att = new CNeuronGraphAttention(); if(!att || !att.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, emb_dimension, experts, sparse, optimization, iBatch) || !cMixModule.Add(att)) { DeleteObj(att); ReturnFalse; } } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, dimension + emb_dimension, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } for(uint i = (att_layers == layers ? 0 : att_layers - 1); i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_in, experts, sparse, emb_dimension, uint(sparse * (dimension + emb_dimension)), optimization, iBatch) || !cMixModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, dimension + emb_dimension, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(TANH); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension + emb_dimension, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, variables, time_steps_out, 1, optimization, iBatch) || !cMixModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } //--- Spatial Module cSpatialModule.Clear(); cSpatialModule.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, time_steps_in, variables, optimization, iBatch) || !cSpatialModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_in, time_steps_in, dimension, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; semb = new CNeuronSpatialEmbedding(); if(!semb || !semb.Init(0, index, OpenCL, variables, dimension, emb_dimension, optimization, iBatch) || !cSpatialModule.Add(semb)) { DeleteObj(semb); ReturnFalse; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, semb.Neurons(), iBatch, optimization) || !cSpatialModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } for(uint i = 0; i < layers; i++) { index++; glatt = new CNeuronGlobLocGraphAtt(); if(!glatt || !glatt.Init(0, index, OpenCL, variables, dimension + emb_dimension, experts, sparse, emb_dimension, m_units, optimization, iBatch) || !cSpatialModule.Add(glatt)) { DeleteObj(glatt); ReturnFalse; } } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, dimension + emb_dimension, dimension + emb_dimension, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, time_steps_out, time_steps_out, time_steps_out, variables, 1, optimization, iBatch) || !cSpatialModule.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMixModule.Add(norm)) { DeleteObj(norm); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables, time_steps_out, optimization, iBatch) || !cMixModule.Add(transp)) { DeleteObj(transp); ReturnFalse; } //--- index++; if(!cConcatResults.Init(0, index, OpenCL, 3 * variables * time_steps_out, optimization, iBatch)) ReturnFalse; cConcatResults.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeConv : public CNeuronBaseOCL { protected: CNeuronConvOCL cConv; CParams cLevels; //--- virtual bool FloatToSpike(void); virtual bool FloatToSpikeGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeConv(void) {}; ~CNeuronSpikeConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual CBufferFloat* GetWeightsConv(void) { return cConv.GetWeightsConv(); } virtual int Type(void) override const { return defNeuronSpikeConv; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cConv.GetWindow(); } virtual uint GetFilters(void) const { return cConv.GetFilters(); } virtual uint GetVariables(void) const { return cConv.GetVariables(); } virtual uint GetUnits(void) const { return cConv.GetUnits(); } virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count * variables, optimization_type, batch)) ReturnFalse; activation = None; //--- if(!cConv.Init(0, 0, OpenCL, window, step, window_out, units_count, variables, optimization, iBatch)) ReturnFalse; cConv.SetActivationFunction(TANH); if(!cLevels.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cLevels.SetActivationFunction(SIGMOID); CBufferFloat* temp = cLevels.getWeightsParams(); if(!temp) ReturnFalse; if(!temp.Fill(-4)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::FloatToSpike(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; uint kernel = def_k_FloatToSpike; setBuffer(kernel, def_k_fts_values, cConv.getOutputIndex()) setBuffer(kernel, def_k_fts_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_fts_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- #ifdef _DEBUG cConv.getOutput().BufferRead(); cLevels.getOutput().BufferRead(); Output.BufferRead(); Sleep(0); #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::FloatToSpikeGrad(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; uint kernel = def_k_FloatToSpikeGrad; setBuffer(kernel, def_k_ftsg_values, cConv.getOutputIndex()) setBuffer(kernel, def_k_ftsg_values_gr, cConv.getGradientIndex()) setBuffer(kernel, def_k_ftsg_levels_gr, cLevels.getGradientIndex()) setBuffer(kernel, def_k_ftsg_gradients, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cConv.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) if(!cLevels.FeedForward()) ReturnFalse; //--- return FloatToSpike(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!FloatToSpikeGrad()) ReturnFalse; Deactivation(cConv) Deactivation(cLevels) if(!NeuronOCL.CalcHiddenGradients(cConv.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cConv.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cLevels.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeConv* Source = source; if(!cConv.WeightsUpdate(Source.cConv.AsObject(), tau)) ReturnFalse; if(!cLevels.WeightsUpdate(Source.cLevels.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cConv.Save(file_handle)) ReturnFalse; if(!cLevels.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLevels.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cConv.SetOpenCL(OpenCL); cLevels.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronGateLineAttention : public CNeuronSpikeConv { protected: uint iDimensionK; uint iVariables; uint iHeads; //--- CNeuronConvOCL cQKV; CNeuronBaseOCL cQ; CNeuronBaseOCL cK; CNeuronBaseOCL cV; CNeuronBaseOCL cKV; CNeuronConvOCL cGate; CNeuronBaseOCL cState; CNeuronBaseOCL cMHAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronGateLineAttention(void) {}; ~CNeuronGateLineAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint dimension, uint dimension_k, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronGateLineAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetVariables(void) const { return iVariables; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cState.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint variables, uint dimension, uint dimension_k, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConv::Init(numOutputs, myIndex, open_cl, heads * dimension_k, heads * dimension_k, dimension, variables, 1, optimization_type, batch)) ReturnFalse; //--- iDimensionK = dimension_k; iHeads = heads; iVariables = variables; //--- int index = 0; if(!cQKV.Init(0, index, OpenCL, dimension, dimension, 3 * iHeads * iDimensionK, iVariables, 1, optimization, iBatch)) ReturnFalse; cQKV.SetActivationFunction(SIGMOID); index++; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index++; if(!cGate.Init(0, index, OpenCL, dimension, dimension, iDimensionK * iHeads, iVariables, 1, optimization, iBatch)) ReturnFalse; cGate.SetActivationFunction(SIGMOID); index++; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(!cGate.FeedForward(NeuronOCL)) ReturnFalse; //--- if(!DeConcat(cQ.getOutput(), cK.getOutput(), cV.getOutput(), cQKV.getOutput(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; //--- if(!MatMul(cK.getOutput(), cV.getOutput(), cKV.getOutput(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!cState.SwapOutputs()) ReturnFalse; if(!DiagMatMul(cGate.getOutput(), cState.getPrevOutput(), cState.getOutput(), iDimensionK, iDimensionK, iHeads * iVariables, cState.Activation())) ReturnFalse; if(!SumAndNormilize(cState.getOutput(), cKV.getOutput(), cState.getOutput(), iDimensionK, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cQ.getOutput(), cState.getOutput(), cMHAttention.getOutput(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!CNeuronSpikeConv::feedForward(cMHAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConv::calcInputGradients(cMHAttention.AsObject())) ReturnFalse; //--- if(!MatMulGrad(cQ.getOutput(), cQ.getGradient(), cState.getOutput(), cState.getGradient(), cMHAttention.getGradient(), 1, iDimensionK, iDimensionK, iVariables * iHeads, true)) ReturnFalse; if(!DeActivation(cKV.getOutput(), cKV.getGradient(), cState.getGradient(), cKV.Activation())) ReturnFalse; if(!DiagMatMulGrad(cGate.getOutput(), cGate.getGradient(), cState.getPrevOutput(), cKV.getPrevOutput(), cState.getGradient(), iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cGate) if(!MatMulGrad(cK.getOutput(), cK.getGradient(), cV.getOutput(), cV.getGradient(), cKV.getGradient(), iDimensionK, 1, iDimensionK, iHeads * iVariables, true)) ReturnFalse; if(!Concat(cQ.getGradient(), cK.getGradient(), cV.getGradient(), cQKV.getGradient(), iDimensionK, iDimensionK, iDimensionK, iHeads * iVariables)) ReturnFalse; Deactivation(cQKV) //--- if(!NeuronOCL.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGate.AsObject())) ReturnFalse; if(!SumAndNormilize(PrevOutput, temp, temp, GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGate.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConv::updateInputWeights(cMHAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronGateLineAttention* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cGate.WeightsUpdate(Source.cGate.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Save(const int file_handle) { if(!CNeuronSpikeConv::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cGate.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iDimensionK) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronGateLineAttention::Load(const int file_handle) { if(!CNeuronSpikeConv::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGate.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iDimensionK = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); //--- int index = 1; if(!cQ.Init(0, index, OpenCL, iHeads * iDimensionK * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); index++; if(!cKV.Init(0, index, OpenCL, cQ.Neurons()*iDimensionK, optimization, iBatch)) ReturnFalse; cKV.SetActivationFunction(None); index += 2; if(!cState.Init(0, index, OpenCL, cKV.Neurons(), optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); cState.Clear(); index++; if(!cMHAttention.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttention.SetActivationFunction(None); //--- return Clear(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronGateLineAttention::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConv::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cGate.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cV.SetOpenCL(OpenCL); cKV.SetOpenCL(OpenCL); cState.SetOpenCL(OpenCL); cMHAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSparseAttention : public CNeuronSpikeConv { protected: CNeuronConvOCL cValue; CNeuronGraphons cGraphs; CNeuronSparseSoftMax cScores; CNeuronBaseOCL cAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeSparseAttention(void) {}; ~CNeuronSpikeSparseAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSparseAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConv::Init(numOutputs, myIndex, open_cl, window, window, window, units, 1, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cValue.Init(0, index, OpenCL, window, window, window, units, 1, optimization, iBatch)) ReturnFalse; cValue.SetActivationFunction(None); index++; if(!cGraphs.Init(0, index, OpenCL, units, window, emb_dimension, experts, dropout, optimization, iBatch)) ReturnFalse; index++; if(!cScores.Init(0, index, OpenCL, units, units, sparse_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cValue.FeedForward(NeuronOCL)) ReturnFalse; if(!cGraphs.FeedForward(NeuronOCL)) ReturnFalse; if(!cScores.FeedForward(cGraphs.AsObject())) ReturnFalse; if(!SparseMatMul(cScores.GetIndexes(), cScores.getOutput(), cValue.getOutput(), cAttention.getOutput(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; //--- return CNeuronSpikeConv::feedForward(cAttention.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConv::calcInputGradients(cAttention.AsObject())) ReturnFalse; if(!SparseMatMulGrad(cScores.GetIndexes(), cScores.getOutput(), cScores.getGradient(), cValue.getOutput(), cValue.getGradient(), cAttention.getGradient(), cScores.Heads(), cScores.DimensionOut(), cValue.GetUnits(), cValue.GetFilters())) ReturnFalse; if(!cGraphs.CalcHiddenGradients(cScores.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cGraphs.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cValue.AsObject())) ReturnFalse; if(!SumAndNormilize(PrevOutput, temp, temp, GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cValue.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cGraphs.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeConv::updateInputWeights(cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSparseAttention* Source = source; if(!cValue.WeightsUpdate(Source.cValue.AsObject(), tau)) ReturnFalse; if(!cGraphs.WeightsUpdate(Source.cGraphs.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::Save(const int file_handle) { if(!CNeuronSpikeConv::Save(file_handle)) ReturnFalse; if(!cValue.Save(file_handle)) ReturnFalse; if(!cGraphs.Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSparseAttention::Load(const int file_handle) { if(!CNeuronSpikeConv::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cValue.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGraphs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScores.AsObject())) ReturnFalse; //--- int index = 3; if(!cAttention.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cAttention.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSparseAttention::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConv::SetOpenCL(obj); //--- cValue.SetOpenCL(OpenCL); cGraphs.SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSparseAttention::TrainMode(bool flag) { CNeuronSpikeConv::TrainMode(flag); //--- cValue.TrainMode(bTrain); cGraphs.TrainMode(bTrain); cScores.TrainMode(bTrain); cAttention.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMoEConv : public CNeuronBaseOCL { protected: CLayer acProbability; CParams cExperts; CNeuronBaseOCL cWeightsConv; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMoEConv(void) {}; ~CNeuronMoEConv(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint experts, float dropout, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMoEConv; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint window_out, uint experts, float dropout, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch)) ReturnFalse; //--- int index = 0; //--- CNeuronConvOCL *conv = NULL; CNeuronDropoutOCL *dout = NULL; CNeuronSparseSoftMax *softmax = NULL; //--- Probability acProbability.Clear(); acProbability.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(experts, index, OpenCL, window, window, 2 * experts, units, 1, optimization, iBatch) || !acProbability.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(experts, index, OpenCL, 2 * experts, 2 * experts, experts, units, 1, optimization, iBatch) || !acProbability.Add(conv)) { DeleteObj(conv); ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; dout = new CNeuronDropoutOCL(); if(!dout || !dout.Init(0, index, OpenCL, conv.Neurons(), dropout, optimization, iBatch) || !acProbability.Add(dout)) { DeleteObj(dout); ReturnFalse; } index++; softmax = new CNeuronSparseSoftMax; if(!softmax || !softmax.Init(0, index, OpenCL, units, experts, topK, optimization, iBatch) || !acProbability.Add(softmax)) { DeleteObj(softmax); ReturnFalse; } softmax.SetHeads(units); //--- index++; if(!cExperts.Init(0, index, OpenCL, window * window_out * experts, optimization, iBatch)) ReturnFalse; index++; if(!cWeightsConv.Init(0, index, OpenCL, window * window_out * units, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; CNeuronConvOCL *conv = NULL; CNeuronSparseSoftMax *softmax = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } conv = acProbability[0]; softmax = prev; //--- Experts if(bTrain) if(!cExperts.FeedForward()) ReturnFalse; //--- uint units = conv.GetUnits(); uint window_in = conv.GetWindow(); uint window_out = Neurons() / units; uint topK = softmax.DimensionOut(); uint experts = cExperts.Neurons() / (window_in * window_out); //--- Generate Weights if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cExperts.getOutput(), cWeightsConv.getOutput(), units, topK, window_in * window_out, experts)) ReturnFalse; //--- Result if(!MatMul(NeuronOCL.getOutput(), cWeightsConv.getOutput(), Output, 1, window_in, window_out, units, true)) ReturnFalse; if(activation != None) if(!Activation(Output, Output, activation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!acProbability[0] || !acProbability[-1]) ReturnFalse; //--- CNeuronConvOCL* conv = acProbability[0]; CNeuronSparseSoftMax* softmax = acProbability[-1]; uint units = conv.GetUnits(); uint window_in = conv.GetWindow(); uint window_out = Neurons() / units; uint topK = softmax.DimensionOut(); uint experts = cExperts.Neurons() / (window_in * window_out); //--- if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), cWeightsConv.getOutput(), cWeightsConv.getGradient(), Gradient, 1, window_in, window_out, units, true)) ReturnFalse; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cExperts.getOutput(), cExperts.getGradient(), cWeightsConv.getGradient(), units, topK, window_in * window_out, experts)) ReturnFalse; //--- CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = acProbability.Total() - 1; i >= 0; i--) { next = acProbability[i]; current = (i == 0 ? NeuronOCL : acProbability[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), window_in, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- Probability for(int i = 0; i < acProbability.Total(); i++) { current = acProbability[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- Experts if(!cExperts.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMoEConv *Source = source; if(!acProbability.WeightsUpdate(Source.acProbability.AsObject(), tau)) ReturnFalse; if(!cExperts.WeightsUpdate(Source.cExperts.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!acProbability.Save(file_handle)) ReturnFalse; if(!cExperts.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMoEConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!acProbability.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cExperts.AsObject())) ReturnFalse; //--- CNeuronConvOCL* conv = acProbability[0]; uint window_in = conv.GetWindow(); int index = acProbability.Total() + 1; if(!cWeightsConv.Init(0, index, OpenCL, window_in * Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMoEConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); acProbability.SetOpenCL(OpenCL); cExperts.SetOpenCL(OpenCL); cWeightsConv.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMoEConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); acProbability.TrainMode(bTrain); cExperts.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikingBrain : public CNeuronBaseOCL { protected: CNeuronSpikeSparseAttention cTimeAttention; CLayer caSpatialAttention; CNeuronBaseOCL cResidual; CLayer caFFN; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikingBrain(void) {}; ~CNeuronSpikingBrain(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikingBrain; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint experts, float dropout, uint emb_dimension, uint sparse_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cTimeAttention.Init(0, index, OpenCL, units, window, experts, dropout, emb_dimension, sparse_dimension, optimization, iBatch)) ReturnFalse; //--- CNeuronTransposeOCL* transp = NULL; CNeuronGateLineAttention* line_att = NULL; caSpatialAttention.Clear(); caSpatialAttention.SetOpenCL(OpenCL); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units, window, optimization, iBatch) || !caSpatialAttention.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; line_att = new CNeuronGateLineAttention(); if(!line_att || !line_att.Init(0, index, OpenCL, window, units, emb_dimension, experts, optimization, iBatch) || !caSpatialAttention.Add(line_att)) { DeleteObj(line_att) ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window, units, optimization, iBatch) || !caSpatialAttention.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; if(!cResidual.Init(0, index, OpenCL, window * units, optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- CNeuronSpikeConv* conv = NULL; CNeuronMoEConv* moe = NULL; caFFN.Clear(); caFFN.SetOpenCL(OpenCL); index++; moe = new CNeuronMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, units, window, 2 * window, experts, dropout, sparse_dimension, optimization, iBatch) || !caFFN.Add(moe)) { DeleteObj(moe) ReturnFalse; } moe.SetActivationFunction(SoftPlus); index++; conv = new CNeuronSpikeConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * window, 2 * window, window, units, 1, optimization, iBatch) || !caFFN.Add(conv)) { DeleteObj(conv) ReturnFalse; } if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- if(!cTimeAttention.FeedForward(prev)) ReturnFalse; for(int i = 0; i < caSpatialAttention.Total(); i++) { current = caSpatialAttention[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } if(!SumAndNormilize(prev.getOutput(), cTimeAttention.getOutput(), cResidual.getOutput(), cTimeAttention.GetFilters(), false, 0, 0, 0, 1) || !SumAndNormilize(cResidual.getOutput(), NeuronOCL.getOutput(), cResidual.getOutput(), cTimeAttention.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- prev = cResidual.AsObject(); for(int i = 0; i < caFFN.Total(); i++) { current = caFFN[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } if(!SumAndNormilize(cResidual.getOutput(), prev.getOutput(), Output, cTimeAttention.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* current = caFFN[-1]; if(!current || !DeActivation(current.getOutput(), current.getGradient(), Gradient, current.Activation())) ReturnFalse; for(int i = caFFN.Total() - 1; i >= 0; i--) { next = caFFN[i]; current = (i == 0 ? cResidual.AsObject() : caFFN[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormilize(cResidual.getGradient(), Gradient, cResidual.getGradient(), cTimeAttention.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- current = caSpatialAttention[-1]; if(!DeActivation(current.getOutput(), current.getGradient(), cResidual.getGradient(), current.Activation())) ReturnFalse; if(!DeActivation(cTimeAttention.getOutput(), cTimeAttention.getGradient(), cResidual.getGradient(), cTimeAttention.Activation())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cResidual.getPrevOutput(), cResidual.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cTimeAttention.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getPrevOutput(), cResidual.getPrevOutput(), cTimeAttention.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = caSpatialAttention.Total() - 1; i >= 0; i--) { next = caSpatialAttention[i]; current = (i == 0 ? NeuronOCL : caSpatialAttention[i - 1]); if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; } if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getPrevOutput(), NeuronOCL.getGradient(), cTimeAttention.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; //--- if(!cTimeAttention.UpdateInputWeights(prev)) ReturnFalse; for(int i = 0; i < caSpatialAttention.Total(); i++) { current = caSpatialAttention[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- prev = cResidual.AsObject(); for(int i = 0; i < caFFN.Total(); i++) { current = caFFN[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikingBrain* Source = source; if(!cTimeAttention.WeightsUpdate(Source.cTimeAttention.AsObject(), tau)) ReturnFalse; if(!caSpatialAttention.WeightsUpdate(Source.caSpatialAttention.AsObject(), tau)) ReturnFalse; if(!caFFN.WeightsUpdate(Source.caFFN.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cTimeAttention.Save(file_handle)) ReturnFalse; if(!caSpatialAttention.Save(file_handle)) ReturnFalse; if(!caFFN.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cTimeAttention.AsObject())) ReturnFalse; if(!caSpatialAttention.Load(file_handle)) ReturnFalse; if(!caFFN.Load(file_handle)) ReturnFalse; //--- int index = caSpatialAttention.Total() + 1; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikingBrain::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cTimeAttention.SetOpenCL(OpenCL); caSpatialAttention.SetOpenCL(OpenCL); caFFN.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikingBrain::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cTimeAttention.TrainMode(bTrain); caSpatialAttention.TrainMode(bTrain); caFFN.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikingBrain::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cTimeAttention.Clear()) ReturnFalse; for(int i = 0; i < caSpatialAttention.Total(); i++) if(!caSpatialAttention[i] || !caSpatialAttention[i].Clear()) ReturnFalse; for(int i = 0; i < caFFN.Total(); i++) if(!caFFN[i] || !caFFN[i].Clear()) ReturnFalse; if(!cResidual.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeActivation : public CNeuronBaseOCL { protected: CNeuronBaseOCL cHV; CParams cLevels; CParams cGates; //--- virtual bool FloatToSpike(void); virtual bool FloatToSpikeGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeActivation(void) {}; ~CNeuronSpikeActivation(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint neurons, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeActivation; } //--- methods for working with files virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, numNeurons, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cHV.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cHV.SetActivationFunction(None); cHV.Clear(); //--- index++; if(!cLevels.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cLevels.SetActivationFunction(SoftPlus); CBufferFloat* temp = cLevels.getWeightsParams(); if(!temp || !temp.Fill(-3.0f)) ReturnFalse; //--- index++; if(!cGates.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cGates.SetActivationFunction(SIGMOID); temp = cGates.getWeightsParams(); if(!temp || !temp.Fill(0.0f)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!cHV.SwapOutputs()) ReturnFalse; if(bTrain) { if(!cLevels.FeedForward()) ReturnFalse; if(!cGates.FeedForward()) ReturnFalse; } //--- if(!GateElementMult(NeuronOCL.getOutput(), cHV.getPrevOutput(), cGates.getOutput(), cHV.getOutput())) ReturnFalse; return FloatToSpike(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!FloatToSpikeGrad()) ReturnFalse; //--- if(!GateElementMultGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cHV.getPrevOutput(), cLevels.getPrevOutput(), cGates.getOutput(), cGates.getGradient(), cHV.getGradient(), NeuronOCL.Activation(), cHV.Activation(), cGates.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::FloatToSpike(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; uint kernel = def_k_FloatToSpike; setBuffer(kernel, def_k_fts_values, cHV.getOutputIndex()) setBuffer(kernel, def_k_fts_levels, cLevels.getOutputIndex()) setBuffer(kernel, def_k_fts_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::FloatToSpikeGrad(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = { Neurons() }; uint kernel = def_k_FloatToSpikeGrad; setBuffer(kernel, def_k_ftsg_values, cHV.getOutputIndex()) setBuffer(kernel, def_k_ftsg_values_gr, cHV.getGradientIndex()) setBuffer(kernel, def_k_ftsg_levels_gr, cLevels.getGradientIndex()) setBuffer(kernel, def_k_ftsg_gradients, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cLevels.UpdateInputWeights()) ReturnFalse; if(!cGates.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeActivation* Source = source; if(!cLevels.WeightsUpdate(Source.cLevels.AsObject(), tau)) ReturnFalse; if(!cGates.WeightsUpdate(Source.cGates.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cLevels.Save(file_handle)) ReturnFalse; if(!cGates.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLevels.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cGates.AsObject())) ReturnFalse; //--- uint index = 0; if(!cHV.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cHV.SetActivationFunction(None); cHV.Clear(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeActivation::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cLevels.SetOpenCL(OpenCL); cGates.SetOpenCL(OpenCL); cHV.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeActivation::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cLevels.TrainMode(bTrain); cGates.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeActivation::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cHV.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeResNeXtBottleneck : public CNeuronSpikeActivation { protected: CLayer cBottleneck; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeResNeXtBottleneck(void) {}; ~CNeuronSpikeResNeXtBottleneck(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeResNeXtBottleneck; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, chanels_out * units_out, optimization_type, batch)) ReturnFalse; //--- CNeuronConvOCL *conv = NULL; CNeuronMultiWindowsConvOCL *mwconv = NULL; CNeuronBatchNormOCL *norm = NULL; CNeuronSpikeActivation *spike = NULL; CNeuronTransposeOCL* transp = NULL; //--- cBottleneck.Clear(); cBottleneck.SetOpenCL(OpenCL); //--- Projection in uint index = 0; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, chanels_in, chanels_in, group_size * groups, units_in, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cBottleneck.Add(spike)) { DeleteObj(spike) ReturnFalse; } //--- Feature Extraction index++; uint windows[]; if(ArrayResize(windows, groups) < (int)groups) ReturnFalse; ArrayFill(windows, 0, groups, group_size); mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv || !mwconv.Init(0, index, OpenCL, windows, group_size, 1, units_in, optimization, iBatch) || !cBottleneck.Add(mwconv)) { DeleteObj(mwconv) ReturnFalse; } mwconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, mwconv.Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cBottleneck.Add(spike)) { DeleteObj(spike) ReturnFalse; } //--- Projection out index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, group_size * groups, group_size * groups, chanels_out, units_in, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); if(units_in == units_out) return true; index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cBottleneck.Add(spike)) { DeleteObj(spike) ReturnFalse; } //--- Projection units index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, chanels_out, optimization, iBatch) || !cBottleneck.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, units_out, chanels_out, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, chanels_out, units_out, optimization, iBatch) || !cBottleneck.Add(transp)) { DeleteObj(transp) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; #ifdef _DEBUG vector res; #endif //--- for(int i = 0; i < cBottleneck.Total(); i++) { current = cBottleneck[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; #ifdef _DEBUG if(!current.getOutputVal(res)) ReturnFalse; #endif prev = current; } //--- return CNeuronSpikeActivation::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL *current = (cBottleneck.Total() > 0 ? cBottleneck[-1] : NeuronOCL); if(!CNeuronSpikeActivation::calcInputGradients(current)) ReturnFalse; for(int i = cBottleneck.Total() - 1; i >= 0; i--) { current = (i > 0 ? cBottleneck[i - 1] : NeuronOCL); if(!current || !current.CalcHiddenGradients(cBottleneck[i])) ReturnFalse; } //--- result return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *current = NULL; //--- for(int i = 0; i < cBottleneck.Total(); i++) { current = cBottleneck[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeActivation::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; if(!cBottleneck.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; if(!cBottleneck.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeResNeXtBottleneck::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); cBottleneck.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikeResNeXtBottleneck* Source = source; if(!cBottleneck.WeightsUpdate(Source.cBottleneck.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeResNeXtBottleneck::TrainMode(bool flag) { CNeuronSpikeActivation::TrainMode(flag); cBottleneck.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBottleneck::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; for(int i = 0; i < cBottleneck.Total(); i++) if(!cBottleneck[i] || !cBottleneck[i].Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeResNeXtResidual : public CNeuronSpikeResNeXtBottleneck { public: CNeuronSpikeResNeXtResidual(void) {}; ~CNeuronSpikeResNeXtResidual(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeResNeXtResidual; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtResidual::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, chanels_out * units_out, optimization_type, batch)) ReturnFalse; //--- CNeuronConvOCL *conv = NULL; CNeuronBatchNormOCL *norm = NULL; CNeuronSpikeActivation *spike = NULL; CNeuronTransposeOCL* transp = NULL; //--- cBottleneck.Clear(); cBottleneck.SetOpenCL(OpenCL); //--- Projection Chanels uint index = 0; if(chanels_in != chanels_out) { conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, chanels_in, chanels_in, chanels_out, units_in, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; } //--- Projection Units if(units_in != units_out) { if(chanels_in != chanels_out) { norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cBottleneck.Add(spike)) { DeleteObj(spike) ReturnFalse; } index++; } transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, chanels_out, optimization, iBatch) || !cBottleneck.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, units_out, chanels_out, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, chanels_out, units_out, optimization, iBatch) || !cBottleneck.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, Neurons(), iBatch, optimization) || !cBottleneck.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeResNeXtBlock : public CNeuronSpikeActivation { protected: CNeuronTransposeOCL caTranspose[2]; CNeuronSpikeResNeXtBottleneck caBottleneck[2]; CNeuronSpikeResNeXtResidual cResidual; CLayer cG; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeResNeXtBlock(void) {}; ~CNeuronSpikeResNeXtBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeResNeXtBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; //--- virtual uint GetChannelsOut(void) const { return caTranspose[1].GetCount(); } virtual uint GetUnitsOut(void) const { return caTranspose[1].GetWindow(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, chanels_out * units_out, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!caTranspose[0].Init(0, index, OpenCL, units_in, chanels_in, optimization, iBatch)) ReturnFalse; index++; if(!caTranspose[1].Init(0, index, OpenCL, chanels_out, units_out, optimization, iBatch)) ReturnFalse; //--- index++; if(!caBottleneck[0].Init(0, index, OpenCL, chanels_in, chanels_out, units_in, units_out, group_size, groups, optimization, iBatch)) ReturnFalse; index++; if(!caBottleneck[1].Init(0, index, OpenCL, units_in, units_out, chanels_in, chanels_out, group_size, groups, optimization, iBatch)) ReturnFalse; //--- index++; if(!cResidual.Init(0, index, OpenCL, chanels_in, chanels_out, units_in, units_out, optimization, iBatch)) ReturnFalse; //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; cG.Clear(); cG.SetOpenCL(OpenCL); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, Neurons(), optimization, iBatch) || !cG.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); //--- if(!cResidual.SetGradient(neuron.getGradient(), true) || !caTranspose[1].SetGradient(neuron.getGradient(), true) || !caBottleneck[0].SetGradient(neuron.getGradient(), true)) ReturnFalse; //--- index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, Neurons(), iBatch, optimization) || !cG.Add(norm)) { DeleteObj(norm) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!caTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; if(!caBottleneck[0].FeedForward(NeuronOCL)) ReturnFalse; if(!caBottleneck[1].FeedForward(caTranspose[0].AsObject())) ReturnFalse; if(!caTranspose[1].FeedForward(caBottleneck[1].AsObject())) ReturnFalse; if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = cG[0]; if(!current) ReturnFalse; if(!SumAndNormilize(caBottleneck[0].getOutput(), caTranspose[1].getOutput(), current.getOutput(), caTranspose[1].GetCount(), false, 0, 0, 0, 1) || !SumAndNormilize(current.getOutput(), cResidual.getOutput(), current.getOutput(), caTranspose[1].GetCount(), false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.FeedForward(cG[i - 1])) ReturnFalse; } //--- return CNeuronSpikeActivation::feedForward(cG[-1]); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeActivation::calcInputGradients(cG[-1])) ReturnFalse; CNeuronBaseOCL* current = NULL; for(int i = cG.Total() - 2; i >= 0; i--) { current = cG[i]; if(!current || !current.CalcHiddenGradients(cG[i + 1])) ReturnFalse; } //--- if(!caBottleneck[1].CalcHiddenGradients(caTranspose[1].AsObject())) ReturnFalse; if(!caTranspose[0].CalcHiddenGradients(caBottleneck[1].AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(caTranspose[0].getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(caBottleneck[0].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, caTranspose[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(caTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, caTranspose[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!caBottleneck[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caBottleneck[1].UpdateInputWeights(caTranspose[0].AsObject())) ReturnFalse; if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.UpdateInputWeights(cG[i - 1])) ReturnFalse; } //--- return CNeuronSpikeActivation::updateInputWeights(cG[-1]); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeResNeXtBlock* Source = source; for(uint i = 0; i < caBottleneck.Size(); i++) if(!caBottleneck[i].WeightsUpdate(Source.caBottleneck[i].AsObject(), tau)) ReturnFalse; if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; for(int i = 0; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.WeightsUpdate(Source.cG[i], tau)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caBottleneck.Size(); i++) if(!caBottleneck[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < caTranspose.Size(); i++) if(!caTranspose[i].Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; if(!cG.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caBottleneck.Size(); i++) if(!LoadInsideLayer(file_handle, caBottleneck[i].AsObject())) ReturnFalse; for(uint i = 0; i < caTranspose.Size(); i++) if(!LoadInsideLayer(file_handle, caTranspose[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; if(!cG.Load(file_handle)) ReturnFalse; //--- CNeuronBaseOCL* neuron = cG[0]; if(!neuron || !cResidual.SetGradient(neuron.getGradient(), true) || !caTranspose[1].SetGradient(neuron.getGradient(), true) || !caBottleneck[0].SetGradient(neuron.getGradient(), true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeResNeXtBlock::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; //--- for(uint i = 0; i < caBottleneck.Size(); i++) if(!caBottleneck[i].Clear()) ReturnFalse; for(uint i = 0; i < caTranspose.Size(); i++) if(!caTranspose[i].Clear()) ReturnFalse; if(!cResidual.Clear()) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; for(int i = 0; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeResNeXtBlock::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); //--- for(uint i = 0; i < caBottleneck.Size(); i++) caBottleneck[i].SetOpenCL(OpenCL); for(uint i = 0; i < caTranspose.Size(); i++) caTranspose[i].SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cG.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeResNeXtBlock::TrainMode(bool flag) { CNeuronSpikeActivation::TrainMode(flag); //--- for(uint i = 0; i < caBottleneck.Size(); i++) caBottleneck[i].TrainMode(bTrain); for(uint i = 0; i < caTranspose.Size(); i++) caTranspose[i].TrainMode(bTrain); cResidual.TrainMode(bTrain); cG.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSSAM : public CNeuronSpikeActivation { protected: uint iUnits; uint iHeads; uint iDimension; bool bMaskFuture; CNeuronSpikeConv cQKV; CParams cDiaganalBias; int ciScore; CNeuronBaseOCL cMHAttentionOut; CNeuronSpikeConv cW0; CNeuronBaseOCL cResidual; CNeuronBatchNormOCL cNorm; //--- virtual bool SpikeMHAttention(void); virtual bool SpikeMHAttentionGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSSAM(void) : ciScore(INVALID_HANDLE) {}; ~CNeuronSSAM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, bool mask_future, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSSAM; } //--- methods for working with files virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::SpikeMHAttention(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = {0}; uint global_work_size[3] = { iUnits, iUnits, iHeads }; uint local_work_size[3] = { 1, iUnits, 1 }; uint kernel = def_k_SpikeMHAttention; setBuffer(kernel, def_k_smhat_qkv, cQKV.getOutputIndex()) setBuffer(kernel, def_k_smhat_diag_bias, cDiaganalBias.getOutputIndex()) setBuffer(kernel, def_k_smhat_scores, ciScore) setBuffer(kernel, def_k_smhat_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_smhat_dimension, int(iDimension)) setArgument(kernel, def_k_smhat_mask_future, int(bMaskFuture)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::SpikeMHAttentionGrad(void) { if(!OpenCL) ReturnFalse; uint global_work_offset[3] = { 0 }; uint global_work_size[3] = { iUnits, iUnits, iHeads }; uint local_work_size[3] = { 1, iUnits, 1 }; uint kernel = def_k_SpikeMHAttentionGrad; setBuffer(kernel, def_k_smhat_gr_qkv, cQKV.getOutputIndex()) setBuffer(kernel, def_k_smhat_gr_qkv_gr, cQKV.getGradientIndex()) setBuffer(kernel, def_k_smhat_gr_diag_bias, cDiaganalBias.getOutputIndex()) setBuffer(kernel, def_k_smhat_gr_diag_bias_gr, cDiaganalBias.getGradientIndex()) setBuffer(kernel, def_k_smhat_gr_scores, ciScore) setBuffer(kernel, def_k_smhat_gr_gradients, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_smhat_gr_dimension, int(iDimension)) setArgument(kernel, def_k_smhat_gr_mask_future, int(bMaskFuture)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, bool mask_future, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units * window, optimization_type, batch)) ReturnFalse; //--- iHeads = (uint)MathMax(heads, 1); iDimension = (uint)MathMax(dimension_k, 1); iUnits = units; bMaskFuture = mask_future; //--- uint index = 0; if(!cQKV.Init(0, index, OpenCL, window, window, 3 * iHeads * iDimension, iUnits, 1, optimization, iBatch)) ReturnFalse; index++; if(!cDiaganalBias.Init(0, index, OpenCL, iUnits, optimization, iBatch)) ReturnFalse; cDiaganalBias.SetActivationFunction(None); CBufferFloat* temp = cDiaganalBias.getWeightsParams(); if(!temp || !temp.Fill(0)) ReturnFalse; ciScore = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * iHeads, CL_MEM_READ_WRITE); if(ciScore == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, iUnits * iDimension * iHeads, optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, iHeads * iDimension, iHeads * iDimension, window, iUnits, 1, optimization, iBatch)) ReturnFalse; index++; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); if(!cResidual.SetGradient(cW0.getGradient(), true)) ReturnFalse; index++; if(!cNorm.Init(0, index, OpenCL, Neurons(), iBatch, optimization)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) if(!cDiaganalBias.FeedForward()) ReturnFalse; if(!SpikeMHAttention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cResidual.getOutput(), Neurons() / iUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!cNorm.FeedForward(cResidual.AsObject())) ReturnFalse; //--- return CNeuronSpikeActivation::feedForward(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeActivation::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!cResidual.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!SpikeMHAttentionGrad()) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cQKV.AsObject())) ReturnFalse; if(NeuronOCL.Activation() == None) { if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getGradient(), NeuronOCL.getGradient(), Neurons() / iUnits, false, 0, 0, 0, 1)) ReturnFalse; } else { if(!DeActivation(NeuronOCL.getOutput(), cResidual.getPrevOutput(), cResidual.getGradient(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cResidual.getPrevOutput(), NeuronOCL.getGradient(), Neurons() / iUnits, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQKV.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cDiaganalBias.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cResidual.AsObject())) ReturnFalse; //--- return CNeuronSpikeActivation::updateInputWeights(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSSAM* Source = source; if(!cQKV.WeightsUpdate(Source.cQKV.AsObject(), tau)) ReturnFalse; if(!cDiaganalBias.WeightsUpdate(Source.cDiaganalBias.AsObject(), tau)) ReturnFalse; if(!cW0.WeightsUpdate(Source.cW0.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; //--- if(!cQKV.Save(file_handle)) ReturnFalse; if(!cDiaganalBias.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iUnits) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iHeads) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)bMaskFuture) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cQKV.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDiaganalBias.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iUnits = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; bMaskFuture = (bool)FileReadInteger(file_handle); //--- uint index = 4; if(!cResidual.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); if(!cW0.SetGradient(cResidual.getGradient(), true)) ReturnFalse; if(ciScore != INVALID_HANDLE) OpenCL.BufferFree(ciScore); ciScore = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * iHeads, CL_MEM_READ_WRITE); if(ciScore == INVALID_HANDLE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAM::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; //--- if(!cQKV.Clear()) ReturnFalse; if(!cW0.Clear()) ReturnFalse; if(!cNorm.Clear()) ReturnFalse; if(!cResidual.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSAM::SetOpenCL(COpenCLMy *obj) { if(OpenCL && ciScore != INVALID_HANDLE) OpenCL.BufferFree(ciScore); //--- CNeuronSpikeActivation::SetOpenCL(obj); //--- cQKV.SetOpenCL(OpenCL); cDiaganalBias.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); ciScore = OpenCL.AddBuffer(sizeof(float) * iUnits * iUnits * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSAM::TrainMode(bool flag) { CNeuronSpikeActivation::TrainMode(flag); //--- cQKV.TrainMode(bTrain); cDiaganalBias.TrainMode(bTrain); cW0.TrainMode(bTrain); cNorm.TrainMode(bTrain); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSSAMResNeXtBlock : public CNeuronSpikeResNeXtBlock { protected: CNeuronSSAM caSSAM[2]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSSAMResNeXtBlock(void) {}; ~CNeuronSSAMResNeXtBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSSAMResNeXtBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, uint group_size, uint groups, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeResNeXtBlock::Init(numOutputs, myIndex, open_cl, chanels_in, chanels_out, units_in, units_out, group_size, groups, optimization_type, batch)) ReturnFalse; //--- if(!caSSAM[0].Init(0, 0, OpenCL, units_out, chanels_out, heads, dimension_k, true, optimization, iBatch)) ReturnFalse; if(!caSSAM[1].Init(0, 0, OpenCL, chanels_out, units_out, heads, dimension_k, false, optimization, iBatch)) ReturnFalse; //--- CBufferFloat* temp = caBottleneck[0].getGradient(); if(!caBottleneck[0].SetGradient(caSSAM[0].getGradient(), false)) ReturnFalse; if(!caSSAM[0].SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!caTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; if(!caBottleneck[0].FeedForward(NeuronOCL)) ReturnFalse; if(!caBottleneck[1].FeedForward(caTranspose[0].AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) { if(!caSSAM[i].FeedForward(caBottleneck[i].AsObject())) ReturnFalse; } if(!caTranspose[1].FeedForward(caSSAM[1].AsObject())) ReturnFalse; if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = cG[0]; if(!current) ReturnFalse; if(!SumAndNormilize(caSSAM[0].getOutput(), caTranspose[1].getOutput(), current.getOutput(), caTranspose[1].GetCount(), false, 0, 0, 0, 1) || !SumAndNormilize(current.getOutput(), cResidual.getOutput(), current.getOutput(), caTranspose[1].GetCount(), false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.FeedForward(cG[i - 1])) ReturnFalse; } //--- return CNeuronSpikeActivation::feedForward(cG[-1]); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeActivation::calcInputGradients(cG[-1])) ReturnFalse; CNeuronBaseOCL* current = NULL; for(int i = cG.Total() - 2; i >= 0; i--) { current = cG[i]; if(!current || !current.CalcHiddenGradients(cG[i + 1])) ReturnFalse; } //--- if(!caSSAM[1].CalcHiddenGradients(caTranspose[1].AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!caBottleneck[i].CalcHiddenGradients(caSSAM[i].AsObject())) ReturnFalse; if(!caTranspose[0].CalcHiddenGradients(caBottleneck[1].AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(caTranspose[0].getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(caBottleneck[0].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, caTranspose[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(caTranspose[0].AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, caTranspose[0].GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!caBottleneck[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!caBottleneck[1].UpdateInputWeights(caTranspose[0].AsObject())) ReturnFalse; for(int i = 0; i < 2; i++) if(!caSSAM[i].UpdateInputWeights(caBottleneck[i].AsObject())) ReturnFalse; if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.UpdateInputWeights(cG[i - 1])) ReturnFalse; } //--- return CNeuronSpikeActivation::updateInputWeights(cG[-1]); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::Save(const int file_handle) { if(!CNeuronSpikeResNeXtBlock::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSAM.Size(); i++) if(!caSSAM[i].Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::Load(const int file_handle) { if(!CNeuronSpikeResNeXtBlock::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < caSSAM.Size(); i++) if(!LoadInsideLayer(file_handle, caSSAM[i].AsObject())) ReturnFalse; //--- CBufferFloat* temp = caBottleneck[0].getGradient(); if(!caBottleneck[0].SetGradient(caSSAM[0].getGradient(), false)) ReturnFalse; if(!caSSAM[0].SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeResNeXtBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSSAMResNeXtBlock* Source = source; for(uint i = 0; i < caSSAM.Size(); i++) if(!caSSAM[i].WeightsUpdate(Source.caSSAM[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSSAMResNeXtBlock::Clear(void) { if(!CNeuronSpikeResNeXtBlock::Clear()) ReturnFalse; //--- for(uint i = 0; i < caSSAM.Size(); i++) if(!caSSAM[i].Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSAMResNeXtBlock::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeResNeXtBlock::SetOpenCL(obj); //--- for(uint i = 0; i < caSSAM.Size(); i++) caSSAM[i].SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSSAMResNeXtBlock::TrainMode(bool flag) { CNeuronSpikeResNeXtBlock::TrainMode(flag); //--- for(uint i = 0; i < caSSAM.Size(); i++) caSSAM[i].TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTFS : public CNeuronBaseOCL { protected: CNeuronBaseOCL cMaskTime; CNeuronBaseOCL cMaskSpatial; CNeuronBaseOCL cConcatenated; CNeuronConvOCL cSpatialDScale; CNeuronConvOCL cTimeDScale; CNeuronTransposeVRCOCL caTranspose[2]; CBufferFloat cOne; //--- virtual bool STFS(CNeuronBaseOCL *NeuronOCL); virtual bool STFSGrad(CNeuronBaseOCL *NeuronOCL); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTFS(void) {}; ~CNeuronSTFS(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSTFS; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, chanels_out * units_out, optimization_type, batch)) ReturnFalse; activation = None; //--- int index = 0; if(!cMaskTime.Init(0, index, OpenCL, units_in, optimization, iBatch)) ReturnFalse; cMaskTime.SetActivationFunction(None); index++; if(!cMaskSpatial.Init(0, index, OpenCL, chanels_in, optimization, iBatch)) ReturnFalse; cMaskSpatial.SetActivationFunction(None); index++; if(!cConcatenated.Init(0, index, OpenCL, chanels_in * units_in * 5, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); if(!cSpatialDScale.Init(0, index, OpenCL, chanels_in, chanels_in, chanels_out, units_in, 5, optimization, iBatch)) ReturnFalse; cSpatialDScale.SetActivationFunction(SoftPlus); index++; if(!cTimeDScale.Init(0, index, OpenCL, units_in, units_in, units_out, chanels_out, 5, optimization, iBatch)) ReturnFalse; cTimeDScale.SetActivationFunction(TANH); index++; if(!caTranspose[0].Init(0, index, OpenCL, 5, units_in, chanels_out, optimization, iBatch)) ReturnFalse; if(!caTranspose[1].Init(0, index, OpenCL, 5, chanels_out, units_out, optimization, iBatch)) ReturnFalse; if(!cOne.BufferInit(5, 1) || !cOne.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!STFS(NeuronOCL)) ReturnFalse; if(!cSpatialDScale.FeedForward(cConcatenated.AsObject())) ReturnFalse; if(!caTranspose[0].FeedForward(cSpatialDScale.AsObject())) ReturnFalse; if(!cTimeDScale.FeedForward(caTranspose[0].AsObject())) ReturnFalse; if(!caTranspose[1].FeedForward(cTimeDScale.AsObject())) ReturnFalse; if(!DeConcat(Output, caTranspose[1].getPrevOutput(), caTranspose[1].getOutput(), Neurons(), caTranspose[1].Neurons() - Neurons(), 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::STFS(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; if(NeuronOCL.Neurons() < (cMaskSpatial.Neurons()*cMaskTime.Neurons())) ReturnFalse; //--- Time mask { int total = cMaskTime.Neurons(); vector vec = vector::Zeros(total); for(int i = 0; i < total / 2; i++) { int id = RND(total); vec[id] = 1; } CBufferFloat* out = cMaskTime.getOutput(); if(!out || !out.Fill(vec)) ReturnFalse; } //--- Spatial msak { int total = cMaskSpatial.Neurons(); vector vec = vector::Zeros(total); for(int i = 0; i < total / 2; i++) { int id = RND(total); vec[id] = 1; } CBufferFloat* out = cMaskSpatial.getOutput(); if(!out || !out.Fill(vec)) ReturnFalse; } //--- uint global_work_offset[3] = { 0, 0, 0 }; uint global_work_size[3] = { cMaskTime.Neurons(), cMaskSpatial.Neurons(), 5 }; uint local_work_size[3] = { 1, 1, global_work_size[2] }; uint kernel = def_k_STFS; setBuffer(kernel, def_k_stfs_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_stfs_mask_time, cMaskTime.getOutputIndex()) setBuffer(kernel, def_k_stfs_mask_spatial, cMaskSpatial.getOutputIndex()) setBuffer(kernel, def_k_stfs_outputs, cConcatenated.getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::STFSGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; if(NeuronOCL.Neurons() < (cMaskSpatial.Neurons()*cMaskTime.Neurons())) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = { cMaskTime.Neurons(), cMaskSpatial.Neurons(), 5 }; uint local_work_size[3] = { 1, 1, 5 }; uint kernel = def_k_STFSGrad; setBuffer(kernel, def_k_stfs_inputs, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_stfs_mask_time, cMaskTime.getOutputIndex()) setBuffer(kernel, def_k_stfs_mask_spatial, cMaskSpatial.getOutputIndex()) setBuffer(kernel, def_k_stfs_outputs, cConcatenated.getGradientIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!SumAndNormilize(Output, Gradient, PrevOutput, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(GetPointer(cOne), PrevOutput, caTranspose[1].getGradient(), cOne.Total(), 1, Neurons(), 1, false)) ReturnFalse; float error = 1; if(!caTranspose[1].calcOutputGradients(caTranspose[1].getGradient(), error)) ReturnFalse; if(!cTimeDScale.CalcHiddenGradients(caTranspose[1].AsObject())) ReturnFalse; if(!caTranspose[0].CalcHiddenGradients(cTimeDScale.AsObject())) ReturnFalse; if(!cSpatialDScale.CalcHiddenGradients(caTranspose[0].AsObject())) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cSpatialDScale.AsObject())) ReturnFalse; if(!STFSGrad(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cSpatialDScale.UpdateInputWeights(cConcatenated.AsObject())) ReturnFalse; if(!cTimeDScale.UpdateInputWeights(caTranspose[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTFS* Source = source; if(!cSpatialDScale.WeightsUpdate(Source.cSpatialDScale.AsObject(), tau)) ReturnFalse; if(!cTimeDScale.WeightsUpdate(Source.cTimeDScale.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cSpatialDScale.Save(file_handle)) ReturnFalse; if(!cTimeDScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFS::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cSpatialDScale.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTimeDScale.AsObject())) ReturnFalse; //--- uint units_in = cTimeDScale.GetWindow(); uint units_out = cTimeDScale.GetFilters(); uint chanels_in = cSpatialDScale.GetWindow(); uint chanels_out = cSpatialDScale.GetFilters(); //--- int index = 0; if(!cMaskTime.Init(0, index, OpenCL, units_in, optimization, iBatch)) ReturnFalse; cMaskTime.SetActivationFunction(None); index++; if(!cMaskSpatial.Init(0, index, OpenCL, chanels_in, optimization, iBatch)) ReturnFalse; cMaskSpatial.SetActivationFunction(None); index++; if(!cConcatenated.Init(0, index, OpenCL, chanels_in * units_in * 5, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); index += 2; if(!caTranspose[0].Init(0, index, OpenCL, 5, units_in, chanels_out, optimization, iBatch)) ReturnFalse; if(!caTranspose[1].Init(0, index, OpenCL, 5, chanels_out, units_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTFS::SetActivationFunction(ENUM_ACTIVATION value) { cTimeDScale.SetActivationFunction(value); activation = None; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTFS::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cMaskTime.SetOpenCL(OpenCL); cMaskSpatial.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cSpatialDScale.SetOpenCL(OpenCL); cTimeDScale.SetOpenCL(OpenCL); caTranspose[0].SetOpenCL(OpenCL); caTranspose[1].SetOpenCL(OpenCL); cOne.BufferFree(); cOne.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeConvGRU : public CNeuronSpikeActivation { protected: CNeuronBaseOCL cConcatenated[2]; CNeuronConvOCL cZR; CNeuronBaseOCL cZ; CNeuronBaseOCL cR; CNeuronConvOCL cHt; CNeuronBaseOCL cH; CNeuronBatchNormOCL cNorm; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeConvGRU(void) {}; ~CNeuronSpikeConvGRU(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint chanels_in, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeConvGRU; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual uint GetChanelsIn(void) const { return cHt.GetWindow() - cHt.GetFilters(); } virtual uint GetChanelsOut(void) const { return cHt.GetFilters(); } virtual uint GetUnits(void) const { return cHt.GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint chanels_in, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units * chanels_out, optimization_type, batch)) ReturnFalse; //--- uint index = 0; for(uint i = 0; i < cConcatenated.Size(); i++) { if(!cConcatenated[i].Init(0, index, OpenCL, units * (chanels_in + chanels_out), optimization, iBatch)) ReturnFalse; cConcatenated[i].SetActivationFunction(None); index++; } if(!cZR.Init(0, index, OpenCL, chanels_in + chanels_out, chanels_in + chanels_out, 2 * chanels_out, units, 1, optimization, iBatch)) ReturnFalse; cZR.SetActivationFunction(SIGMOID); index++; if(!cZ.Init(0, index, OpenCL, cZR.Neurons() / 2, optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction((ENUM_ACTIVATION)cZR.Activation()); index++; if(!cR.Init(0, index, OpenCL, cZR.Neurons() / 2, optimization, iBatch)) ReturnFalse; cR.SetActivationFunction((ENUM_ACTIVATION)cZR.Activation()); index++; if(!cHt.Init(0, index, OpenCL, chanels_in + chanels_out, chanels_in + chanels_out, chanels_out, units, 1, optimization, iBatch)) ReturnFalse; cHt.SetActivationFunction(TANH); index++; if(!cH.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cNorm.Init(0, index, OpenCL, Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cHt.GetUnits(); uint hidden = cHt.GetFilters(); uint inputs = cHt.GetWindow() - hidden; //--- if(!cH.SwapOutputs()) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cH.getPrevOutput(), cConcatenated[0].getOutput(), inputs, hidden, units)) ReturnFalse; if(!cZR.FeedForward(cConcatenated[0].AsObject())) ReturnFalse; if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZR.getOutput(), hidden, hidden, units)) ReturnFalse; if(!ElementMult(cR.getOutput(), cH.getPrevOutput(), cR.getPrevOutput())) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cR.getPrevOutput(), cConcatenated[1].getOutput(), inputs, hidden, units)) ReturnFalse; if(!cHt.FeedForward(cConcatenated[1].AsObject())) ReturnFalse; if(!GateElementMult(cHt.getOutput(), cH.getPrevOutput(), cZ.getOutput(), cH.getOutput())) ReturnFalse; //--- if(!cNorm.FeedForward(cH.AsObject())) ReturnFalse; if(!CNeuronSpikeActivation::feedForward(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cHt.GetUnits(); uint hidden = cHt.GetFilters(); uint inputs = cHt.GetWindow() - hidden; //--- if(!CNeuronSpikeActivation::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!cH.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!GateElementMultGrad(cHt.getOutput(), cHt.getGradient(), cH.getPrevOutput(), cHt.getPrevOutput(), cZ.getOutput(), cZ.getGradient(), cH.getGradient(), cHt.Activation(), cH.Activation(), cZR.Activation())) ReturnFalse; if(!cConcatenated[1].CalcHiddenGradients(cHt.AsObject())) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cR.getPrevOutput(), cConcatenated[1].getGradient(), inputs, hidden, units)) ReturnFalse; if(!ElementMultGrad(cR.getOutput(), cR.getGradient(), cH.getPrevOutput(), cHt.getPrevOutput(), cR.getPrevOutput(), cZR.Activation(), cH.Activation())) ReturnFalse; if(!Concat(cZ.getGradient(), cR.getGradient(), cZR.getGradient(), hidden, hidden, units)) ReturnFalse; if(!cConcatenated[0].CalcHiddenGradients(cZR.AsObject())) ReturnFalse; if(!DeConcat(cConcatenated[0].getPrevOutput(), cH.getPrevOutput(), cConcatenated[0].getGradient(), inputs, hidden, units)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cConcatenated[0].getPrevOutput(), NeuronOCL.getGradient(), inputs, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cZR.UpdateInputWeights(cConcatenated[0].AsObject())) ReturnFalse; if(!cHt.UpdateInputWeights(cConcatenated[1].AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cH.AsObject())) ReturnFalse; if(!CNeuronSpikeActivation::updateInputWeights(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeConvGRU* Source = source; if(!cZR.WeightsUpdate(Source.cZR.AsObject(), tau)) ReturnFalse; if(!cHt.WeightsUpdate(Source.cHt.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; if(!cZR.Save(file_handle)) ReturnFalse; if(!cHt.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cZR.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHt.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; //--- uint units = cHt.GetUnits(); uint chanels_out = cHt.GetFilters(); uint chanels_in = cHt.GetWindow() - chanels_out; //--- uint index = 0; for(uint i = 0; i < cConcatenated.Size(); i++) { if(!cConcatenated[i].Init(0, index, OpenCL, units * (chanels_in + chanels_out), optimization, iBatch)) ReturnFalse; cConcatenated[i].SetActivationFunction(None); index++; } index += 2; if(!cZ.Init(0, index, OpenCL, cZR.Neurons() / 2, optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction((ENUM_ACTIVATION)cZR.Activation()); index++; if(!cR.Init(0, index, OpenCL, cZR.Neurons() / 2, optimization, iBatch)) ReturnFalse; cR.SetActivationFunction((ENUM_ACTIVATION)cZR.Activation()); index += 2; if(!cH.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeConvGRU::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); cZR.SetOpenCL(OpenCL); cHt.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); for(uint i = 0; i < cConcatenated.Size(); i++) cConcatenated[i].SetOpenCL(OpenCL); cZ.SetOpenCL(OpenCL); cR.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; if(!cH.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAddToStack : public CNeuronBaseOCL { protected: uint iStackSize; uint iDimension; uint iVariables; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAddToStack(void) : iStackSize(0), iDimension(0), iVariables(0) {}; ~CNeuronAddToStack(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronAddToStack; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; //--- virtual uint GetStackSize(void) const { return iStackSize; } virtual uint GetDimension(void) const { return iDimension; } virtual uint GetVariables(void) const { return iVariables; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAddToStack::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, stack_size * dimension * variables, optimization_type, batch)) ReturnFalse; //--- activation = None; iStackSize = stack_size; iDimension = dimension; iVariables = variables; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAddToStack::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- if(NeuronOCL.Neurons() < int(iDimension * iVariables)) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = { iDimension, MathMin((uint)OpenCL.GetMaxLocalSize(1), iStackSize), iVariables }; uint local_work_size[3] = { 1, global_work_size[1], 1 }; uint kernel = def_k_AddToStack; setBuffer(kernel, def_k_ats_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_ats_stack, getOutputIndex()) setArgument(kernel, def_k_ats_stack_size, (int)iStackSize) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAddToStack::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; if(NeuronOCL.Neurons() < int(iDimension * iVariables)) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), PrevOutput, Gradient, iDimension, iDimension * (iStackSize - 1), iVariables)) ReturnFalse; Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAddToStack::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, (int)iStackSize) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iDimension) < INT_VALUE) ReturnFalse; if(FileWriteInteger(file_handle, (int)iVariables) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAddToStack::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iStackSize = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iDimension = (uint)FileReadInteger(file_handle); if(FileIsEnding(file_handle)) ReturnFalse; iVariables = (uint)FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronStackCorrelation : public CNeuronBaseOCL { protected: CNeuronAddToStack cStack; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronStackCorrelation(void) {}; ~CNeuronStackCorrelation(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronStackCorrelation; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; //--- virtual uint GetStackSize(void) const { return cStack.GetStackSize(); } virtual uint GetDimension(void) const { return cStack.GetDimension(); } virtual uint GetVariables(void) const { return cStack.GetVariables(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, variables * stack_size, optimization_type, batch)) ReturnFalse; if(!cStack.Init(0, 0, OpenCL, stack_size, dimension, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cStack.FeedForward(NeuronOCL)) ReturnFalse; if(!MatMul(cStack.getOutput(), NeuronOCL.getOutput(), Output, cStack.GetStackSize(), cStack.GetDimension(), 1, cStack.GetVariables(), true)) ReturnFalse; if(activation != None) if(!Activation(Output, Output, activation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cStack.getOutput(), cStack.getGradient(), NeuronOCL.getOutput(), cStack.getPrevOutput(), Gradient, cStack.GetStackSize(), cStack.GetDimension(), 1, cStack.GetVariables(), true)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cStack.getPrevOutput(), cStack.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cStack.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cStack.getPrevOutput(), NeuronOCL.getGradient(), cStack.GetDimension(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronStackCorrelation::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cStack.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronStackCorrelation::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTEFlowNetEncoder : public CNeuronSpikeConvGRU { protected: CLayer cRecurrentLine; CLayer cFlow; CLayer cCorrelation; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTEFlowNetEncoder(void) {}; ~CNeuronSTEFlowNetEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual CNeuronBaseOCL* GetLayer(uint layer); virtual uint Layers(void) const { return (cRecurrentLine.Total() + 1) / 3; } //-- virtual int Type(void) override const { return defNeuronSTEFlowNetEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels.Size() != units.Size()) ReturnFalse; //--- const uint layers = chanels.Size() - 1; if(!CNeuronSpikeConvGRU::Init(numOutputs, myIndex, open_cl, units[layers], chanels[layers] + stack_size, chanels[layers], optimization_type, batch)) ReturnFalse; //--- CNeuronSSAMResNeXtBlock* conv_res = NULL; CNeuronStackCorrelation* correl = NULL; CNeuronSpikeConvGRU* gru = NULL; CNeuronBaseOCL* neuron = NULL; //--- cRecurrentLine.Clear(); cFlow.Clear(); cCorrelation.Clear(); cRecurrentLine.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); //--- uint index = 0; for(uint i = 0; i < layers; i++) { conv_res = new CNeuronSSAMResNeXtBlock(); if(!conv_res || !conv_res.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, heads, dimension_k, optimization, iBatch) || !cRecurrentLine.Add(conv_res)) { DeleteObj(conv_res) ReturnFalse; } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, units[i + 1] * (chanels[i + 1]*stack_size), optimization, iBatch) || !cRecurrentLine.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } if(i == layers) break; //--- if(i < layers - 1) { index++; gru = new CNeuronSpikeConvGRU(); if(!gru || !gru.Init(0, index, OpenCL, units[i + 1], chanels[i + 1] + stack_size, chanels[i + 1], optimization, iBatch) || !cRecurrentLine.Add(gru)) { DeleteObj(gru) ReturnFalse; } } //--- index++; correl = new CNeuronStackCorrelation(); if(!correl || !correl.Init(0, index, OpenCL, stack_size, chanels[i + 1], units[i + 1], optimization, iBatch) || !cCorrelation.Add(correl)) { DeleteObj(correl) ReturnFalse; } //--- if(i == 0) continue; index++; conv_res = new CNeuronSSAMResNeXtBlock(); if(!conv_res || !conv_res.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, heads, dimension_k, optimization, iBatch) || !cFlow.Add(conv_res)) { DeleteObj(conv_res) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *recur = NeuronOCL; CNeuronBaseOCL *flow = cRecurrentLine[0]; CNeuronStackCorrelation *correl = NULL; CNeuronSSAMResNeXtBlock* resnext = NULL; int layers = (int)Layers(); //--- for(int i = 0; i < layers; i++) { //--- ResNeXt if(!cRecurrentLine[i * 3] || !cRecurrentLine[i * 3].FeedForward(recur)) ReturnFalse; recur = cRecurrentLine[i * 3]; //--- Correlation correl = cCorrelation[i]; if(!correl || !correl.FeedForward(flow)) ReturnFalse; //--- Concatenate resnext = recur; recur = cRecurrentLine[i * 3 + 1]; if(!recur || !Concat(resnext.getOutput(), correl.getOutput(), recur.getOutput(), resnext.GetChannelsOut(), correl.GetStackSize(), resnext.GetUnitsOut())) ReturnFalse; if((i * 3 + 2) == cRecurrentLine.Total()) break; //--- GRU if(!cRecurrentLine[i * 3 + 1] || !cRecurrentLine[i * 3 + 2].FeedForward(recur)) ReturnFalse; recur = cRecurrentLine[i * 3 + 1]; if(!cFlow[i] || !cFlow[i].FeedForward(flow)) ReturnFalse; flow = cFlow[i]; } //--- return CNeuronSpikeConvGRU::feedForward(recur); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL *recur = NeuronOCL; CNeuronBaseOCL *flow = cRecurrentLine[0]; CNeuronStackCorrelation *correl = NULL; CNeuronSSAMResNeXtBlock* resnext = NULL; int layers = (int)Layers(); //--- for(int i = 0; i < layers; i++) { //--- ResNeXt if(!cRecurrentLine[i * 3] || !cRecurrentLine[i * 3].UpdateInputWeights(recur)) ReturnFalse; recur = cRecurrentLine[i * 3]; //--- Correlation correl = cCorrelation[i]; if(!correl || !correl.UpdateInputWeights(flow)) ReturnFalse; //--- Concatenate recur = cRecurrentLine[i * 3 + 1]; if((i * 3 + 2) == cRecurrentLine.Total()) break; //--- GRU if(!cRecurrentLine[i * 3 + 1] || !cRecurrentLine[i * 3 + 2].UpdateInputWeights(recur)) ReturnFalse; recur = cRecurrentLine[i * 3 + 1]; if(!cFlow[i] || !cFlow[i].UpdateInputWeights(flow)) ReturnFalse; flow = cFlow[i]; } //--- return CNeuronSpikeConvGRU::updateInputWeights(recur); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvGRU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTEFlowNetEncoder* Source = source; if(!cRecurrentLine.WeightsUpdate(Source.cRecurrentLine.AsObject(), tau)) ReturnFalse; if(!cCorrelation.WeightsUpdate(Source.cCorrelation.AsObject(), tau)) ReturnFalse; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvGRU::calcInputGradients(cRecurrentLine[-1])) ReturnFalse; //--- CNeuronBaseOCL *concat = cRecurrentLine[-1]; CNeuronBaseOCL *flow = cFlow[-1]; CNeuronStackCorrelation *correl = cCorrelation[-1]; CNeuronSSAMResNeXtBlock* resnext = cRecurrentLine[-2]; int layers = (int)Layers() - 1; //--- for(int i = layers; i >= 0; i--) { //--- Concatenate if(!resnext || !concat || !correl || !DeConcat(resnext.getGradient(), correl.getGradient(), concat.getGradient(), resnext.GetChannelsOut(), correl.GetStackSize(), resnext.GetUnitsOut())) ReturnFalse; if(i == 0) { CBufferFloat* temp = resnext.getGradient(); //--- Correlation if(!resnext.SetGradient(resnext.getPrevOutput(), false) || !resnext.CalcHiddenGradients(correl) || !SumAndNormilize(temp, resnext.getGradient(), temp, resnext.GetChannelsOut(), false, 0, 0, 0, 1)) ReturnFalse; //--- Flow if(i < layers) if(!resnext.CalcHiddenGradients(cFlow[i]) || !SumAndNormilize(temp, resnext.getGradient(), temp, 1, false, 0, 0, 0, 1)) ReturnFalse; if(!resnext.SetGradient(temp, false)) ReturnFalse; //--- ResNeXt if(!NeuronOCL.CalcHiddenGradients(resnext)) ReturnFalse; } else { //--- Correlation if(!flow || !flow.CalcHiddenGradients(correl)) ReturnFalse; //--- Flow if(i < layers) { CBufferFloat* temp = flow.getGradient(); if(!flow.SetGradient(flow.getPrevOutput(), false) || !flow.CalcHiddenGradients(cFlow[i]) || !SumAndNormilize(temp, flow.getGradient(), temp, 1, false, 0, 0, 0, 1) || !flow.SetGradient(temp, false)) ReturnFalse; } //--- ResNeXt if(!cRecurrentLine[i * 3 - 1] || !cRecurrentLine[i * 3 - 1].CalcHiddenGradients(resnext)) ReturnFalse; //--- concat = cRecurrentLine[(i - 1) * 3 + 1]; flow = cFlow[i - 2]; correl = cCorrelation[i - 1]; resnext = cRecurrentLine[(i - 1) * 3]; //--- GRU if(!concat || !concat.CalcHiddenGradients(cRecurrentLine[i * 3 - 1])) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::Save(const int file_handle) { if(!CNeuronSpikeConvGRU::Save(file_handle)) ReturnFalse; //--- if(!cRecurrentLine.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cCorrelation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::Load(const int file_handle) { if(!CNeuronSpikeConvGRU::Load(file_handle)) ReturnFalse; //--- if(!cRecurrentLine.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; if(!cCorrelation.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetEncoder::Clear(void) { if(!CNeuronSpikeConvGRU::Clear()) ReturnFalse; //--- CNeuronBaseOCL* neuron = NULL; for(int i = 0; i < cRecurrentLine.Total(); i++) { neuron = cRecurrentLine[i]; if(!neuron || !neuron.Clear()) ReturnFalse; } for(int i = 0; i < cCorrelation.Total(); i++) { neuron = cCorrelation[i]; if(!neuron || !neuron.Clear()) ReturnFalse; } for(int i = 0; i < cFlow.Total(); i++) { neuron = cFlow[i]; if(!neuron || !neuron.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvGRU::SetOpenCL(obj); //--- cRecurrentLine.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetEncoder::TrainMode(bool flag) { CNeuronSpikeConvGRU::TrainMode(flag); //--- cRecurrentLine.TrainMode(bTrain); cFlow.TrainMode(bTrain); cCorrelation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CNeuronBaseOCL* CNeuronSTEFlowNetEncoder::GetLayer(uint layer) { if(layer >= Layers()) return NULL; if(layer == (Layers() - 1)) return GetPointer(this); //--- return cRecurrentLine[layer * 3 + 2]; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTEFlowNetResidualBlock : public CNeuronBaseOCL { protected: CLayer cBlock; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTEFlowNetResidualBlock(void) {}; ~CNeuronSTEFlowNetResidualBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels, uint units, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch); //-- virtual int Type(void) override const { return defNeuronSTEFlowNetResidualBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels, uint units, uint group_size, uint groups, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, 2 * chanels * units, optimization_type, batch)) ReturnFalse; activation = None; //--- CNeuronSpikeResNeXtBottleneck* conv = NULL; CNeuronSpikeResNeXtBlock* residual = NULL; //--- cBlock.Clear(); cBlock.SetOpenCL(OpenCL); //--- uint index = 0; conv = new CNeuronSpikeResNeXtBottleneck(); if(!conv || !conv.Init(0, index, OpenCL, chanels, chanels, units, units, group_size, groups, optimization, iBatch) || !cBlock.Add(conv)) { DeleteObj(conv) ReturnFalse; } for(int i = 0; i < 2; i++) { index++; residual = new CNeuronSpikeResNeXtBlock(); if(!residual || !residual.Init(0, index, OpenCL, chanels, chanels, units, units, group_size, groups, optimization, iBatch) || !cBlock.Add(residual)) { DeleteObj(residual) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(cBlock.Total() <= 0) ReturnFalse; CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; for(int i = 0; i < cBlock.Total(); i++) { current = cBlock[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- if(!Concat(prev.getOutput(), NeuronOCL.getOutput(), Output, 1, 1, Neurons() / 2)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(cBlock.Total() <= 0) ReturnFalse; CNeuronBaseOCL* current = cBlock[-1]; CNeuronBaseOCL* next = NULL; //--- if(!DeConcat(current.getGradient(), PrevOutput, Gradient, 1, 1, Neurons() / 2)) ReturnFalse; next = current; for(int i = cBlock.Total() - 2; i >= 0; i--) { current = cBlock[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; next = current; } //--- if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, PrevOutput, NeuronOCL.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(next) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(cBlock.Total() <= 0) ReturnFalse; CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* current = NULL; for(int i = 0; i < cBlock.Total(); i++) { current = cBlock[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTEFlowNetResidualBlock* Source = source; if(!cBlock.WeightsUpdate(Source.cBlock.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::Clear(void) { for(int i = 0; i < cBlock.Total(); i++) { if(!cBlock[i] || !cBlock[i].Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetResidualBlock::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cBlock.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetResidualBlock::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cBlock.TrainMode(flag); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cBlock.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetResidualBlock::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cBlock.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTEFlowNetDecoder : public CNeuronSpikeActivation { protected: CLayer cMainLine; CLayer cFlow; CNeuronSTEFlowNetEncoder* cEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTEFlowNetDecoder(void) {}; ~CNeuronSTEFlowNetDecoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, CNeuronSTEFlowNetEncoder* encoder, ENUM_OPTIMIZATION optimization_type, uint batch); //-- virtual int Type(void) override const { return defNeuronSTEFlowNetDecoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetEncoder(CNeuronSTEFlowNetEncoder* encoder) { cEncoder = encoder; } //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, CNeuronSTEFlowNetEncoder *encoder, ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels.Size() != units.Size()) ReturnFalse; //--- const uint layers = chanels.Size() - 1; if(layers < 1) ReturnFalse; if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units[layers]*chanels[layers], optimization_type, batch)) ReturnFalse; cEncoder = encoder; uint enc_layers = (!cEncoder ? 0 : cEncoder.Layers()); //--- CNeuronSSAMResNeXtBlock* conv_res = NULL; CNeuronConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeActivation* sa = NULL; CNeuronSpikeConvGRU* gru = NULL; CNeuronBaseOCL* concat = NULL; //--- cMainLine.Clear(); cFlow.Clear(); cMainLine.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- uint index = 0; //--- Main Line conv_res = new CNeuronSSAMResNeXtBlock(); if(!conv_res || !conv_res.Init(0, index, OpenCL, chanels[0], chanels[1], units[0], units[1], group_size, groups, heads, dimension_k, optimization, iBatch) || !cMainLine.Add(conv_res)) { DeleteObj(conv_res) ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, chanels[1], chanels[1], chanels[1], units[1], 1, optimization, iBatch) || !cMainLine.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMainLine.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); uint ch_in = chanels[0]; //--- for(uint i = 1; i < layers; i++) { //--- Main Line index++; sa = new CNeuronSpikeActivation(); if(!sa || !sa.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cMainLine.Add(sa)) { DeleteObj(sa) ReturnFalse; } //--- Flow index++; conv_res = new CNeuronSSAMResNeXtBlock(); if(!conv_res || !conv_res.Init(0, index, OpenCL, ch_in, chanels[i], units[i - 1], units[i], group_size, groups, heads, dimension_k, optimization, iBatch) || !cFlow.Add(conv_res)) { DeleteObj(conv_res) ReturnFalse; } index++; ch_in = 2 * chanels[i]; if(enc_layers - i - 1 >= 0) { gru = cEncoder.GetLayer(enc_layers - i - 1); if(!gru) ReturnFalse; if(gru.GetUnits() != units[i]) ReturnFalse; ch_in += gru.GetChanelsOut(); } concat = new CNeuronBaseOCL(); if(!concat || !concat.Init(0, index, OpenCL, ch_in * units[i], optimization, iBatch) || !cFlow.Add(concat)) { DeleteObj(concat) ReturnFalse; } concat.SetActivationFunction(None); //--- Main Line conv_res = new CNeuronSSAMResNeXtBlock(); if(!conv_res || !conv_res.Init(0, index, OpenCL, ch_in, ch_in, units[i], units[i + 1], group_size, groups, heads, dimension_k, optimization, iBatch) || !cMainLine.Add(conv_res)) { DeleteObj(conv_res) ReturnFalse; } index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, chanels[i + 1], chanels[i + 1], chanels[i + 1], units[i + 1], 1, optimization, iBatch) || !cMainLine.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cMainLine.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Main Line CNeuronSSAMResNeXtBlock* conv_res = NULL; CNeuronConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeActivation* sa = NULL; CNeuronSpikeConvGRU* gru = NULL; //--- Flow CNeuronSSAMResNeXtBlock* flow = NULL; CNeuronBaseOCL* concat = NULL; //--- CNeuronBaseOCL* prev = NeuronOCL; int layers = (cMainLine.Total() + 1) / 4; int enc_layers = int(!cEncoder ? 0 : cEncoder.Layers()); int enc_dim = 0; //--- for(int i = 0; i < layers; i++) { conv_res = cMainLine[i * 4]; conv = cMainLine[i * 4 + 1]; norm = cMainLine[i * 4 + 2]; sa = cMainLine[i * 4 + 3]; if(enc_layers - i - 2 >= 0) { gru = cEncoder.GetLayer(uint(enc_layers - i - 2)); if(!gru) ReturnFalse; enc_dim = (int)gru.GetChanelsOut(); } else { gru = NULL; enc_dim = 0; } flow = cFlow[i * 2]; concat = cFlow[i * 2 + 1]; //--- Main Line if(!conv_res || !conv_res.FeedForward(prev)) ReturnFalse; if(!conv || !conv.FeedForward(conv_res)) ReturnFalse; if(!norm || !norm.FeedForward(conv)) ReturnFalse; if(i == (layers - 1)) break; if(!sa || !sa.FeedForward(norm)) ReturnFalse; //--- Flow if(!flow || !flow.FeedForward(prev)) ReturnFalse; if(!concat) ReturnFalse; if(!gru) { if(!Concat(sa.getOutput(), flow.getOutput(), concat.getOutput(), conv.GetFilters(), flow.GetChannelsOut(), conv.GetUnits())) ReturnFalse; } else { if(conv.GetUnits() != gru.GetUnits()) ReturnFalse; if(!Concat(sa.getOutput(), flow.getOutput(), gru.getOutput(), concat.getOutput(), conv.GetFilters(), flow.GetChannelsOut(), enc_dim, conv.GetUnits())) ReturnFalse; } prev = concat; } //--- if(!CNeuronSpikeActivation::feedForward(norm)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Main Line CNeuronSSAMResNeXtBlock* conv_res = NULL; CNeuronConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeActivation* sa = NULL; CNeuronSpikeConvGRU* gru = NULL; //--- Flow CNeuronSSAMResNeXtBlock* flow = NULL; CNeuronBaseOCL* concat = NULL; //--- CNeuronBaseOCL* prev = NeuronOCL; int layers = (cMainLine.Total() + 1) / 4; int enc_layers = int(!cEncoder ? 0 : cEncoder.Layers()); int enc_dim = 0; //--- for(int i = 0; i < layers; i++) { conv_res = cMainLine[i * 4]; conv = cMainLine[i * 4 + 1]; norm = cMainLine[i * 4 + 2]; sa = cMainLine[i * 4 + 3]; if(enc_layers - i >= 0) { gru = cEncoder.GetLayer(uint(enc_layers - i - 1)); if(!gru) ReturnFalse; enc_dim = (int)gru.GetChanelsOut(); } else { gru = NULL; enc_dim = 0; } flow = cFlow[i * 2]; concat = cFlow[i * 2 + 1]; //--- Main Line if(!conv_res || !conv_res.UpdateInputWeights(prev)) ReturnFalse; if(!conv || !conv.UpdateInputWeights(conv_res)) ReturnFalse; if(!norm || !norm.UpdateInputWeights(conv)) ReturnFalse; if(i == (layers - 1)) break; if(!sa || !sa.UpdateInputWeights(norm)) ReturnFalse; //--- Flow if(!flow || !flow.UpdateInputWeights(prev)) ReturnFalse; prev = concat; } //--- if(!CNeuronSpikeActivation::updateInputWeights(norm)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { //--- Main Line CNeuronSSAMResNeXtBlock* conv_res = cMainLine[-3]; CNeuronConvOCL* conv = cMainLine[-2]; CNeuronBatchNormOCL* norm = cMainLine[-1]; CNeuronSpikeActivation* sa = cMainLine[-4]; CNeuronSpikeConvGRU* gru = NULL; //--- Flow CNeuronSSAMResNeXtBlock* flow = cFlow[-2]; //--- int layers = cMainLine.Total() / 4; CNeuronBaseOCL* prev = (layers > 1 ? cFlow[-1] : NeuronOCL); int enc_layers = int(!cEncoder ? 0 : cEncoder.Layers()); int enc_dim = 0; //--- if(!CNeuronSpikeActivation::calcInputGradients(norm)) ReturnFalse; if(!conv || !conv.CalcHiddenGradients(norm)) ReturnFalse; if(!conv_res || !conv_res.CalcHiddenGradients(conv)) ReturnFalse; if(!prev || !prev.CalcHiddenGradients(conv_res)) ReturnFalse; //--- for(int i = layers - 1; i >= 0; i--) { if(enc_layers - i - 2 >= 0) { gru = cEncoder.GetLayer(uint(enc_layers - i - 1)); if(!gru) ReturnFalse; enc_dim = (int)gru.GetChanelsOut(); } else { gru = NULL; enc_dim = 0; } if(!sa || !flow) ReturnFalse; if(!gru) { if(!DeConcat(sa.getGradient(), flow.getGradient(), prev.getGradient(), conv.GetFilters(), flow.GetChannelsOut(), conv.GetUnits())) ReturnFalse; } else if(!DeConcat(sa.getGradient(), flow.getGradient(), gru.getGradient(), prev.getGradient(), conv.GetFilters(), flow.GetChannelsOut(), enc_dim, conv.GetUnits())) ReturnFalse; //--- Flow prev = (i > 0 ? cFlow[i * 2 - 1] : NeuronOCL); if(!prev || !prev.CalcHiddenGradients(flow)) ReturnFalse; //--- Main Line conv_res = cMainLine[i * 4]; conv = cMainLine[i * 4 + 1]; norm = cMainLine[i * 4 + 2]; if(!norm.CalcHiddenGradients(sa)) ReturnFalse; if(!conv || !conv.CalcHiddenGradients(norm)) ReturnFalse; if(!conv_res || !conv_res.CalcHiddenGradients(conv)) ReturnFalse; CBufferFloat* temp = prev.getGradient(); if(!prev.SetGradient(prev.getPrevOutput(), false) || !prev.CalcHiddenGradients(conv_res) || !SumAndNormilize(temp, prev.getGradient(), temp, 1, false, 0, 0, 0, 1) || !prev.SetGradient(temp, false)) ReturnFalse; if(i == 0) continue; sa = cMainLine[i * 4 - 1]; //--- Flow flow = cFlow[(i - 1) * 2]; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; if(!cMainLine.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; if(!cMainLine.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTEFlowNetDecoder* Source = source; if(!cMainLine.WeightsUpdate(Source.cMainLine.AsObject(), tau)) ReturnFalse; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNetDecoder::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; //--- CNeuronBaseOCL* neuron = NULL; for(int i = 0; i < cMainLine.Total(); i++) { neuron = cMainLine[i]; if(!neuron || !neuron.Clear()) ReturnFalse; } for(int i = 0; i < cFlow.Total(); i++) { neuron = cFlow[i]; if(!neuron || !neuron.Clear()) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetDecoder::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); cMainLine.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNetDecoder::TrainMode(bool flag) { CNeuronSpikeActivation::TrainMode(flag); cMainLine.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTEFlowNet : public CNeuronSTEFlowNetDecoder { //--- protected: CNeuronSTEFlowNetEncoder cSTEEncoder; CNeuronSTEFlowNetResidualBlock cSTEResidualBlock; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTEFlowNet(void) {}; ~CNeuronSTEFlowNet(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch); //-- virtual int Type(void) override const { return defNeuronSTEFlowNet; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &chanels[], uint &units[], uint group_size, uint groups, uint heads, uint dimension_k, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels.Size() != units.Size()) ReturnFalse; //--- uint rev_chanels[], rev_units[]; if(ArrayResize(rev_chanels, chanels.Size()) < 0 || ArrayResize(rev_units, units.Size()) < 0) ReturnFalse; uint total = chanels.Size(); for(uint i = 0; i < total; i++) rev_chanels[total - i - 1] = chanels[i]; rev_chanels[0] *= 2; for(uint i = 0; i < total; i++) rev_units[total - i - 1] = units[i]; //--- if(!cSTEEncoder.Init(0, 0, open_cl, chanels, units, group_size, groups, heads, dimension_k, stack_size, optimization_type, batch)) ReturnFalse; if(!cSTEResidualBlock.Init(0, 1, open_cl, chanels[total - 1], units[total - 1], group_size, groups, optimization, iBatch)) ReturnFalse; rev_chanels[0] = cSTEResidualBlock.Neurons() / rev_units[0]; if(!CNeuronSTEFlowNetDecoder::Init(numOutputs, myIndex, open_cl, rev_chanels, rev_units, group_size, groups, heads, dimension_k, cSTEEncoder.AsObject(), optimization_type, batch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cSTEEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cSTEResidualBlock.FeedForward(cSTEEncoder.AsObject())) ReturnFalse; //--- return CNeuronSTEFlowNetDecoder::feedForward(cSTEResidualBlock.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cSTEEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSTEResidualBlock.UpdateInputWeights(cSTEEncoder.AsObject())) ReturnFalse; //--- return CNeuronSTEFlowNetDecoder::updateInputWeights(cSTEResidualBlock.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSTEFlowNetDecoder::calcInputGradients(cSTEResidualBlock.AsObject())) ReturnFalse; if(!cSTEEncoder.CalcHiddenGradients(cSTEResidualBlock.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSTEEncoder.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::Save(const int file_handle) { if(!CNeuronSTEFlowNetDecoder::Save(file_handle)) ReturnFalse; if(!cSTEEncoder.Save(file_handle)) ReturnFalse; if(!cSTEResidualBlock.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::Load(const int file_handle) { if(!CNeuronSTEFlowNetDecoder::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSTEEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSTEResidualBlock.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSTEFlowNetDecoder::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTEFlowNet* Source = source; if(!cSTEEncoder.WeightsUpdate(Source.cSTEEncoder.AsObject(), tau)) ReturnFalse; if(!cSTEResidualBlock.WeightsUpdate(Source.cSTEResidualBlock.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTEFlowNet::Clear(void) { if(!CNeuronSTEFlowNetDecoder::Clear()) ReturnFalse; if(!cSTEEncoder.Clear()) ReturnFalse; if(!cSTEResidualBlock.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNet::SetOpenCL(COpenCLMy *obj) { CNeuronSTEFlowNetDecoder::SetOpenCL(obj); cSTEEncoder.SetOpenCL(OpenCL); cSTEResidualBlock.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTEFlowNet::TrainMode(bool flag) { CNeuronSTEFlowNetDecoder::TrainMode(flag); cSTEEncoder.TrainMode(bTrain); cSTEResidualBlock.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleStackCorrelation : public CNeuronStackCorrelation { protected: uint iLevels; CNeuronBaseOCL cScales; //--- virtual bool AggregationByTime(CNeuronBaseOCL *NeuronOCL); virtual bool AggregationByTimeGrad(CNeuronBaseOCL *NeuronOCL); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultiScaleStackCorrelation(void) {}; ~CNeuronMultiScaleStackCorrelation(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultiScaleStackCorrelation; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual uint GetLevels(void) const { return iLevels; } virtual uint GetVariables(void) override const { return cStack.GetVariables() / iLevels; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint variables, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronStackCorrelation::Init(numOutputs, myIndex, open_cl, stack_size, dimension, variables * levels, optimization_type, batch)) ReturnFalse; iLevels = levels; if(!cScales.Init(0, 0, OpenCL, dimension * levels * variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::AggregationByTime(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- if(NeuronOCL.Neurons() < int(GetDimension()*GetVariables())) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { GetDimension(), GetVariables() }; uint kernel = def_k_AggregationByTime; setBuffer(kernel, def_k_agt_inputs, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_agt_stack, cStack.getOutputIndex()) setBuffer(kernel, def_k_agt_outputs, cScales.getOutputIndex()) setArgument(kernel, def_k_agt_stack_size, (int)cStack.GetStackSize()) setArgument(kernel, def_k_agt_levels, (int)GetLevels()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::AggregationByTimeGrad(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; //--- if(NeuronOCL.Neurons() < int(GetDimension()*GetVariables())) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { GetDimension(), GetVariables() }; uint kernel = def_k_AggregationByTimeGrad; setBuffer(kernel, def_k_agtg_inputs_gr, NeuronOCL.getGradientIndex()) setBuffer(kernel, def_k_agtg_outputs_gr, cStack.getGradientIndex()) setArgument(kernel, def_k_agtg_levels, (int)GetLevels()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!AggregationByTime(NeuronOCL)) ReturnFalse; if(!cStack.FeedForward(cScales.AsObject())) ReturnFalse; if(!MatMul(cStack.getOutput(), NeuronOCL.getOutput(), Output, GetStackSize()*GetLevels(), GetDimension(), 1, GetVariables(), true)) ReturnFalse; if(activation != None) if(!Activation(Output, Output, activation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MatMulGrad(cStack.getOutput(), cStack.getGradient(), NeuronOCL.getOutput(), cScales.getGradient(), Gradient, GetStackSize()*GetLevels(), GetDimension(), 1, GetVariables(), true)) ReturnFalse; if(!AggregationByTimeGrad(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cScales.getGradient(), NeuronOCL.getGradient(), GetDimension(), false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::Save(const int file_handle) { if(!CNeuronStackCorrelation::Save(file_handle)) ReturnFalse; if(!cScales.Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, (int)iLevels) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleStackCorrelation::Load(const int file_handle) { if(!CNeuronStackCorrelation::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cScales.AsObject())) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iLevels = FileReadInteger(file_handle); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleStackCorrelation::SetOpenCL(COpenCLMy *obj) { CNeuronStackCorrelation::SetOpenCL(obj); cScales.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeConvGRU2D : public CNeuronSpikeConvGRU { protected: CNeuronConvOCL cProjectionX; CNeuronBaseOCL cConcatenated2[2]; CNeuronConvOCL cZR2; CNeuronBaseOCL cZ2; CNeuronBaseOCL cR2; CNeuronConvOCL cHt2; CNeuronBaseOCL cH2; CNeuronTransposeOCL cTranspose[3]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeConvGRU2D(void) {}; ~CNeuronSpikeConvGRU2D(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint chanels_in, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeConvGRU2D; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint chanels_in, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvGRU::Init(numOutputs, myIndex, open_cl, units, chanels_in, chanels_out, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cProjectionX.Init(0, index, OpenCL, chanels_in, chanels_in, chanels_out, units, optimization, iBatch)) ReturnFalse; cProjectionX.SetActivationFunction(None); index++; for(uint i = 0; i < cConcatenated2.Size(); i++) { if(!cConcatenated2[i].Init(0, index, OpenCL, 2 * units * chanels_in, optimization, iBatch)) ReturnFalse; cConcatenated[i].SetActivationFunction(None); index++; } if(!cTranspose[0].Init(0, index, OpenCL, units, chanels_out, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose[1].Init(0, index, OpenCL, units, chanels_out, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose[2].Init(0, index, OpenCL, chanels_out, units, optimization, iBatch)) ReturnFalse; index++; if(!cZR2.Init(0, index, OpenCL, 2 * units, 2 * units, 2 * units, chanels_out, 1, optimization, iBatch)) ReturnFalse; cZR2.SetActivationFunction(SIGMOID); index++; if(!cZ2.Init(0, index, OpenCL, cZR2.Neurons() / 2, optimization, iBatch)) ReturnFalse; cZ2.SetActivationFunction((ENUM_ACTIVATION)cZR2.Activation()); index++; if(!cR2.Init(0, index, OpenCL, cZR2.Neurons() / 2, optimization, iBatch)) ReturnFalse; cR2.SetActivationFunction((ENUM_ACTIVATION)cZR2.Activation()); index++; if(!cHt2.Init(0, index, OpenCL, 2 * units, 2 * units, units, chanels_out, 1, optimization, iBatch)) ReturnFalse; cHt2.SetActivationFunction(TANH); index++; if(!cH2.Init(0, index, OpenCL, units * chanels_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cProjectionX.FeedForward(NeuronOCL)) ReturnFalse; if(!cTranspose[0].FeedForward(cH.AsObject())) ReturnFalse; if(!cTranspose[1].FeedForward(cProjectionX.AsObject())) ReturnFalse; //--- uint units = cHt.GetUnits(); uint hidden = cHt.GetFilters(); uint inputs = cHt.GetWindow() - hidden; //--- if(!Concat(cTranspose[0].getOutput(), cTranspose[1].getOutput(), cConcatenated2[0].getOutput(), units, units, hidden)) ReturnFalse; if(!cZR2.FeedForward(cConcatenated2[0].AsObject())) ReturnFalse; if(!DeConcat(cZ2.getOutput(), cR2.getOutput(), cZR2.getOutput(), units, units, hidden)) ReturnFalse; if(!ElementMult(cR2.getOutput(), cTranspose[0].getOutput(), cR2.getPrevOutput())) ReturnFalse; if(!Concat(cR2.getPrevOutput(), cTranspose[1].getOutput(), cConcatenated2[1].getOutput(), units, units, hidden)) ReturnFalse; if(!cHt2.FeedForward(cConcatenated2[1].AsObject())) ReturnFalse; if(!GateElementMult(cHt2.getOutput(), cTranspose[0].getOutput(), cZ2.getOutput(), cH2.getOutput())) ReturnFalse; if(!cTranspose[2].FeedForward(cH2.AsObject())) ReturnFalse; //--- if(!Concat(NeuronOCL.getOutput(), cTranspose[2].getOutput(), cConcatenated[0].getOutput(), inputs, hidden, units)) ReturnFalse; if(!cZR.FeedForward(cConcatenated[0].AsObject())) ReturnFalse; if(!DeConcat(cZ.getOutput(), cR.getOutput(), cZR.getOutput(), hidden, hidden, units)) ReturnFalse; if(!ElementMult(cR.getOutput(), cTranspose[2].getOutput(), cR.getPrevOutput())) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cR.getPrevOutput(), cConcatenated[1].getOutput(), inputs, hidden, units)) ReturnFalse; if(!cHt.FeedForward(cConcatenated[1].AsObject())) ReturnFalse; if(!GateElementMult(cHt.getOutput(), cTranspose[2].getOutput(), cZ.getOutput(), cH.getOutput())) ReturnFalse; //--- if(!cNorm.FeedForward(cH.AsObject())) ReturnFalse; if(!CNeuronSpikeActivation::feedForward(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cHt.GetUnits(); uint hidden = cHt.GetFilters(); uint inputs = cHt.GetWindow() - hidden; //--- if(!CNeuronSpikeActivation::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!cH.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!GateElementMultGrad(cHt.getOutput(), cHt.getGradient(), cTranspose[2].getOutput(), cTranspose[2].getGradient(), cZ.getOutput(), cZ.getGradient(), cH.getGradient(), cHt.Activation(), cTranspose[2].Activation(), cZR.Activation())) ReturnFalse; if(!cConcatenated[1].CalcHiddenGradients(cHt.AsObject())) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cR.getPrevOutput(), cConcatenated[1].getGradient(), inputs, hidden, units)) ReturnFalse; if(!ElementMultGrad(cR.getOutput(), cR.getGradient(), cTranspose[2].getOutput(), cTranspose[2].getPrevOutput(), cR.getPrevOutput(), cZR.Activation(), cH.Activation())) ReturnFalse; if(!SumAndNormilize(cTranspose[2].getGradient(), cTranspose[2].getPrevOutput(), cTranspose[2].getGradient(), hidden, false, 0, 0, 0, 1)) ReturnFalse; if(!Concat(cZ.getGradient(), cR.getGradient(), cZR.getGradient(), hidden, hidden, units)) ReturnFalse; if(!cConcatenated[0].CalcHiddenGradients(cZR.AsObject())) ReturnFalse; if(!DeConcat(cConcatenated[0].getPrevOutput(), cTranspose[2].getPrevOutput(), cConcatenated[0].getGradient(), inputs, hidden, units)) ReturnFalse; if(cTranspose[2].Activation() != None) if(!DeActivation(cTranspose[2].getOutput(), cTranspose[2].getPrevOutput(), cTranspose[2].getPrevOutput(), cTranspose[2].Activation())) ReturnFalse; if(!SumAndNormilize(cTranspose[2].getGradient(), cTranspose[2].getPrevOutput(), cTranspose[2].getGradient(), hidden, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cConcatenated[0].getPrevOutput(), NeuronOCL.getGradient(), inputs, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL) //--- if(!cH2.CalcHiddenGradients(cTranspose[2].AsObject())) ReturnFalse; if(!GateElementMultGrad(cHt2.getOutput(), cHt2.getGradient(), cTranspose[0].getOutput(), cTranspose[0].getGradient(), cZ2.getOutput(), cZ2.getGradient(), cH2.getGradient(), cHt2.Activation(), cTranspose[1].Activation(), cZR2.Activation())) ReturnFalse; if(!cConcatenated2[1].CalcHiddenGradients(cHt2.AsObject())) ReturnFalse; if(!DeConcat(cR2.getPrevOutput(), cTranspose[1].getGradient(), cConcatenated2[1].getOutput(), units, units, hidden)) ReturnFalse; if(!ElementMultGrad(cR2.getOutput(), cR2.getGradient(), cTranspose[0].getOutput(), cTranspose[0].getGradient(), cR2.getPrevOutput(), cZR2.Activation(), cTranspose[0].Activation())) ReturnFalse; if(!Concat(cZ2.getGradient(), cR2.getGradient(), cZR2.getGradient(), units, units, hidden)) ReturnFalse; if(!cConcatenated2[0].CalcHiddenGradients(cZR2.AsObject())) ReturnFalse; if(!DeConcat(cTranspose[0].getGradient(), cTranspose[1].getPrevOutput(), cConcatenated2[0].getGradient(), units, units, hidden)) ReturnFalse; if(!SumAndNormilize(cTranspose[1].getGradient(), cTranspose[1].getPrevOutput(), cTranspose[1].getGradient(), units, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cTranspose[1]) if(!cProjectionX.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cProjectionX.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, inputs, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cProjectionX.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cZR2.UpdateInputWeights(cConcatenated2[0].AsObject())) ReturnFalse; if(!cHt2.UpdateInputWeights(cConcatenated2[1].AsObject())) ReturnFalse; if(!cZR.UpdateInputWeights(cConcatenated[0].AsObject())) ReturnFalse; if(!cHt.UpdateInputWeights(cConcatenated[1].AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cH.AsObject())) ReturnFalse; if(!CNeuronSpikeActivation::updateInputWeights(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvGRU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeConvGRU2D* Source = source; if(!cProjectionX.WeightsUpdate(Source.cProjectionX.AsObject(), tau)) ReturnFalse; if(!cZR2.WeightsUpdate(Source.cZR2.AsObject(), tau)) ReturnFalse; if(!cHt2.WeightsUpdate(Source.cHt2.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::Save(const int file_handle) { if(!CNeuronSpikeConvGRU::Save(file_handle)) ReturnFalse; if(!cProjectionX.Save(file_handle)) ReturnFalse; if(!cZR2.Save(file_handle)) ReturnFalse; if(!cHt2.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvGRU2D::Load(const int file_handle) { if(!CNeuronSpikeConvGRU::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cProjectionX.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cZR2.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cHt2.AsObject())) ReturnFalse; //--- uint units = cHt.GetUnits(); uint chanels_out = cHt.GetFilters(); uint chanels_in = cHt.GetWindow() - chanels_out; //--- uint index = 1; for(uint i = 0; i < cConcatenated2.Size(); i++) { if(!cConcatenated2[i].Init(0, index, OpenCL, 2 * units * chanels_in, optimization, iBatch)) ReturnFalse; cConcatenated[i].SetActivationFunction(None); index++; } if(!cTranspose[0].Init(0, index, OpenCL, units, chanels_out, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose[1].Init(0, index, OpenCL, units, chanels_out, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose[2].Init(0, index, OpenCL, chanels_out, units, optimization, iBatch)) ReturnFalse; index += 2; if(!cZ2.Init(0, index, OpenCL, cZR2.Neurons() / 2, optimization, iBatch)) ReturnFalse; cZ2.SetActivationFunction((ENUM_ACTIVATION)cZR2.Activation()); index++; if(!cR2.Init(0, index, OpenCL, cZR2.Neurons() / 2, optimization, iBatch)) ReturnFalse; cR2.SetActivationFunction((ENUM_ACTIVATION)cZR2.Activation()); index += 2; if(!cH2.Init(0, index, OpenCL, units * chanels_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeConvGRU2D::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvGRU::SetOpenCL(obj); cProjectionX.SetOpenCL(OpenCL); cZR2.SetOpenCL(OpenCL); cHt2.SetOpenCL(OpenCL); for(uint i = 0; i < cConcatenated2.Size(); i++) cConcatenated2[i].SetOpenCL(OpenCL); for(uint i = 0; i < cTranspose.Size(); i++) cTranspose[i].SetOpenCL(OpenCL); cZ2.SetOpenCL(OpenCL); cR2.SetOpenCL(OpenCL); cH2.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronRAFT : public CNeuronSpikeActivation { protected: CLayer cEncoder; CLayer cContext; CLayer cCorrelation; CNeuronBaseOCL cConcatenated; CNeuronSpikeConvGRU2D cGRU; CNeuronSpikeResNeXtResidual cResidual; CLayer cG; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronRAFT(void) {}; ~CNeuronRAFT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &units[], uint &chanels[], uint group_size, uint groups, uint stack_size, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronRAFT; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &units[], uint &chanels[], uint group_size, uint groups, uint stack_size, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units.Size() != chanels.Size()) ReturnFalse; uint layers = units.Size() - 1; if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units[layers]*chanels[layers], optimization_type, batch)) ReturnFalse; //--- int index = 0; CNeuronSpikeResNeXtBlock* resblock = NULL; CNeuronSpikeConvGRU2D* gru = NULL; CNeuronMultiScaleStackCorrelation* correlation = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; //--- cEncoder.Clear(); cContext.Clear(); cCorrelation.Clear(); cG.Clear(); cEncoder.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); cG.SetOpenCL(OpenCL); //--- for(uint i = 0; i < layers; i++) { resblock = new CNeuronSpikeResNeXtBlock(); if(!resblock || !resblock.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, optimization, iBatch) || !cEncoder.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; resblock = new CNeuronSpikeResNeXtBlock(); if(!resblock || !resblock.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, optimization, iBatch) || !cContext.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; } //--- gru = new CNeuronSpikeConvGRU2D(); if(!gru || !gru.Init(0, index, OpenCL, units[layers], chanels[layers], chanels[layers], optimization, iBatch) || !cContext.Add(gru)) { DeleteObj(gru) ReturnFalse; } index++; //--- correlation = new CNeuronMultiScaleStackCorrelation(); if(!correlation || !correlation.Init(0, index, OpenCL, stack_size, chanels[layers], units[layers], levels, optimization, iBatch) || !cCorrelation.Add(correlation)) { DeleteObj(correlation) ReturnFalse; } index++; resblock = new CNeuronSpikeResNeXtBlock(); if(!resblock || !resblock.Init(0, index, OpenCL, correlation.Neurons() / units[layers], chanels[layers], units[layers], units[layers], group_size, groups, optimization, iBatch) || !cCorrelation.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; //--- if(!cConcatenated.Init(0, index, OpenCL, 3 * chanels[layers]*units[layers], optimization, iBatch)) ReturnFalse; index++; //--- if(!cGRU.Init(0, index, OpenCL, units[layers], 3 * chanels[layers], chanels[layers], optimization, iBatch)) ReturnFalse; index++; //--- if(!cResidual.Init(0, index, OpenCL, chanels[0], chanels[layers], units[0], units[layers], optimization, iBatch)) ReturnFalse; index++; //--- neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, Neurons(), optimization, iBatch) || !cG.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); if(!neuron.SetGradient(cGRU.getGradient(), true) || !cResidual.SetGradient(cGRU.getGradient(), true)) ReturnFalse; //--- index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, Neurons(), iBatch, optimization) || !cG.Add(norm)) { DeleteObj(norm) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- prev = NeuronOCL; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- prev = cEncoder[-1]; for(int i = 0; i < cCorrelation.Total(); i++) { current = cCorrelation[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- uint units = cGRU.GetUnits(); if(!Concat(cEncoder[-1].getOutput(), cContext[-1].getOutput(), cCorrelation[-1].getOutput(), cConcatenated.getOutput(), cEncoder[-1].Neurons() / units, cContext[-1].Neurons() / units, cCorrelation[-1].Neurons() / units, units)) ReturnFalse; if(!cGRU.FeedForward(cConcatenated.AsObject())) ReturnFalse; //--- current = cG[0]; if(!current || !SumAndNormilize(cGRU.getOutput(), cResidual.getOutput(), current.getOutput(), cGRU.GetChanelsOut(), false, 0, 0, 0, 1)) ReturnFalse; prev = current; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeActivation::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeActivation::calcInputGradients(cG[-1])) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* next = cG[-1]; for(int i = cG.Total() - 2; i >= 0; i--) { current = cG[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; next = current; } //--- if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; //--- if(!cConcatenated.CalcHiddenGradients(cGRU.AsObject())) ReturnFalse; uint units = cGRU.GetChanelsOut(); if(!DeConcat(cEncoder[-1].getGradient(), cContext[-1].getGradient(), cCorrelation[-1].getGradient(), cConcatenated.getGradient(), cEncoder[-1].Neurons() / units, cContext[-1].Neurons() / units, cCorrelation[-1].Neurons() / units, units)) ReturnFalse; //--- next = cCorrelation[-1]; for(int i = cCorrelation.Total() - 2; i >= 0; i--) { current = cCorrelation[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; next = current; } //--- current = cEncoder[-1]; if(!current) ReturnFalse; CBufferFloat* temp = current.getGradient(); if(!current.SetGradient(current.getPrevOutput(), false) || !current.CalcHiddenGradients(next) || !SumAndNormilize(temp, current.getGradient(), temp, temp.Total() / units, false, 0, 0, 0, 1) || !current.SetGradient(temp, false)) ReturnFalse; //--- next = current; for(int i = cEncoder.Total() - 2; i >= 0; i--) { current = cEncoder[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; next = current; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- next = cContext[-1]; for(int i = cContext.Total() - 2; i >= 0; i--) { current = cContext[i]; if(!current || !current.CalcHiddenGradients(next)) ReturnFalse; next = current; } temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(next) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- prev = NeuronOCL; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- prev = cEncoder[-1]; for(int i = 0; i < cCorrelation.Total(); i++) { current = cCorrelation[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- if(!cGRU.UpdateInputWeights(cConcatenated.AsObject())) ReturnFalse; //--- prev = cG[0]; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeActivation::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronRAFT* Source = source; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; if(!cContext.WeightsUpdate(Source.cContext.AsObject(), tau)) ReturnFalse; if(!cCorrelation.WeightsUpdate(Source.cCorrelation.AsObject(), tau)) ReturnFalse; if(!cG.WeightsUpdate(Source.cG.AsObject(), tau)) ReturnFalse; if(!cGRU.WeightsUpdate(Source.cGRU.AsObject(), tau)) ReturnFalse; if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; //--- if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cCorrelation.Save(file_handle)) ReturnFalse; if(!cG.Save(file_handle)) ReturnFalse; if(!cGRU.Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; //--- if(!cEncoder.Load(file_handle)) ReturnFalse; if(!cContext.Load(file_handle)) ReturnFalse; if(!cCorrelation.Load(file_handle)) ReturnFalse; if(!cG.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGRU.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; //--- if(!cConcatenated.Init(0, 0, OpenCL, cContext[-1].Neurons() + cEncoder[-1].Neurons() + cCorrelation[-1].Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRAFT::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); cG.SetOpenCL(OpenCL); cGRU.SetOpenCL(OpenCL); cResidual.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronRAFT::TrainMode(bool flag) { CNeuronSpikeActivation::TrainMode(flag); cEncoder.TrainMode(bTrain); cContext.TrainMode(bTrain); cCorrelation.TrainMode(bTrain); cG.TrainMode(bTrain); cGRU.TrainMode(bTrain); cResidual.TrainMode(bTrain); cConcatenated.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronRAFT::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; //--- if(!cEncoder.ClearStates()) ReturnFalse; if(!cContext.ClearStates()) ReturnFalse; if(!cCorrelation.ClearStates()) ReturnFalse; if(!cG.ClearStates()) ReturnFalse; if(!cGRU.Clear()) ReturnFalse; if(!cResidual.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultScalStackCorrelBySegments : public CNeuronAddToStack { protected: CNeuronMultiScaleStackCorrelation cCorrelation; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultScalStackCorrelBySegments(void) {}; ~CNeuronMultScalStackCorrelBySegments(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint segments, uint segment_size, uint dimension, uint variables, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultScalStackCorrelBySegments; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override; virtual bool Clear(void) override; //--- virtual uint GetLevels(void) const { return cCorrelation.GetLevels(); } virtual uint SetSegmentSize(void) const { return cCorrelation.GetStackSize(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint segments, uint segment_size, uint dimension, uint variables, uint levels, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAddToStack::Init(numOutputs, myIndex, open_cl, segments, segment_size * levels, variables, optimization_type, batch)) ReturnFalse; if(!cCorrelation.Init(0, 0, OpenCL, segment_size, dimension, variables, levels, optimization, iBatch)) ReturnFalse; SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cCorrelation.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronAddToStack::feedForward(cCorrelation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronAddToStack::calcInputGradients(cCorrelation.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cCorrelation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::Save(const int file_handle) { if(!CNeuronAddToStack::Save(file_handle)) ReturnFalse; if(!cCorrelation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::Load(const int file_handle) { if(!CNeuronAddToStack::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cCorrelation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultScalStackCorrelBySegments::Clear(void) { if(!CNeuronAddToStack::Clear()) ReturnFalse; if(!cCorrelation.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultScalStackCorrelBySegments::SetOpenCL(COpenCLMy *obj) { CNeuronAddToStack::SetOpenCL(obj); cCorrelation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultScalStackCorrelBySegments::SetActivationFunction(ENUM_ACTIVATION value) { cCorrelation.SetActivationFunction(value); activation = (ENUM_ACTIVATION)cCorrelation.Activation(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTMAMFE : public CNeuronConvOCL { protected: CLayer cEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTMAMFE(void) {}; ~CNeuronTMAMFE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTMAMFE; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(windows.Size() != 2) ReturnFalse; //--- if(!CNeuronConvOCL::Init(numOutputs, myIndex, open_cl, 2 * window_out, 2 * window_out, window_out, units, 1, optimization_type, batch)) ReturnFalse; //--- uint index = 0; cEncoder.Clear(); cEncoder.SetOpenCL(OpenCL); CNeuronMultiWindowsConvOCL* mw_conv = NULL; CNeuronBatchNormOCL* normal = NULL; //--- mw_conv = new CNeuronMultiWindowsConvOCL(); if(!mw_conv || !mw_conv.Init(0, index, OpenCL, windows, 2 * window_out, units, 1, optimization, iBatch) || !cEncoder.Add(mw_conv)) { DeleteObj(mw_conv) ReturnFalse; } mw_conv.SetActivationFunction(SoftPlus); index++; normal = new CNeuronBatchNormOCL(); if(!normal || !normal.Init(0, index, OpenCL, mw_conv.Neurons(), iBatch, optimization) || !cEncoder.Add(normal)) { DeleteObj(normal) ReturnFalse; } index++; //--- uint temp[] = {2 * window_out, 2 * window_out}; mw_conv = new CNeuronMultiWindowsConvOCL(); if(!mw_conv || !mw_conv.Init(0, index, OpenCL, temp, window_out, units, 1, optimization, iBatch) || !cEncoder.Add(mw_conv)) { DeleteObj(mw_conv) ReturnFalse; } mw_conv.SetActivationFunction(SoftPlus); index++; normal = new CNeuronBatchNormOCL(); if(!normal || !normal.Init(0, index, OpenCL, mw_conv.Neurons(), iBatch, optimization) || !cEncoder.Add(normal)) { DeleteObj(normal) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronConvOCL::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronConvOCL::calcInputGradients(cEncoder[-1])) ReturnFalse; CNeuronBaseOCL* next = cEncoder[-1]; CNeuronBaseOCL* curr = NULL; //--- for(int i = cEncoder.Total() - 2; i >= 0; i--) { curr = cEncoder[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronConvOCL::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::Save(const int file_handle) { if(!CNeuronConvOCL::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::Load(const int file_handle) { if(!CNeuronConvOCL::Load(file_handle)) ReturnFalse; if(!cEncoder.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMFE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronConvOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronTMAMFE* Source = source; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTMAMFE::SetOpenCL(COpenCLMy *obj) { CNeuronConvOCL::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTMAMFE::TrainMode(bool flag) { CNeuronConvOCL::TrainMode(flag); cEncoder.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTMAMPA : public CNeuronBaseOCL { protected: CNeuronRelativeCrossAttention cCrossAttention; CLayer cProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTMAMPA(void) {}; ~CNeuronTMAMPA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units, uint segments, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTMAMPA; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint units, uint segments, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units * segments, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cCrossAttention.Init(0, index, OpenCL, window, window_key, units * segments, heads, window, segments, optimization, iBatch)) ReturnFalse; index++; //--- cProjection.Clear(); cProjection.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; //--- Concatenated neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 2 * Neurons(), optimization, iBatch) || !cProjection.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; //--- Projection conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 2 * window, 2 * window, window, units * segments, 1, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cProjection.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; //--- Feed Forward conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, 2 * window, units * segments, 1, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 2 * window, 2 * window, window, units * segments, 1, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!cCrossAttention.FeedForward(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; CNeuronBaseOCL* neuron = cProjection[0]; uint window = cCrossAttention.GetWindow(); uint units = cCrossAttention.GetUnits(); uint segments = cCrossAttention.GetUnitsKV(); if(!neuron || !Concat(NeuronOCL.getOutput(), cCrossAttention.getOutput(), neuron.getOutput(), window, window, units)) ReturnFalse; //--- for(int i = 1; i < cProjection.Total(); i++) { neuron = cProjection[i]; if(!neuron || !neuron.FeedForward(cProjection[i - 1])) ReturnFalse; } //--- if(!SumAndNormilize(NeuronOCL.getOutput(), neuron.getOutput(), Output, window * segments, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* neuron = cProjection[-1]; if(!neuron || !DeActivation(neuron.getOutput(), neuron.getGradient(), Gradient, neuron.Activation())) ReturnFalse; for(int i = cProjection.Total() - 2; i >= 0; i--) { neuron = cProjection[0]; if(!neuron || !neuron.CalcHiddenGradients(cProjection[i + 1])) ReturnFalse; } //--- uint window = cCrossAttention.GetWindow(); uint units = cCrossAttention.GetUnits(); if(!DeConcat(PrevOutput, cCrossAttention.getGradient(), neuron.getGradient(), window, window, units)) ReturnFalse; Deactivation(cCrossAttention) //--- if(!SumAndNormilize(PrevOutput, Gradient, PrevOutput, window, false, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, PrevOutput, NeuronOCL.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cCrossAttention.AsObject(), NeuronOCL.getOutput(), cProjection[-1].getPrevOutput(), (ENUM_ACTIVATION)NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cProjection[-1].getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cCrossAttention.UpdateInputWeights(NeuronOCL, NeuronOCL.getOutput())) ReturnFalse; //--- CNeuronBaseOCL* neuron = cProjection[0]; for(int i = 1; i < cProjection.Total(); i++) { neuron = cProjection[i]; if(!neuron || !neuron.UpdateInputWeights(cProjection[i - 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronTMAMPA* Source = source; if(!cCrossAttention.WeightsUpdate(Source.cCrossAttention.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cCrossAttention.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMAMPA::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; if(!cProjection.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTMAMPA::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cCrossAttention.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronTMA : public CNeuronRAFT { protected: CNeuronAddToStack cStatesStack; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronTMA(void) {}; ~CNeuronTMA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &units[], uint &chanels[], uint group_size, uint groups, uint segments, uint segment_size, uint levels, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronTMA; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &units[], uint &chanels[], uint group_size, uint groups, uint segments, uint segment_size, uint levels, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(units.Size() != chanels.Size()) ReturnFalse; uint layers = units.Size() - 1; if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units[layers]*chanels[layers], optimization_type, batch)) ReturnFalse; //--- int index = 0; CNeuronSpikeResNeXtBlock* resblock = NULL; CNeuronSpikeConvGRU2D* gru = NULL; CNeuronMultScalStackCorrelBySegments* correlation = NULL; CNeuronTMAMFE* mfe = NULL; CNeuronTMAMPA* mpa = NULL; CNeuronTransposeRCDOCL* transpose = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; //--- cEncoder.Clear(); cContext.Clear(); cCorrelation.Clear(); cG.Clear(); cEncoder.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); cG.SetOpenCL(OpenCL); //--- for(uint i = 0; i < layers; i++) { resblock = new CNeuronSpikeResNeXtBlock(); if(!resblock || !resblock.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, optimization, iBatch) || !cEncoder.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; resblock = new CNeuronSpikeResNeXtBlock(); if(!resblock || !resblock.Init(0, index, OpenCL, chanels[i], chanels[i + 1], units[i], units[i + 1], group_size, groups, optimization, iBatch) || !cContext.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; } //--- gru = new CNeuronSpikeConvGRU2D(); if(!gru || !gru.Init(0, index, OpenCL, units[layers], chanels[layers], chanels[layers], optimization, iBatch) || !cContext.Add(gru)) { DeleteObj(gru) ReturnFalse; } index++; //--- if(!cStatesStack.Init(0, index, OpenCL, segments, chanels[layers], units[layers], optimization, iBatch)) ReturnFalse; index++; //--- correlation = new CNeuronMultScalStackCorrelBySegments(); if(!correlation || !correlation.Init(0, index, OpenCL, segments, segment_size, chanels[layers], units[layers], levels, optimization, iBatch) || !cCorrelation.Add(correlation)) { DeleteObj(correlation) ReturnFalse; } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, correlation.Neurons() + cStatesStack.Neurons(), optimization, iBatch) || !cCorrelation.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } index++; { uint temp[] = { cStatesStack.GetDimension(), correlation.GetDimension() }; mfe = new CNeuronTMAMFE(); if(!mfe || !mfe.Init(0, index, OpenCL, segments * units[layers], temp, chanels[layers], optimization, iBatch) || !cCorrelation.Add(mfe)) { DeleteObj(mfe) ReturnFalse; } } index++; transpose = new CNeuronTransposeRCDOCL(); if(!transpose || !transpose.Init(0, index, OpenCL, units[layers], segments, chanels[layers], optimization, iBatch) || !cCorrelation.Add(transpose)) { DeleteObj(transpose) ReturnFalse; } index++; for(int i = 0; i < 2; i++) { mpa = new CNeuronTMAMPA(); if(!mpa || !mpa.Init(0, index, OpenCL, chanels[layers], window_key, segments, units[layers], heads, optimization, iBatch) || !cCorrelation.Add(mpa)) { DeleteObj(mpa) ReturnFalse; } index++; } transpose = new CNeuronTransposeRCDOCL(); if(!transpose || !transpose.Init(0, index, OpenCL, segments, units[layers], chanels[layers], optimization, iBatch) || !cCorrelation.Add(transpose)) { DeleteObj(transpose) ReturnFalse; } index++; //--- if(!cConcatenated.Init(0, index, OpenCL, units[layers] * ((segments + 2) * chanels[layers]), optimization, iBatch)) ReturnFalse; index++; //--- if(!cGRU.Init(0, index, OpenCL, units[layers], (segments + 2) * chanels[layers], chanels[layers], optimization, iBatch)) ReturnFalse; index++; //--- if(!cResidual.Init(0, index, OpenCL, chanels[0], chanels[layers], units[0], units[layers], optimization, iBatch)) ReturnFalse; index++; //--- neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, Neurons(), optimization, iBatch) || !cG.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); if(!neuron.SetGradient(cGRU.getGradient(), true) || !cResidual.SetGradient(cGRU.getGradient(), true)) ReturnFalse; //--- index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, Neurons(), iBatch, optimization) || !cG.Add(norm)) { DeleteObj(norm) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- prev = NeuronOCL; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- if(!cStatesStack.FeedForward(cEncoder[-1])) ReturnFalse; CNeuronMultScalStackCorrelBySegments* correlation = cCorrelation[0]; if(!correlation || !correlation.FeedForward(cEncoder[-1])) ReturnFalse; //--- uint variables = cStatesStack.GetVariables(); uint segments = cStatesStack.GetStackSize(); uint dimension = cStatesStack.GetDimension(); uint corr_dim = correlation.GetDimension(); //--- prev = cCorrelation[1]; if(!Concat(cStatesStack.getOutput(), correlation.getOutput(), prev.getOutput(), dimension, corr_dim, variables * segments)) ReturnFalse; for(int i = 2; i < cCorrelation.Total(); i++) { current = cCorrelation[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- if(!Concat(cEncoder[-1].getOutput(), cContext[-1].getOutput(), cCorrelation[-1].getOutput(), cConcatenated.getOutput(), dimension, dimension, segments * dimension, variables)) ReturnFalse; if(!cGRU.FeedForward(cConcatenated.AsObject())) ReturnFalse; //--- current = cG[0]; if(!current || !SumAndNormilize(cGRU.getOutput(), cResidual.getOutput(), current.getOutput(), cGRU.GetChanelsOut(), false, 0, 0, 0, 1)) ReturnFalse; prev = current; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeActivation::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeActivation::calcInputGradients(cG[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; //--- for(int i = cG.Total() - 2; i >= 0; i--) { curr = cG[i]; if(!curr || !curr.CalcHiddenGradients(cG[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; //--- if(!cConcatenated.CalcHiddenGradients(cGRU.AsObject())) ReturnFalse; //--- uint variables = cStatesStack.GetVariables(); uint segments = cStatesStack.GetStackSize(); uint dimension = cStatesStack.GetDimension(); //--- if(!cEncoder[-1] || !cContext[-1] || !cCorrelation[-1] || !DeConcat(cEncoder[-1].getGradient(), cContext[-1].getGradient(), cCorrelation[-1].getGradient(), cConcatenated.getGradient(), dimension, dimension, segments * dimension, variables)) ReturnFalse; Deactivation(cContext[-1]) Deactivation(cCorrelation[-1]) if(cEncoder[-1].Activation() != None) if(!DeActivation(cEncoder[-1].getOutput(), cEncoder[-1].getPrevOutput(), cEncoder[-1].getGradient(), cEncoder[-1].Activation())) ReturnFalse; //--- for(int i = cCorrelation.Total() - 2; i >= 1; i--) { curr = cCorrelation[i]; if(!curr || !curr.CalcHiddenGradients(cCorrelation[i + 1])) ReturnFalse; } CNeuronMultScalStackCorrelBySegments* correlation = cCorrelation[0]; uint corr_dim = correlation.GetDimension(); if(!DeConcat(cStatesStack.getGradient(), correlation.getGradient(), curr.getGradient(), dimension, corr_dim, variables * segments)) ReturnFalse; curr = cEncoder[-1]; if(!curr.CalcHiddenGradients(cStatesStack.AsObject()) || !SumAndNormilize(curr.getGradient(), curr.getPrevOutput(), curr.getPrevOutput(), dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!curr.CalcHiddenGradients(correlation) || !SumAndNormilize(curr.getGradient(), curr.getPrevOutput(), curr.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = cEncoder.Total() - 2; i >= 0; i--) { curr = cEncoder[i]; if(!curr || !curr.CalcHiddenGradients(cEncoder[i + 1])) ReturnFalse; } if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- for(int i = cContext.Total() - 2; i >= 0; i--) { curr = cContext[i]; if(!curr || !curr.CalcHiddenGradients(cContext[i + 1])) ReturnFalse; } CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(curr) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- prev = NeuronOCL; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- CNeuronMultScalStackCorrelBySegments* correlation = cCorrelation[0]; if(!correlation || !correlation.UpdateInputWeights(cEncoder[-1])) ReturnFalse; //--- prev = cCorrelation[1]; for(int i = 2; i < cCorrelation.Total(); i++) { current = cCorrelation[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- if(!cGRU.UpdateInputWeights(cConcatenated.AsObject())) ReturnFalse; //--- prev = cG[0]; for(int i = 1; i < cG.Total(); i++) { current = cG[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeActivation::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::Save(const int file_handle) { if(!CNeuronRAFT::Save(file_handle)) ReturnFalse; if(!cStatesStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::Load(const int file_handle) { if(!CNeuronRAFT::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStatesStack.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronTMA::Clear(void) { if(!CNeuronRAFT::Clear()) ReturnFalse; if(!cStatesStack.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronTMA::SetOpenCL(COpenCLMy *obj) { CNeuronRAFT::SetOpenCL(obj); cStatesStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSTConvGRU : public CNeuronSpikeActivation { protected: CNeuronBaseOCL cConcatenated[2]; CNeuronMultiWindowsConvOCL cFSM; CNeuronBaseOCL cSM; CNeuronConvOCL cOs; CNeuronConvOCL cOt; CNeuronBaseOCL cFo; CNeuronTransposeRCDOCL cS; CNeuronBatchNormOCL cNorm; //--- virtual bool GRU(void); virtual bool GRU_grad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronSpikeSTConvGRU(void) {}; ~CNeuronSpikeSTConvGRU(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels_in[], uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSTConvGRU; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual uint GetChanels(void) const { return cOt.GetFilters(); } virtual uint GetUnits(void) const { return cOt.GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels_in[], uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels_in.Size() != 2) ReturnFalse; if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units * chanels_out, optimization_type, batch)) ReturnFalse; //--- Concatenate [2*F0,S0,M0]*Units uint index = 0; if(!cConcatenated[0].Init(0, index, OpenCL, (2 * chanels_in[0] + chanels_in[1] + chanels_out)*units, optimization, iBatch)) ReturnFalse; cConcatenated[0].SetActivationFunction(None); //--- [6*F,3*S,3*M]*Units index++; uint windows[] = { chanels_in[0], chanels_in[0], chanels_out, chanels_in[1] }; if(!cFSM.Init(0, index, OpenCL, windows, 3 * chanels_out, units, 1, optimization, iBatch)) ReturnFalse; cFSM.SetActivationFunction(None); //--- [S,M]*Units index++; if(!cSM.Init(0, index, OpenCL, 2 * chanels_out * units, optimization, iBatch)) ReturnFalse; cSM.SetActivationFunction(None); //--- [F,S,M]*Units index++; if(!cConcatenated[1].Init(0, index, OpenCL, (chanels_in[0] + 2 * chanels_out)*units, optimization, iBatch)) ReturnFalse; //--- [S]*Units index++; if(!cOs.Init(0, index, OpenCL, chanels_in[0] + 2 * chanels_out, chanels_in[0] + 2 * chanels_out, chanels_out, units, 1, optimization, iBatch)) ReturnFalse; cOs.SetActivationFunction(SIGMOID); //--- [S]*Units index++; if(!cOt.Init(0, index, OpenCL, 2 * chanels_out, 2 * chanels_out, chanels_out, units, 1, optimization, iBatch)) ReturnFalse; cOt.SetActivationFunction(TANH); //--- [S]*Units index++; if(!cFo.Init(0, index, OpenCL, chanels_out * units, optimization, iBatch)) ReturnFalse; cFo.SetActivationFunction(None); //--- [S]*Units index++; if(!cS.Init(0, index, OpenCL, units, 2, chanels_out, optimization, iBatch)) ReturnFalse; cS.SetActivationFunction(None); //--- [S]*Units index++; if(!cNorm.Init(0, index, OpenCL, Neurons(), iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; //--- uint Fdimension = cFSM.GetWindow(0); uint Sdimension = cS.GetDimension(); uint Mdimension = cFSM.GetWindow(3); uint units = cFSM.GetUnits(); //--- if(!cS.FeedForward(cSM.AsObject())) ReturnFalse; //--- if(!cSM.SwapOutputs()) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), NeuronOCL.getOutput(), cS.getOutput(), SecondInput, cConcatenated[0].getOutput(), Fdimension, Fdimension, Sdimension, Mdimension, units)) ReturnFalse; if(!cFSM.FeedForward(cConcatenated[0].AsObject())) ReturnFalse; if(!GRU()) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), cSM.getOutput(), cConcatenated[1].getOutput(), Fdimension, 2 * Sdimension, units)) ReturnFalse; if(!cOs.FeedForward(cConcatenated[1].AsObject())) ReturnFalse; if(!cOt.FeedForward(cSM.AsObject())) ReturnFalse; if(!ElementMult(cOs.getOutput(), cOt.getOutput(), cFo.getOutput())) ReturnFalse; if(!cNorm.FeedForward(cFo.AsObject())) ReturnFalse; //--- return CNeuronSpikeActivation::feedForward(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::GRU(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { cFSM.GetUnits(), cSM.Neurons() / cFSM.GetUnits() }; uint kernel = def_k_GRU; setBuffer(kernel, def_k_gru_XH, cFSM.getOutputIndex()) setBuffer(kernel, def_k_gru_prev_state, cSM.getPrevOutIndex()) setBuffer(kernel, def_k_gru_outputs, cSM.getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::GRU_grad(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { cFSM.GetUnits(), cSM.Neurons() / cFSM.GetUnits() }; uint kernel = def_k_GRU_Grad; setBuffer(kernel, def_k_gru_gr_XH, cFSM.getOutputIndex()) setBuffer(kernel, def_k_gru_gr_XH_gr, cFSM.getGradientIndex()) setBuffer(kernel, def_k_gru_gr_prev_state, cSM.getPrevOutIndex()) setBuffer(kernel, def_k_gru_gr_outputs_gr, cSM.getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient || !SecondInput) ReturnFalse; //--- uint Fdimension = cFSM.GetWindow(0); uint Sdimension = cS.GetDimension(); uint Mdimension = cFSM.GetWindow(3); uint units = cFSM.GetUnits(); //--- if(!CNeuronSpikeActivation::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!cFo.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!ElementMultGrad(cOs.getOutput(), cOs.getGradient(), cOt.getOutput(), cOt.getGradient(), cFo.getGradient(), cOs.Activation(), cOt.Activation())) ReturnFalse; if(!cSM.CalcHiddenGradients(cOt.AsObject())) ReturnFalse; if(!cConcatenated[1].CalcHiddenGradients(cOs.AsObject())) ReturnFalse; if(!DeConcat(NeuronOCL.getGradient(), cConcatenated[1].getPrevOutput(), cConcatenated[1].getGradient(), Fdimension, 2 * Sdimension, units)) ReturnFalse; if(cSM.Activation() != None) if(!DeActivation(cSM.getOutput(), cConcatenated[1].getPrevOutput(), cConcatenated[1].getPrevOutput(), cSM.Activation())) ReturnFalse; if(!SumAndNormilize(cSM.getGradient(), cConcatenated[1].getGradient(), cSM.getGradient(), 2 * Sdimension, false, 0, 0, 0, 1)) ReturnFalse; if(!GRU_grad()) ReturnFalse; if(!cConcatenated[0].CalcHiddenGradients(cFSM.AsObject())) ReturnFalse; if(!DeConcat(cConcatenated[0].getPrevOutput(), cConcatenated[1].getPrevOutput(), cS.getPrevOutput(), SecondGradient, cConcatenated[0].getGradient(), Fdimension, Fdimension, Sdimension, Mdimension, units)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cConcatenated[0].getPrevOutput(), NeuronOCL.getGradient(), Fdimension, false, 0, 0, 0, 1) || !SumAndNormilize(NeuronOCL.getGradient(), cConcatenated[1].getPrevOutput(), NeuronOCL.getGradient(), Fdimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cFSM.UpdateInputWeights(cConcatenated[0].AsObject())) ReturnFalse; if(!cOs.UpdateInputWeights(cConcatenated[1].AsObject())) ReturnFalse; if(!cOt.UpdateInputWeights(cSM.AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cFo.AsObject())) ReturnFalse; //--- return CNeuronSpikeActivation::updateInputWeights(cNorm.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSTConvGRU* Source = source; if(!cFSM.WeightsUpdate(Source.cFSM.AsObject(), tau)) ReturnFalse; if(!cOs.WeightsUpdate(Source.cOs.AsObject(), tau)) ReturnFalse; if(!cOt.WeightsUpdate(Source.cOt.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; if(!cFSM.Save(file_handle)) ReturnFalse; if(!cOs.Save(file_handle)) ReturnFalse; if(!cOt.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cFSM.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cOs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cOt.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; //--- uint Fdimension = cFSM.GetWindow(0); uint Sdimension = cFSM.GetWindow(2); uint Mdimension = cFSM.GetWindow(3); uint units = cFSM.GetUnits(); //--- Concatenate [2*F0,S0,M0]*Units uint index = 0; if(!cConcatenated[0].Init(0, index, OpenCL, (2 * Fdimension + Sdimension + Mdimension)*units, optimization, iBatch)) ReturnFalse; cConcatenated[0].SetActivationFunction(None); //--- [S,M]*Units index += 2; if(!cSM.Init(0, index, OpenCL, 2 * Sdimension * units, optimization, iBatch)) ReturnFalse; cSM.SetActivationFunction(None); //--- [F,S,M]*Units index++; if(!cConcatenated[1].Init(0, index, OpenCL, cOs.GetWindow()*units, optimization, iBatch)) ReturnFalse; //--- [S]*Units index += 3; if(!cFo.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cFo.SetActivationFunction(None); //--- [S]*Units index++; if(!cS.Init(0, index, OpenCL, units, 2, Sdimension, optimization, iBatch)) ReturnFalse; cS.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTConvGRU::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; if(!cSM.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSTConvGRU::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); //--- cFSM.SetOpenCL(OpenCL); cOs.SetOpenCL(OpenCL); cOt.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); for(uint i = 0; i < cConcatenated.Size(); i++) cConcatenated[i].SetOpenCL(OpenCL); cSM.SetOpenCL(OpenCL); cFo.SetOpenCL(OpenCL); cS.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFERENet : public CNeuronSpikeConvGRU { protected: CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronFERENet(void) {}; ~CNeuronFERENet(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFERENet; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual CNeuronBaseOCL* GetLayer(int index) const { return cFlow[index]; } virtual CNeuronBaseOCL* operator[](const int index) const { return GetLayer(index); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels[], ENUM_OPTIMIZATION optimization_type, uint batch) { uint layers = chanels.Size() - 1; if(layers < 2) ReturnFalse; if(!CNeuronSpikeConvGRU::Init(numOutputs, myIndex, open_cl, units, chanels[layers - 1], chanels[layers], optimization_type, batch)) ReturnFalse; //--- cFlow.Clear(); cFlow.SetOpenCL(OpenCL); //--- uint index = 0; uint windows[] = { chanels[0], chanels[layers] }; CNeuronSpikeSTConvGRU* st_gru = new CNeuronSpikeSTConvGRU(); if(!st_gru || !st_gru.Init(0, index, OpenCL, units, windows, chanels[1], optimization, iBatch) || !cFlow.Add(st_gru)) { DeleteObj(st_gru) ReturnFalse; } //--- CNeuronSpikeConvGRU* gru = NULL; for(uint i = 1; i < layers - 1; i++) { CNeuronSpikeConvGRU* gru = new CNeuronSpikeConvGRU(); if(!gru || !gru.Init(0, index, OpenCL, units, chanels[i], chanels[i + 1], optimization, iBatch) || !cFlow.Add(gru)) { DeleteObj(gru) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *curr = cFlow[0]; if(!curr || !curr.FeedForward(prev, cH.getOutput())) ReturnFalse; prev = curr; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- return CNeuronSpikeConvGRU::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvGRU::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CNeuronBaseOCL *next = cFlow[-1]; CNeuronBaseOCL *curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next, cH.getPrevOutput(), cH.getGradient(), (ENUM_ACTIVATION)cH.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *curr = cFlow[0]; if(!curr || !curr.UpdateInputWeights(prev, cH.getOutput())) ReturnFalse; prev = curr; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return CNeuronSpikeConvGRU::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvGRU::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFERENet* Source = source; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::Save(const int file_handle) { if(!CNeuronSpikeConvGRU::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::Load(const int file_handle) { if(!CNeuronSpikeConvGRU::Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFERENet::Clear(void) { if(!CNeuronSpikeConvGRU::Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFERENet::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvGRU::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFGDModule : public CNeuronBaseOCL { protected: CNeuronBatchNormOCL cNorm; CNeuronConvOCL cD; CNeuronBaseOCL cFlowIn; CNeuronConvOCL cFlow; CNeuronBaseOCL cConcatenated; CNeuronMultiWindowsConvOCL cXH; //--- virtual bool GRU(void); virtual bool GRU_grad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput)override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second)override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronFGDModule(void) {}; ~CNeuronFGDModule(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint main_dimension, uint flow_dimension, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronFGDModule; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual CNeuronConvOCL* GetD(void) { return cD.AsObject(); } virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::GRU(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { cXH.GetUnits(), cFlow.GetFilters() }; uint kernel = def_k_GRU; setBuffer(kernel, def_k_gru_XH, cXH.getOutputIndex()) setBuffer(kernel, def_k_gru_prev_state, cFlow.getOutputIndex()) setBuffer(kernel, def_k_gru_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::GRU_grad(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[2] = { cXH.GetUnits(), cFlow.GetFilters() }; uint kernel = def_k_GRU_Grad; setBuffer(kernel, def_k_gru_gr_XH, cXH.getOutputIndex()) setBuffer(kernel, def_k_gru_gr_XH_gr, cXH.getGradientIndex()) setBuffer(kernel, def_k_gru_gr_prev_state, cFlow.getOutputIndex()) setBuffer(kernel, def_k_gru_gr_outputs_gr, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint main_dimension, uint flow_dimension, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * dimension_out, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cNorm.Init(0, index, OpenCL, main_dimension * units, iBatch, optimization)) ReturnFalse; cNorm.SetActivationFunction(None); index++; if(!cD.Init(0, index, OpenCL, main_dimension, main_dimension, dimension_out, units, 1, optimization, iBatch)) ReturnFalse; cD.SetActivationFunction(TANH); index++; if(!cFlowIn.Init(0, index, OpenCL, flow_dimension * units, optimization, iBatch)) ReturnFalse; DeleteObj(cFlowIn.getOutput()) cFlowIn.SetActivationFunction(None); index++; if(!cFlow.Init(0, index, OpenCL, flow_dimension, flow_dimension, dimension_out, units, 1, optimization, iBatch)) ReturnFalse; cFlow.SetActivationFunction(None); index++; if(!cConcatenated.Init(0, index, OpenCL, 2 * dimension_out * units, optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); index++; uint windows[] = { dimension_out, dimension_out }; if(!cXH.Init(0, index, OpenCL, windows, 3 * dimension_out, units, 1, optimization, iBatch)) ReturnFalse; cXH.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!cFlowIn.SetOutput(SecondInput, false)) ReturnFalse; //--- if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cD.FeedForward(cNorm.AsObject())) ReturnFalse; if(!cFlow.FeedForward(cFlowIn.AsObject())) ReturnFalse; if(!Concat(cD.getOutput(), cFlow.getOutput(), cConcatenated.getOutput(), cD.GetFilters(), cFlow.GetFilters(), cFlow.GetUnits())) ReturnFalse; if(!cXH.FeedForward(cConcatenated.AsObject())) ReturnFalse; if(!GRU()) ReturnFalse; if(!cD.getGradient().Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; if(!cFlowIn.SetGradient(SecondGradient, true)) ReturnFalse; cFlowIn.SetActivationFunction(SecondActivation); //--- if(!GRU_grad()) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cXH.AsObject())) ReturnFalse; if(!DeConcat(cD.getPrevOutput(), cFlow.getGradient(), cConcatenated.getGradient(), cD.GetFilters(), cFlow.GetFilters(), cFlow.GetUnits())) ReturnFalse; if(!SumAndNormilize(cD.getGradient(), cD.getPrevOutput(), cD.getGradient(), cD.GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cFlow) Deactivation(cD) if(!cFlowIn.CalcHiddenGradients(cFlow.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cD.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cD.UpdateInputWeights(cNorm.AsObject())) ReturnFalse; if(!cFlow.UpdateInputWeights(cFlowIn.AsObject())) ReturnFalse; if(!cXH.UpdateInputWeights(cConcatenated.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronFGDModule* Source = source; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; if(!cD.WeightsUpdate(Source.cD.AsObject(), tau)) ReturnFalse; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; if(!cXH.WeightsUpdate(Source.cXH.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; if(!cD.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cXH.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFGDModule::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cD.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFlow.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cXH.AsObject())) ReturnFalse; //--- if(!cFlowIn.Init(0, 2, OpenCL, cFlow.GetWindow() * cFlow.GetUnits(), optimization, iBatch)) ReturnFalse; cFlowIn.SetActivationFunction(None); if(!cConcatenated.Init(0, 4, OpenCL, cFlow.Neurons() + cD.Neurons(), optimization, iBatch)) ReturnFalse; cConcatenated.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFGDModule::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cNorm.SetOpenCL(OpenCL); cD.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cXH.SetOpenCL(OpenCL); cFlowIn.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronEVMGRFlowNet : public CNeuronSpikeActivation { protected: CNeuronFERENet cEncoder; CNeuronSTEFlowNetResidualBlock cNeck; CLayer cDecoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronEVMGRFlowNet(void) {}; ~CNeuronEVMGRFlowNet(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronEVMGRFlowNet; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual bool Clear(void) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &chanels[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels.Size() < 2) ReturnFalse; if(!CNeuronSpikeActivation::Init(numOutputs, myIndex, open_cl, units * chanels[0], optimization_type, batch)) ReturnFalse; uint layers = chanels.Size() - 1; //--- Encoder int index = 0; if(!cEncoder.Init(0, index, OpenCL, units, chanels, optimization, iBatch)) ReturnFalse; index++; if(!cNeck.Init(0, index, OpenCL, chanels[layers], units, 32, 8, optimization, iBatch)) ReturnFalse; //--- Decoder cDecoder.Clear(); cDecoder.SetOpenCL(OpenCL); CNeuronFGDModule* fgd = NULL; CNeuronBaseOCL* concat = cNeck.AsObject(); uint flow_dimension = chanels[0]; for(uint i = 0; i < layers; i++) { uint main_dimension = concat.Neurons() / units; fgd = new CNeuronFGDModule(); if(!fgd || !fgd.Init(0, index, OpenCL, units, main_dimension, flow_dimension, chanels[layers - i - 1], optimization, iBatch) || !cDecoder.Add(fgd)) { DeleteObj(fgd) ReturnFalse; } if(i == (layers - 1)) break; index++; CNeuronBaseOCL* encoder = cEncoder.GetLayer(-int(i + 1)); CNeuronBaseOCL* d = fgd.GetD(); if(!encoder) { concat = new CNeuronBaseOCL(); if(!concat || !concat.Init(0, index, OpenCL, fgd.Neurons() + d.Neurons(), optimization, iBatch) || !cDecoder.Add(concat)) { DeleteObj(concat) ReturnFalse; } } else { concat = new CNeuronBaseOCL(); if(!concat || !concat.Init(0, index, OpenCL, fgd.Neurons() + d.Neurons() + encoder.Neurons(), optimization, iBatch) || !cDecoder.Add(concat)) { DeleteObj(concat) ReturnFalse; } } index++; } //--- CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, fgd.Neurons(), iBatch, optimization) || !cDecoder.Add(norm)) { DeleteObj(norm) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cEncoder.FeedForward(NeuronOCL)) ReturnFalse; if(!cNeck.FeedForward(cEncoder.AsObject())) ReturnFalse; //--- Decoder int layers = cDecoder.Total() / 2; CNeuronFGDModule* fgd = NULL; CNeuronBaseOCL* d = NULL; CNeuronBaseOCL* encoder = NULL; CNeuronBaseOCL* prev = cNeck.AsObject(); CNeuronBaseOCL* flow = cDecoder[-2]; if(!flow.SwapOutputs()) ReturnFalse; uint units = cEncoder.GetUnits(); if(!flow) ReturnFalse; for(int i = 0; i < layers; i++) { fgd = cDecoder[i * 2]; if(!fgd || !fgd.FeedForward(prev, flow.getPrevOutput())) ReturnFalse; prev = cDecoder[i * 2 + 1]; if(!prev) ReturnFalse; if(i == (layers - 1)) { if(!prev.FeedForward(fgd)) ReturnFalse; break; } d = fgd.GetD(); encoder = cEncoder[-(i + 1)]; if(!encoder) { if(!Concat(fgd.getOutput(), d.getOutput(), prev.getOutput(), fgd.Neurons() / units, d.Neurons() / units, units)) ReturnFalse; } else { if(!Concat(fgd.getOutput(), d.getOutput(), encoder.getOutput(), prev.getOutput(), fgd.Neurons() / units, d.Neurons() / units, encoder.Neurons() / units, units)) ReturnFalse; } } //--- return CNeuronSpikeActivation::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeActivation::calcInputGradients(cDecoder[-1])) ReturnFalse; //--- Decoder int layers = cDecoder.Total() / 2; CNeuronFGDModule* fgd = cDecoder[-2]; CNeuronBaseOCL* d = NULL; CNeuronBaseOCL* encoder = NULL; CNeuronBaseOCL* next = cDecoder[-1]; CNeuronBaseOCL* flow = cDecoder[-2]; if(!fgd || !fgd.CalcHiddenGradients(next)) ReturnFalse; uint units = cEncoder.GetUnits(); for(int i = layers - 2; i >= 0; i--) { next = cDecoder[i * 2 + 1]; if(!next || !next.CalcHiddenGradients(fgd, flow.getOutput(), flow.getGradient(), (ENUM_ACTIVATION)flow.Activation())) ReturnFalse; //--- fgd = cDecoder[i * 2]; if(!fgd) ReturnFalse; d = fgd.GetD(); encoder = cEncoder[-(i + 1)]; if(!encoder) { if(!DeConcat(fgd.getGradient(), d.getGradient(), next.getGradient(), fgd.Neurons() / units, d.Neurons() / units, units)) ReturnFalse; } else { if(!DeConcat(fgd.getGradient(), d.getGradient(), encoder.getGradient(), next.getGradient(), fgd.Neurons() / units, d.Neurons() / units, encoder.Neurons() / units, units)) ReturnFalse; } Deactivation(fgd); } //--- if(!cNeck.CalcHiddenGradients(fgd, flow.getOutput(), flow.getGradient(), (ENUM_ACTIVATION)flow.Activation())) ReturnFalse; if(!cEncoder.CalcHiddenGradients(cNeck.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEncoder.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cEncoder.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cNeck.UpdateInputWeights(cEncoder.AsObject())) ReturnFalse; //--- Decoder int layers = cDecoder.Total() / 2; CNeuronFGDModule* fgd = NULL; CNeuronBaseOCL* prev = cNeck.AsObject(); CNeuronBaseOCL* flow = cDecoder[-2]; uint units = cEncoder.GetUnits(); if(!flow) ReturnFalse; for(int i = 0; i < layers; i++) { fgd = cDecoder[i * 2]; if(!fgd || !fgd.UpdateInputWeights(prev, flow.getPrevOutput())) ReturnFalse; prev = cDecoder[i * 2 + 1]; if(!prev) ReturnFalse; if(i == (layers - 1)) if(!prev.UpdateInputWeights(fgd)) ReturnFalse; } //--- return CNeuronSpikeActivation::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeActivation::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronEVMGRFlowNet* Source = source; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; if(!cNeck.WeightsUpdate(Source.cNeck.AsObject(), tau)) ReturnFalse; if(!cDecoder.WeightsUpdate(Source.cDecoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::Save(const int file_handle) { if(!CNeuronSpikeActivation::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cNeck.Save(file_handle)) ReturnFalse; if(!cDecoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::Load(const int file_handle) { if(!CNeuronSpikeActivation::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNeck.AsObject())) ReturnFalse; if(!cDecoder.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVMGRFlowNet::Clear(void) { if(!CNeuronSpikeActivation::Clear()) ReturnFalse; if(!cEncoder.Clear()) ReturnFalse; if(!cNeck.Clear()) ReturnFalse; if(!cDecoder.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEVMGRFlowNet::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeActivation::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); cNeck.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeConvBlock : public CNeuronBatchNormOCL { protected: CNeuronSpikeActivation cActivation; CNeuronConvOCL cConv; CNeuronAddToStack cStack; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeConvBlock(void) {}; ~CNeuronSpikeConvBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeConvBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cConv.GetWindow(); } virtual uint GetFilters(void) const { return cConv.GetFilters(); } virtual uint GetVariables(void) const { return cConv.GetVariables(); } virtual uint GetUnits(void) const { return cConv.GetUnits(); } //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count * variables, batch, optimization_type)) ReturnFalse; activation = None; if(!cActivation.Init(0, 0, OpenCL, (window - step + step * units_count)*variables, optimization, iBatch)) ReturnFalse; if(!cConv.Init(0, 1, OpenCL, window, step, window_out, units_count, variables, optimization, iBatch)) ReturnFalse; cConv.SetActivationFunction(None); if(!cStack.Init(0, 2, OpenCL, 50, cActivation.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cActivation.FeedForward(NeuronOCL)) ReturnFalse; if(!cConv.FeedForward(cActivation.AsObject())) ReturnFalse; if(!CNeuronBatchNormOCL::feedForward(cConv.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronBatchNormOCL::calcInputGradients(cConv.AsObject())) ReturnFalse; if(!cActivation.CalcHiddenGradients(cConv.AsObject())) ReturnFalse; if(!cStack.FeedForward(cActivation.AsObject()) || !DiversityLoss(cStack.AsObject(), cStack.GetStackSize(), cStack.GetDimension(), false) || !SumAndNormilize(cActivation.getGradient(), cStack.getGradient(), cActivation.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cActivation.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cActivation.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cConv.UpdateInputWeights(cActivation.AsObject())) ReturnFalse; if(!CNeuronBatchNormOCL::updateInputWeights(cConv.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::Save(const int file_handle) { if(!CNeuronBatchNormOCL::Save(file_handle)) ReturnFalse; if(!cActivation.Save(file_handle)) ReturnFalse; if(!cConv.Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::Load(const int file_handle) { if(!CNeuronBatchNormOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cActivation.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::Clear(void) { if(!CNeuronBatchNormOCL::Clear()) ReturnFalse; if(!cActivation.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeConvBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBatchNormOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeConvBlock* Source = source; if(!cActivation.WeightsUpdate(Source.cActivation.AsObject(), tau)) ReturnFalse; if(!cConv.WeightsUpdate(Source.cConv.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeConvBlock::SetOpenCL(COpenCLMy *obj) { CNeuronBatchNormOCL::SetOpenCL(obj); cActivation.SetOpenCL(OpenCL); cConv.SetOpenCL(OpenCL); cStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeConvBlock::TrainMode(bool flag) { CNeuronBatchNormOCL::TrainMode(flag); cActivation.TrainMode(bTrain); cConv.TrainMode(bTrain); cStack.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMSRes : public CNeuronBaseOCL { protected: CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMSRes(void) {}; ~CNeuronMSRes(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMSRes; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; virtual uint GetWindow(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetWindow(); } virtual uint GetVariables(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetVariables(); } virtual uint GetUnits(void) const { return ((CNeuronSpikeConvBlock*)cFlow[0]).GetUnits(); } virtual CBufferFloat *getWeights(void) override { return (!!cFlow[-1] ? cFlow[-1].getWeights() : NULL); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units * variables, optimization_type, batch)) ReturnFalse; activation = None; //--- cFlow.Clear(); cFlow.SetOpenCL(OpenCL); CNeuronSpikeConvBlock *block = new CNeuronSpikeConvBlock(); if(!block || !block.Init(0, 0, OpenCL, window, window, 2 * window, units, variables, optimization, iBatch) || !cFlow.Add(block)) { DeleteObj(block) ReturnFalse; } block = new CNeuronSpikeConvBlock(); if(!block || !block.Init(0, 1, OpenCL, 2 * window, 2 * window, window, units, variables, optimization, iBatch) || !cFlow.Add(block)) { DeleteObj(block) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!SumAndNormilize(NeuronOCL.getOutput(), prev.getOutput(), Output, 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; //--- for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; next = cFlow[-1]; if(!DeActivation(NeuronOCL.getOutput(), next.getPrevOutput(), Gradient, NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), next.getPrevOutput(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::WeightsUpdate(CNeuronBaseOCL *source, float tau)\ { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronMSRes* Source = source; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMSRes::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMSRes::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMSRes::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeQKAttention : public CNeuronBaseOCL { protected: CNeuronSpikeConvBlock cQK; CNeuronSpikeActivation cSpikeQK; CNeuronBaseOCL cQ; CNeuronBaseOCL cK; CNeuronMultiWindowsConvOCL cScore; CNeuronSpikeActivation cSpikeScore; CLayer cProject; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeQKAttention(void) {}; ~CNeuronSpikeQKAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeQKAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cQK.Init(0, index, OpenCL, window, window, dimension_k * heads * 2, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cSpikeQK.Init(0, index, OpenCL, cQK.Neurons(), optimization, iBatch)) ReturnFalse; index++; if(!cQ.Init(0, index, OpenCL, cQK.Neurons() / 2, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction((ENUM_ACTIVATION)cSpikeQK.Activation()); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction((ENUM_ACTIVATION)cSpikeQK.Activation()); index++; uint windows[]; if(ArrayResize(windows, heads) < (int)heads) ReturnFalse; ArrayFill(windows, 0, heads, dimension_k); if(!cScore.Init(0, index, OpenCL, windows, 1, units, 1, optimization, iBatch)) ReturnFalse; cScore.SetActivationFunction(None); index++; if(!cSpikeScore.Init(0, index, OpenCL, cScore.Neurons(), optimization, iBatch)) ReturnFalse; //--- cProject.Clear(); cProject.SetOpenCL(OpenCL); index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, cK.Neurons(), optimization, iBatch) || !cProject.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } index++; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_k * heads, dimension_k * heads, window, units, 1, optimization, iBatch) || !cProject.Add(conv)) { DeleteObj(conv) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cQK.FeedForward(NeuronOCL)) ReturnFalse; //--- uint units = cQK.GetUnits(); uint window = cQK.GetWindow(); uint dimension_k = cScore.GetWindow(0); uint hd = cQK.GetFilters() / 2; uint heads = hd / dimension_k; //--- if(!cSpikeQK.FeedForward(cQK.AsObject())) ReturnFalse; if(!DeConcat(cQ.getOutput(), cK.getOutput(), cSpikeQK.getOutput(), hd, hd, units)) ReturnFalse; //--- if(!cScore.FeedForward(cQ.AsObject())) ReturnFalse; if(!cSpikeScore.FeedForward(cScore.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* neuron = cProject[0]; if(!neuron || !ScalarToVector(cSpikeScore.getOutput(), cK.getOutput(), neuron.getOutput(), dimension_k)) ReturnFalse; for(int i = 1; i < cProject.Total(); i++) { neuron = cProject[i]; if(!neuron || !neuron.FeedForward(cProject[i - 1])) ReturnFalse; } //--- if(!SumAndNormilize(NeuronOCL.getOutput(), neuron.getOutput(), Output, window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cQK.GetUnits(); uint window = cQK.GetWindow(); uint dimension_k = cScore.GetWindow(0); uint hd = cQK.GetFilters() / 2; uint heads = hd / dimension_k; //--- CNeuronBaseOCL* neuron = cProject[-1]; if(!neuron || !DeActivation(neuron.getOutput(), neuron.getGradient(), Gradient, neuron.Activation())) ReturnFalse; for(int i = cProject.Total() - 2; i >= 0; i--) { neuron = cProject[i]; if(!neuron || !neuron.CalcHiddenGradients(cProject[i + 1])) ReturnFalse; } if(!ScalarToVectorGrad(cSpikeScore.getOutput(), cSpikeScore.getGradient(), cK.getOutput(), cK.getGradient(), neuron.getGradient(), dimension_k)) ReturnFalse; //--- if(!cScore.CalcHiddenGradients(cSpikeScore.AsObject())) ReturnFalse; if(!cQ.CalcHiddenGradients(cScore.AsObject())) ReturnFalse; //--- if(!Concat(cQ.getGradient(), cK.getGradient(), cSpikeQK.getGradient(), hd, hd, units)) ReturnFalse; if(!cQK.CalcHiddenGradients(cSpikeQK.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cQK.AsObject())) ReturnFalse; neuron = cProject[-1]; if(!DeActivation(NeuronOCL.getOutput(), neuron.getPrevOutput(), Gradient, NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), neuron.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cQK.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSpikeQK.UpdateInputWeights(cQK.AsObject())) ReturnFalse; //--- if(!cScore.UpdateInputWeights(cQ.AsObject())) ReturnFalse; if(!cSpikeScore.UpdateInputWeights(cScore.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* neuron = NULL; for(int i = 1; i < cProject.Total(); i++) { neuron = cProject[i]; if(!neuron || !neuron.UpdateInputWeights(cProject[i - 1])) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cQK.Save(file_handle)) ReturnFalse; if(!cSpikeQK.Save(file_handle)) ReturnFalse; //--- if(!cScore.Save(file_handle)) ReturnFalse; if(!cSpikeScore.Save(file_handle)) ReturnFalse; //--- if(!cProject.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cQK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpikeQK.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScore.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpikeScore.AsObject())) ReturnFalse; if(!cProject.Load(file_handle)) ReturnFalse; //--- uint index = 2; if(!cQ.Init(0, index, OpenCL, cQK.Neurons() / 2, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction((ENUM_ACTIVATION)cSpikeQK.Activation()); index++; if(!cK.Init(0, index, OpenCL, cQ.Neurons(), optimization, iBatch)) ReturnFalse; cK.SetActivationFunction((ENUM_ACTIVATION)cSpikeQK.Activation()); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeQKAttention* Source = source; if(!cQK.WeightsUpdate(Source.cQK.AsObject(), tau)) ReturnFalse; if(!cSpikeQK.WeightsUpdate(Source.cSpikeQK.AsObject(), tau)) ReturnFalse; //--- if(!cScore.WeightsUpdate(Source.cScore.AsObject(), tau)) ReturnFalse; if(!cSpikeScore.WeightsUpdate(Source.cSpikeScore.AsObject(), tau)) ReturnFalse; //--- if(!cProject.WeightsUpdate(Source.cProject.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeQKAttention::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cQK.Clear()) ReturnFalse; if(!cSpikeQK.Clear()) ReturnFalse; //--- if(!cScore.Clear()) ReturnFalse; if(!cSpikeScore.Clear()) ReturnFalse; //--- if(!cProject.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeQKAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cQK.SetOpenCL(OpenCL); cSpikeQK.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cSpikeScore.SetOpenCL(OpenCL); cProject.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeQKAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cQK.TrainMode(bTrain); cSpikeQK.TrainMode(bTrain); cScore.TrainMode(bTrain); cSpikeScore.TrainMode(bTrain); cProject.TrainMode(bTrain); cQ.TrainMode(bTrain); cK.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSDSA : public CNeuronMSRes { protected: CNeuronSpikeQKAttention cAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSDSA(void) {}; ~CNeuronSDSA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSDSA; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint heads, uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, units, window, 1, optimization_type, batch)) ReturnFalse; if(!cAttention.Init(0, 0, OpenCL, units, window, heads, dimension_k, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.FeedForward(NeuronOCL)) ReturnFalse; return CNeuronMSRes::feedForward(cAttention.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMSRes::calcInputGradients(cAttention.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; return CNeuronMSRes::updateInputWeights(cAttention.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; if(!cAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSDSA* Source = source; if(!cAttention.WeightsUpdate(Source.cAttention.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDSA::Clear(void) { if(!CNeuronMSRes::Clear()) ReturnFalse; if(!cAttention.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDSA::SetOpenCL(COpenCLMy *obj) { CNeuronMSRes::SetOpenCL(obj); cAttention.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDSA::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); cAttention.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSDformerUBlock : public CNeuronSpikeConvBlock { protected: CLayer cEncoder; CNeuronBaseOCL* cNeck; CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSDformerUBlock(void) {}; ~CNeuronSDformerUBlock(void) { DeleteObj(cNeck) }; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint &heads[], uint &layers[], uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSDformerUBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; virtual CBufferFloat* GetStack(void); }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint &heads[], uint &layers[], uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch) { uint block = layers.Size(); if(block <= 0 || block > heads.Size() || (block + 1) > windows.Size()) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, 2 * windows[1], 2 * windows[1], windows[0], units, 1, optimization_type, batch)) ReturnFalse; //--- cEncoder.Clear(); cEncoder.SetOpenCL(OpenCL); //--- uint index = 0; CNeuronSDSA* sdsa = NULL; for(uint i = 0; i < layers[0]; i++) { sdsa = new CNeuronSDSA(); if(!sdsa || !sdsa.Init(0, index, OpenCL, units, windows[0], heads[0], dimension_k, optimization, iBatch) || !cEncoder.Add(sdsa)) { DeleteObj(sdsa) ReturnFalse; } index++; } //--- CNeuronSpikeConvBlock* spm = new CNeuronSpikeConvBlock(); if(!spm || !spm.Init(0, index, OpenCL, 2 * windows[0], 2 * windows[0], 2 * windows[1], (units + 1) / 2, 1, optimization, iBatch) || !cEncoder.Add(spm)) { DeleteObj(spm) ReturnFalse; } index++; //--- if(block == 1) { CNeuronMSRes* neck = new CNeuronMSRes(); if(!neck || !neck.Init(0, index, OpenCL, units, windows[1], 1, optimization, iBatch)) { DeleteObj(neck) ReturnFalse; } cNeck = neck; } else { uint new_layers[], new_windows[], new_heads[]; if(ArrayResize(new_layers, block - 1) < 0 || ArrayResize(new_windows, block) < 0 || ArrayResize(new_heads, block - 1) < 0) ReturnFalse; if(ArrayCopy(new_heads, heads, 0, 1, block - 1) < (int(block) - 1) || ArrayCopy(new_layers, layers, 0, 1, block - 1) < (int(block) - 1) || ArrayCopy(new_windows, windows, 0, 1, block) < int(block)) ReturnFalse; CNeuronSDformerUBlock* neck = new CNeuronSDformerUBlock(); if(!neck || !neck.Init(0, index, OpenCL, units, new_windows, new_heads, new_layers, dimension_k, optimization, iBatch)) { DeleteObj(neck) ReturnFalse; } cNeck = neck; } index++; //--- if(!cConcat.Init(0, index, OpenCL, 2 * windows[1]*units, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *curr = NULL; for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!cNeck || !cNeck.FeedForward(prev)) ReturnFalse; if(!Concat(prev.getOutput(), cNeck.getOutput(), cConcat.getOutput(), GetWindow() / 2, GetWindow() / 2, GetUnits())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cNeck) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(cConcat.getPrevOutput(), cNeck.getGradient(), cConcat.getGradient(), GetWindow() / 2, GetWindow() / 2, GetUnits())) ReturnFalse; Deactivation(cNeck) //--- CNeuronBaseOCL* curr = cEncoder[-1]; if(!curr || !curr.CalcHiddenGradients(cNeck)) ReturnFalse; if(!DeActivation(curr.getOutput(), cConcat.getPrevOutput(), cConcat.getPrevOutput(), curr.Activation()) || !SumAndNormilize(curr.getGradient(), cConcat.getPrevOutput(), curr.getGradient(), GetWindow() / 2, false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = cEncoder.Total() - 2; i >= 0; i--) { curr = cEncoder[i]; if(!curr || !curr.CalcHiddenGradients(cEncoder[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *prev = NeuronOCL; CNeuronBaseOCL *curr = NULL; for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!cNeck || !cNeck.UpdateInputWeights(prev)) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cNeck || !cNeck.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(!cEncoder.Load(file_handle)) ReturnFalse; //--- DeleteObj(cNeck) CNeuronMSRes* msres = NULL; CNeuronSDformerUBlock* ublock = NULL; uint new_layers[] = {1}, new_windows[] = {1, 1}, new_heads[] = {1}; switch(FileReadInteger(file_handle)) { case defNeuronMSRes: msres = new CNeuronMSRes(); if(!msres || !msres.Init(0, cEncoder.Total(), OpenCL, 3, 2, 1, optimization, iBatch)) { DeleteObj(msres) ReturnFalse; } cNeck = msres; break; case defNeuronSDformerUBlock: ublock = new CNeuronSDformerUBlock(); if(!ublock || !ublock.Init(0, cEncoder.Total(), OpenCL, 3, new_windows, new_heads, new_layers, 3, optimization, iBatch)) { DeleteObj(ublock) ReturnFalse; } cNeck = ublock; break; default: ReturnFalse; break; } if(!cNeck || !cNeck.Load(file_handle)) ReturnFalse; //--- if(!cConcat.Init(0, cEncoder.Total() + 1, OpenCL, GetWindow()*GetUnits(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSDformerUBlock* Source = source; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; if(!cNeck || !cNeck.WeightsUpdate(Source.cNeck.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerUBlock::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cEncoder.ClearStates()) ReturnFalse; if(!cNeck || !cNeck.Clear()) ReturnFalse; //--- if(!cConcat.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDformerUBlock::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); //--- cEncoder.SetOpenCL(OpenCL); if(!!cNeck) cNeck.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDformerUBlock::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- cEncoder.TrainMode(bTrain); if(!!cNeck) cNeck.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronSDformerUBlock::GetStack(void) { CBufferFloat* result = new CBufferFloat(); if(!result) return NULL; //--- uint window = GetFilters(); uint units = GetUnits(); //--- if(!cNeck || cNeck.Type() != Type()) { if(!result.BufferInit(window * units, 0) || !result.BufferCreate(OpenCL)) { DeleteObj(result) return NULL; } if(!Concat(Output, Output, result, window, 0, units)) { DeleteObj(result) return NULL; } return result; } //--- CNeuronSDformerUBlock* block = cNeck; CBufferFloat* neck = block.GetStack(); uint neck_window = neck.Total() / units; if(!result.BufferInit((window + neck_window)*units, 0) || !result.BufferCreate(OpenCL)) { DeleteObj(result) DeleteObj(neck) return NULL; } if(!Concat(Output, neck, result, window, neck_window, units)) { DeleteObj(neck) DeleteObj(result) return NULL; } DeleteObj(neck) //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSDformerFlow : public CNeuronSDformerUBlock { protected: CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSDformerFlow(void) {}; ~CNeuronSDformerFlow(void) {}; virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint &heads[], uint &layers[], uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSDformerFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint &heads[], uint &layers[], uint dimension_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(layers.Size() < 3) ReturnFalse; if(heads.Size() != (layers.Size() - 2)) ReturnFalse; if(windows.Size() < (layers[0] + layers.Size() - 1)) ReturnFalse; //--- uint block = layers.Size() - 2; uint new_layers[], new_windows[]; if(ArrayResize(new_layers, block) < 0 || ArrayResize(new_windows, block + 1) < 0) ReturnFalse; if(ArrayCopy(new_layers, layers, 0, 2, block) < int(block) || ArrayCopy(new_windows, windows, 0, layers[0], block + 1) < int(block + 1)) ReturnFalse; //--- if(!CNeuronSDformerUBlock::Init(numOutputs, myIndex, open_cl, units, new_windows, heads, new_layers, dimension_k, optimization_type, batch)) ReturnFalse; //--- cFlow.Clear(); cFlow.SetOpenCL(OpenCL); //--- CNeuronSpikeConvBlock* conv = NULL; uint index = 0; for(uint i = 0; i < layers[0]; i++) { conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, windows[i], windows[i], windows[i + 1], units, 1, optimization, iBatch) || !cFlow.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; } //--- CNeuronMSRes* msres = NULL; for(uint i = 0; i < layers[1]; i++) { msres = new CNeuronMSRes(); if(!msres || !msres.Init(0, index, OpenCL, units, windows[layers[0]], 1, optimization, iBatch) || !cFlow.Add(msres)) { DeleteObj(msres) ReturnFalse; } index++; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- return CNeuronSDformerUBlock::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSDformerUBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(cFlow[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return CNeuronSDformerUBlock::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::Save(const int file_handle) { if(!CNeuronSDformerUBlock::Save(file_handle)) ReturnFalse; //--- if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::Load(const int file_handle) { if(!CNeuronSDformerUBlock::Load(file_handle)) ReturnFalse; //--- if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::Clear(void) { if(!CNeuronSDformerUBlock::Clear()) ReturnFalse; //--- if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSDformerFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSDformerUBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSDformerFlow* Source = source; if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDformerFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSDformerUBlock::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSDformerFlow::TrainMode(bool flag) { CNeuronSDformerUBlock::TrainMode(flag); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikePatchStak : public CNeuronAddToStack { protected: CNeuronSpikeConvBlock cBlock; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikePatchStak(void) {}; ~CNeuronSpikePatchStak(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikePatchStak; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAddToStack::Init(numOutputs, myIndex, open_cl, stack_size, patch_dimension, variables, optimization_type, batch)) ReturnFalse; if(!cBlock.Init(0, 0, OpenCL, dimension, dimension, patch_dimension, variables, 1, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cBlock.FeedForward(NeuronOCL)) ReturnFalse; //--- return CNeuronAddToStack::feedForward(cBlock.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronAddToStack::calcInputGradients(cBlock.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cBlock.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cBlock.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return CNeuronAddToStack::updateInputWeights(cBlock.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronAddToStack::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikePatchStak* Source = source; if(!cBlock.WeightsUpdate(Source.cBlock.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::Save(const int file_handle) { if(!CNeuronAddToStack::Save(file_handle)) ReturnFalse; if(!cBlock.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::Load(const int file_handle) { if(!CNeuronAddToStack::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cBlock.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikePatchStak::Clear(void) { if(!CNeuronAddToStack::Clear()) ReturnFalse; if(!cBlock.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikePatchStak::SetOpenCL(COpenCLMy *obj) { CNeuronAddToStack::SetOpenCL(obj); cBlock.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikePatchStak::TrainMode(bool flag) { CNeuronAddToStack::TrainMode(flag); cBlock.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpike2DSSMOCL : public CNeuronBaseOCL { protected: uint iWindowOut; uint iUnitsOut; //--- CNeuronBaseOCL cHiddenStates; CLayer cProjectionX_Time; CLayer cProjectionX_Variable; CNeuronSpikeConvBlock cA; CNeuronSpikeConvBlock cBC_Time; CNeuronSpikeConvBlock cBC_Variable; CNeuronBaseOCL cB_Time; CNeuronBaseOCL cB_Variable; CNeuronBaseOCL cC_Time; CNeuronBaseOCL cC_Variable; CNeuronSpikeConvBlock cDelta_Time; CNeuronSpikeConvBlock cDelta_Variable; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForwardSSM2D(void); //--- virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradientsSSM2D(void); virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpike2DSSMOCL(void) {}; ~CNeuronSpike2DSSMOCL(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSpike2DSSMOCL; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::Init(uint numOutputs, uint myIndex, COpenCLMy * open_cl, uint window_in, uint window_out, uint units_in, uint units_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * units_out, optimization_type, batch)) ReturnFalse; SetActivationFunction(None); //--- iWindowOut = window_out; iUnitsOut = units_out; //--- int index = 0; CNeuronSpikeConvBlock *conv = NULL; CNeuronTransposeOCL *transp = NULL; //--- Projection Time cProjectionX_Time.Clear(); cProjectionX_Time.SetOpenCL(OpenCL); transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, window_in, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iUnitsOut, window_in, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, window_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Time.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Time.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- Projection Variables cProjectionX_Variable.Clear(); cProjectionX_Variable.SetOpenCL(OpenCL); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, iUnitsOut, units_in, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, iUnitsOut, optimization, iBatch) || !cProjectionX_Variable.Add(transp)) { DeleteObj(transp); ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, iWindowOut, iUnitsOut, 1, optimization, iBatch) || !cProjectionX_Variable.Add(conv)) { DeleteObj(conv); ReturnFalse; } //--- HiddenState index++; if(!cHiddenStates.Init(0, index, OpenCL, 2 * iUnitsOut * iWindowOut, optimization, iBatch)) ReturnFalse; //--- A*H index++; if(!cA.Init(0, index, OpenCL, iWindowOut, iWindowOut, 2 * iWindowOut, iUnitsOut, 2, optimization, iBatch)) ReturnFalse; //--- BC index++; if(!cBC_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut + 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; index++; if(!cBC_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut + 1, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; //--- B index++; if(!cB_Time.Init(0, index, OpenCL, iUnitsOut, optimization, iBatch)) ReturnFalse; cB_Time.SetActivationFunction(None); index++; if(!cB_Variable.Init(0, index, OpenCL, iUnitsOut, optimization, iBatch)) ReturnFalse; cB_Variable.SetActivationFunction(None); //--- C index++; if(!cC_Time.Init(0, index, OpenCL, iUnitsOut * iUnitsOut, optimization, iBatch)) ReturnFalse; cC_Time.SetActivationFunction(None); index++; if(!cC_Variable.Init(0, index, OpenCL, iUnitsOut * iUnitsOut, optimization, iBatch)) ReturnFalse; cC_Variable.SetActivationFunction(None); //--- Delta index++; if(!cDelta_Time.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; index++; if(!cDelta_Variable.Init(0, index, OpenCL, iWindowOut, iWindowOut, iUnitsOut, iUnitsOut, 1, optimization, iBatch)) ReturnFalse; //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::feedForward(CNeuronBaseOCL * NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; //--- Projection Time int total = cProjectionX_Time.Total(); for(int i = 0; i < total; i++) { x_time = cProjectionX_Time.At(i); if(!x_time || !x_time.FeedForward(inp)) ReturnFalse; inp = x_time; } //--- Projection Variable inp = NeuronOCL; total = cProjectionX_Variable.Total(); for(int i = 0; i < total; i++) { x_var = cProjectionX_Variable.At(i); if(!x_var || !x_var.FeedForward(inp)) ReturnFalse; inp = x_var; } //--- if(!cA.FeedForward(cHiddenStates.AsObject())) ReturnFalse; if(!cBC_Time.FeedForward(x_time) || !cBC_Variable.FeedForward(x_var)) ReturnFalse; if(!DeConcat(cB_Time.getOutput(), cC_Time.getOutput(), cBC_Time.getOutput(), 1, iUnitsOut, iUnitsOut) || !DeConcat(cB_Variable.getOutput(), cC_Variable.getOutput(), cBC_Variable.getOutput(), 1, iUnitsOut, iUnitsOut)) ReturnFalse; //--- if(!cDelta_Time.FeedForward(x_time) || !cDelta_Variable.FeedForward(x_var)) ReturnFalse; //--- if(!cHiddenStates.SwapOutputs()) ReturnFalse; //--- return feedForwardSSM2D(); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::feedForwardSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {global_work_size[0], 1}; int kernel = def_k_SSM2D_FeedForward; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2d_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2d_y, getOutputIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::calcInputGradientsSSM2D(void) { CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!OpenCL || !x_time || !x_var) ReturnFalse; //--- uint global_work_offset[2] = {0, 0}; uint global_work_size[2] = {iUnitsOut, iWindowOut}; uint local_work_size[2] = {1, global_work_size[1]}; int kernel = def_k_SSM2D_CalcHiddenGradient; //--- ResetLastError(); setBuffer(kernel, def_k_ssm2dhg_ah, cA.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_ah, cA.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_time, cB_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_time, cB_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_b_var, cB_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_b_var, cB_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_time, cC_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_time, cC_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_c_var, cC_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_c_var, cC_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_time, cDelta_Time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_time, cDelta_Time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_delta_var, cDelta_Variable.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_delta_var, cDelta_Variable.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_hidden, cHiddenStates.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_px_time, x_time.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_time, x_time.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_px_var, x_var.getOutputIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_px_var, x_var.getGradientIndex()) setBuffer(kernel, def_k_ssm2dhg_grad_y, getGradientIndex()) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::calcInputGradients(CNeuronBaseOCL * NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!calcInputGradientsSSM2D()) ReturnFalse; //--- if(!Concat(cB_Time.getGradient(), cC_Time.getGradient(), cBC_Time.getGradient(), 1, iUnitsOut, iUnitsOut) || !Concat(cB_Variable.getGradient(), cC_Variable.getGradient(), cBC_Variable.getGradient(), 1, iUnitsOut, iUnitsOut)) ReturnFalse; //--- Deactivation CNeuronBaseOCL *x_time = cProjectionX_Time[-1]; CNeuronBaseOCL *x_var = cProjectionX_Variable[-1]; if(!x_time || !x_var) ReturnFalse; Deactivation(x_time) Deactivation(x_var) Deactivation(cBC_Time) Deactivation(cBC_Variable) Deactivation(cDelta_Time) Deactivation(cDelta_Variable) Deactivation(cA) //--- Gradient to projections X CBufferFloat *grad_x_time = x_time.getGradient(); CBufferFloat *grad_x_var = x_var.getGradient(); if(!x_time.SetGradient(x_time.getPrevOutput(), false) || !x_var.SetGradient(x_var.getPrevOutput(), false)) ReturnFalse; //--- BC -> X if(!x_time.CalcHiddenGradients(cBC_Time.AsObject()) || !SumAndNormilize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cBC_Variable.AsObject()) || !SumAndNormilize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; //--- Delta -> X if(!x_time.CalcHiddenGradients(cDelta_Time.AsObject()) || !SumAndNormilize(grad_x_time, x_time.getGradient(), grad_x_time, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_var.CalcHiddenGradients(cDelta_Variable.AsObject()) || !SumAndNormilize(grad_x_var, x_var.getGradient(), grad_x_var, iWindowOut, false, 0, 0, 0, 1)) ReturnFalse; if(!x_time.SetGradient(grad_x_time, false) || !x_var.SetGradient(grad_x_var, false)) ReturnFalse; //--- Projection Variable int total = cProjectionX_Variable.Total() - 2; for(int i = total; i >= 0; i--) { x_var = cProjectionX_Variable[i]; if(!x_var || !x_var.CalcHiddenGradients(cProjectionX_Variable[i + 1])) ReturnFalse; } //--- Projection Time total = cProjectionX_Time.Total() - 2; for(int i = total; i >= 0; i--) { x_time = cProjectionX_Time[i]; if(!x_time || !x_time.CalcHiddenGradients(cProjectionX_Time[i + 1])) ReturnFalse; } //--- Projections -> inputs if(!NeuronOCL.CalcHiddenGradients(x_var.AsObject())) ReturnFalse; grad_x_time = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(x_time.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(x_time.AsObject()) || !SumAndNormilize(grad_x_time, NeuronOCL.getGradient(), grad_x_time, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(grad_x_time, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL *inp = NeuronOCL; CNeuronBaseOCL *x_time = NULL; CNeuronBaseOCL *x_var = NULL; //--- Projection Time int total = cProjectionX_Time.Total(); for(int i = 0; i < total; i++) { x_time = cProjectionX_Time.At(i); if(!x_time || !x_time.UpdateInputWeights(inp)) ReturnFalse; inp = x_time; } //--- Projection Variable inp = NeuronOCL; total = cProjectionX_Variable.Total(); for(int i = 0; i < total; i++) { x_var = cProjectionX_Variable.At(i); if(!x_var || !x_var.UpdateInputWeights(inp)) ReturnFalse; inp = x_var; } //--- cHiddenStates.SwapOutputs(); if(!cA.UpdateInputWeights(cHiddenStates.AsObject())) ReturnFalse; cHiddenStates.SwapOutputs(); if(!cBC_Time.UpdateInputWeights(x_time) || !cBC_Variable.UpdateInputWeights(x_var)) ReturnFalse; //--- if(!cDelta_Time.UpdateInputWeights(x_time) || !cDelta_Variable.UpdateInputWeights(x_var)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpike2DSSMOCL* Source = source; //--- Projections if(!cProjectionX_Time.WeightsUpdate(Source.cProjectionX_Time.AsObject(), tau) || !cProjectionX_Variable.WeightsUpdate(Source.cProjectionX_Variable.AsObject(), tau)) ReturnFalse; //--- if(!cA.WeightsUpdate(Source.cA.AsObject(), tau)) ReturnFalse; if(!cBC_Time.WeightsUpdate(Source.cBC_Time.AsObject(), tau) || !cBC_Variable.WeightsUpdate(Source.cBC_Variable.AsObject(), tau)) ReturnFalse; if(!cDelta_Time.WeightsUpdate(Source.cDelta_Time.AsObject(), tau) || !cDelta_Variable.WeightsUpdate(Source.cDelta_Variable.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cProjectionX_Time.Save(file_handle) || !cProjectionX_Variable.Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; if(!cBC_Time.Save(file_handle) || !cBC_Variable.Save(file_handle)) ReturnFalse; if(!cDelta_Time.Save(file_handle) || !cDelta_Variable.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cProjectionX_Time.Load(file_handle) || !cProjectionX_Variable.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cBC_Time.AsObject()) || !LoadInsideLayer(file_handle, cBC_Variable.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDelta_Time.AsObject()) || !LoadInsideLayer(file_handle, cDelta_Variable.AsObject())) ReturnFalse; //--- iWindowOut = cA.GetWindow(); iUnitsOut = cA.GetUnits(); //--- HiddenState int index = cProjectionX_Time.Total() + cProjectionX_Variable.Total(); if(!cHiddenStates.Init(0, index, OpenCL, 2 * iUnitsOut * iWindowOut, optimization, iBatch)) ReturnFalse; //--- B index += 4; if(!cB_Time.Init(0, index, OpenCL, iUnitsOut, optimization, iBatch)) ReturnFalse; cB_Time.SetActivationFunction(None); index++; if(!cB_Variable.Init(0, index, OpenCL, iUnitsOut, optimization, iBatch)) ReturnFalse; cB_Variable.SetActivationFunction(None); //--- C index++; if(!cC_Time.Init(0, index, OpenCL, iUnitsOut * iUnitsOut, optimization, iBatch)) ReturnFalse; cC_Time.SetActivationFunction(None); index++; if(!cC_Variable.Init(0, index, OpenCL, iUnitsOut * iUnitsOut, optimization, iBatch)) ReturnFalse; cC_Variable.SetActivationFunction(None); //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpike2DSSMOCL::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cProjectionX_Time.ClearStates() || !cProjectionX_Variable.ClearStates()) ReturnFalse; if(!cA.Clear()) ReturnFalse; if(!cBC_Time.Clear() || !cBC_Variable.Clear()) ReturnFalse; if(!cDelta_Time.Clear() || !cDelta_Variable.Clear()) ReturnFalse; //--- HiddenState if(!cHiddenStates.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpike2DSSMOCL::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cProjectionX_Time.SetOpenCL(OpenCL); cProjectionX_Variable.SetOpenCL(OpenCL); cA.SetOpenCL(OpenCL); cBC_Time.SetOpenCL(OpenCL); cBC_Variable.SetOpenCL(OpenCL); cDelta_Time.SetOpenCL(OpenCL); cDelta_Variable.SetOpenCL(OpenCL); cHiddenStates.SetOpenCL(OpenCL); cB_Time.SetOpenCL(OpenCL); cB_Variable.SetOpenCL(OpenCL); cC_Time.SetOpenCL(OpenCL); cC_Variable.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpike2DSSMOCL::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cProjectionX_Time.TrainMode(bTrain); cProjectionX_Variable.TrainMode(bTrain); cA.TrainMode(bTrain); cBC_Time.TrainMode(bTrain); cBC_Variable.TrainMode(bTrain); cDelta_Time.TrainMode(bTrain); cDelta_Variable.TrainMode(bTrain); cHiddenStates.TrainMode(bTrain); cB_Time.TrainMode(bTrain); cB_Variable.TrainMode(bTrain); cC_Time.TrainMode(bTrain); cC_Variable.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeMamba2D : public CNeuronSpikeConvBlock { protected: CNeuronSpikeConvBlock cXZProject; CNeuronBaseOCL cX; CNeuronBaseOCL cZ; CNeuronSpike2DSSMOCL cSSM; CNeuronBaseOCL cZSSM; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeMamba2D(void) {}; ~CNeuronSpikeMamba2D(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint inside_dimension, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSpikeMamba2D; } //--- virtual bool Save(int const file_handle); virtual bool Load(int const file_handle); //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj); //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint inside_dimension, uint units_count, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, inside_dimension, inside_dimension, window, units_count, 1, optimization_type, batch)) ReturnFalse; if(!cXZProject.Init(0, 0, OpenCL, window, window, 2 * inside_dimension, units_count, 1, optimization, iBatch)) ReturnFalse; if(!cX.Init(0, 1, OpenCL, inside_dimension * units_count, optimization, iBatch)) ReturnFalse; cX.SetActivationFunction(None); if(!cZ.Init(0, 2, OpenCL, inside_dimension * units_count, optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction(None); if(!cSSM.Init(0, 3, OpenCL, inside_dimension, inside_dimension, units_count, units_count, optimization, iBatch)) ReturnFalse; if(!cZSSM.Init(0, 4, OpenCL, inside_dimension * units_count, optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::feedForward(CNeuronBaseOCL * NeuronOCL) { if(!cXZProject.FeedForward(NeuronOCL)) ReturnFalse; if(!DeConcat(cX.getOutput(), cZ.getOutput(), cXZProject.getOutput(), cXZProject.GetFilters() / 2, cXZProject.GetFilters() / 2, cXZProject.GetUnits())) ReturnFalse; if(!cSSM.FeedForward(cX.AsObject())) ReturnFalse; if(!ElementMult(cSSM.getOutput(), cZ.getOutput(), cZSSM.getOutput())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeConvBlock::calcInputGradients(cZSSM.AsObject())) ReturnFalse; if(!ElementMultGrad(cSSM.getOutput(), cSSM.getGradient(), cZ.getOutput(), cZ.getGradient(), cZSSM.getGradient(), cSSM.Activation(), cZ.Activation())) ReturnFalse; if(!cX.CalcHiddenGradients(cSSM.AsObject())) ReturnFalse; if(!Concat(cX.getGradient(), cZ.getGradient(), cXZProject.getGradient(), cXZProject.GetFilters() / 2, cXZProject.GetFilters() / 2, cXZProject.GetUnits())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cXZProject.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cXZProject.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cSSM.UpdateInputWeights(cX.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cZSSM.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeMamba2D* Source = source; if(!cXZProject.WeightsUpdate(Source.cXZProject.AsObject(), tau)) ReturnFalse; if(!cSSM.WeightsUpdate(Source.cSSM.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cXZProject.Save(file_handle)) ReturnFalse; if(!cSSM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cXZProject.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cSSM.AsObject())) ReturnFalse; //--- if(!cX.Init(0, 1, OpenCL, cSSM.Neurons(), optimization, iBatch)) ReturnFalse; cX.SetActivationFunction(None); if(!cZ.Init(0, 2, OpenCL, cSSM.Neurons(), optimization, iBatch)) ReturnFalse; cZ.SetActivationFunction(None); if(!cZSSM.Init(0, 4, OpenCL, cSSM.Neurons(), optimization, iBatch)) ReturnFalse; cZSSM.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMamba2D::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cXZProject.Clear()) ReturnFalse; if(!cSSM.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMamba2D::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cXZProject.SetOpenCL(OpenCL); cSSM.SetOpenCL(OpenCL); cX.SetOpenCL(OpenCL); cZ.SetOpenCL(OpenCL); cZSSM.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSTSSM : public CNeuronSpikeConvBlock { protected: CNeuronSpikePatchStak cEmbeddings; CLayer cProjection; CLayer cSSM; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeSTSSM(void) {}; ~CNeuronSpikeSTSSM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint ssm_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSTSSM; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool Clear(void) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint ssm_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, ssm_dimension * (stack_size - 2), ssm_dimension * (stack_size - 2), dimension, variables, 1, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cEmbeddings.Init(0, index, OpenCL, stack_size, dimension, patch_dimension, variables, optimization, iBatch)) ReturnFalse; //--- cProjection.Clear(); cProjection.SetOpenCL(OpenCL); CNeuronSpikeConvBlock* conv = NULL; index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 2 * patch_dimension, patch_dimension, patch_dimension, stack_size - 1, variables, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 2 * patch_dimension, patch_dimension, ssm_dimension, stack_size - 2, variables, optimization, iBatch) || !cProjection.Add(conv)) { DeleteObj(conv) ReturnFalse; } //--- cSSM.Clear(); cSSM.SetOpenCL(OpenCL); CNeuronSpikeMamba2D* ssm = NULL; for(int i = 0; i < 2; i++) { index++; ssm = new CNeuronSpikeMamba2D(); if(!ssm || !ssm.Init(0, index, OpenCL, ssm_dimension, ssm_dimension, (stack_size - 2)*variables, optimization, iBatch) || !cSSM.Add(ssm)) { DeleteObj(ssm) ReturnFalse; } } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- if(!cEmbeddings.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cEmbeddings.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint units = GetUnits() * GetVariables(); for(int i = 0; i < cSSM.Total(); i++) { curr = cSSM[i]; if(!curr || !curr.FeedForward(prev) || !SumAndNormilize(curr.getOutput(), prev.getOutput(), curr.getOutput(), curr.Neurons() / units, true, 0, 0, 0, 1)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cSSM[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; uint units = GetUnits() * GetVariables(); for(int i = cSSM.Total() - 2; i >= 0; i--) { curr = cSSM[i]; if(!curr || !curr.CalcHiddenGradients(cSSM[i + 1]) || !SumAndNormilize(curr.getGradient(), cSSM[i + 1].getGradient(), curr.getGradient(), curr.Neurons() / units, false, 0, 0, 0, 1)) ReturnFalse; } //--- CNeuronBaseOCL* next = curr; for(int i = cProjection.Total() - 2; i >= 0; i--) { curr = cProjection[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!cEmbeddings.CalcHiddenGradients(next)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cEmbeddings.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- if(!cEmbeddings.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cEmbeddings.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- uint units = GetUnits() * GetVariables(); for(int i = 0; i < cSSM.Total(); i++) { curr = cSSM[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(!cEmbeddings.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; if(!cSSM.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cEmbeddings.AsObject())) ReturnFalse; if(!cProjection.Load(file_handle)) ReturnFalse; if(!cSSM.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSTSSM* Source = source; if(!cEmbeddings.WeightsUpdate(Source.cEmbeddings.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; if(!cSSM.WeightsUpdate(Source.cSSM.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSTSSM::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cEmbeddings.Clear()) ReturnFalse; if(!cProjection.ClearStates()) ReturnFalse; if(!cSSM.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSTSSM::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); //--- cEmbeddings.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cSSM.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSTSSM::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- cEmbeddings.TrainMode(bTrain); cProjection.TrainMode(bTrain); cSSM.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronESTMEncoder : public CNeuronSpikeConvBlock { protected: CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronESTMEncoder(void) {}; ~CNeuronESTMEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &stack_size[], uint &dimension[], uint patch_dimension, uint ssm_dimension, uint &variables[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronESTMEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool Clear(void) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &stack_size[], uint &dimension[], uint patch_dimension, uint ssm_dimension, uint &variables[], ENUM_OPTIMIZATION optimization_type, uint batch) { uint layers = stack_size.Size(); if(layers < 1 || dimension.Size() != (layers + 1) || variables.Size() != (layers + 1)) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, dimension[layers - 1], dimension[layers - 1], dimension[layers], variables[layers], 1, optimization_type, batch)) ReturnFalse; //--- cFlow.Clear();; cFlow.SetOpenCL(OpenCL); CNeuronTransposeOCL* transp = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronSpikeSTSSM* ssm = NULL; uint index = 0; for(uint i = 0; i < layers; i++) { ssm = new CNeuronSpikeSTSSM(); if(!ssm || !ssm.Init(0, index, OpenCL, stack_size[i], dimension[i], patch_dimension, ssm_dimension, variables[i], optimization, iBatch) || !cFlow.Add(ssm)) { DeleteObj(ssm) ReturnFalse; } index++; if(variables[i] != variables[i + 1]) { transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, variables[i], dimension[i], optimization, iBatch) || !cFlow.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, variables[i], variables[i], variables[i + 1], dimension[i], 1, optimization, iBatch) || !cFlow.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, dimension[i], variables[i + 1], optimization, iBatch) || !cFlow.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; } if(i == (layers - 1)) break; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension[i], dimension[i], dimension[i + 1], variables[i + 1], 1, optimization, iBatch) || !cFlow.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; if(curr.Type() == defNeuronSpikeSTSSM) if(!SumAndNormilize(prev.getOutput(), curr.getOutput(), curr.getOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; prev = curr; } //--- return CNeuronSpikeConvBlock::feedForward(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; if(next.Type() == defNeuronSpikeSTSSM) { if(curr.Activation() != None) { if(!DeActivation(curr.getOutput(), next.getPrevOutput(), next.getGradient(), curr.Activation()) || !SumAndNormilize(next.getPrevOutput(), curr.getGradient(), curr.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormilize(next.getGradient(), curr.getGradient(), curr.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; if(next.Type() == defNeuronSpikeSTSSM) { if(NeuronOCL.Activation() != None) { if(!DeActivation(NeuronOCL.getOutput(), next.getPrevOutput(), next.getGradient(), NeuronOCL.Activation()) || !SumAndNormilize(next.getPrevOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormilize(next.getGradient(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return CNeuronSpikeConvBlock::updateInputWeights(prev); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronESTMEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- if(!cFlow.WeightsUpdate(((CNeuronESTMEncoder*)source).cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronESTMEncoder::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronESTMEncoder::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCreateFlow : public CNeuronSpikePatchStak { protected: CNeuronBaseOCL cState; //--- virtual bool CalcFlow(CNeuronBaseOCL *NeuronOCL); virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCreateFlow(void) {}; ~CNeuronCreateFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCreateFlow; } //--- methods for working with files virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; //--- virtual CNeuronBaseOCL* GetState(void) { return cState.AsObject(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikePatchStak::Init(numOutputs, myIndex, open_cl, stack_size, dimension, patch_dimension, variables, optimization_type, batch)) ReturnFalse; if(!cState.Init(0, 0, OpenCL, dimension * variables, optimization, iBatch)) ReturnFalse; if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::CalcFlow(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !OpenCL) ReturnFalse; if(NeuronOCL.Neurons() > cState.Neurons()) ReturnFalse; uint global_work_offset[1] = {0}; uint global_work_size[1] = { NeuronOCL.Neurons() }; uint kernel = def_k_CalcFlow; setBuffer(kernel, def_k_cf_value, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_cf_prev_value, cState.getPrevOutIndex()) setBuffer(kernel, def_k_cf_flow, cState.getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!CalcFlow(NeuronOCL)) ReturnFalse; //--- return CNeuronSpikePatchStak::feedForward(cState.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikePatchStak::calcInputGradients(cState.AsObject())) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cState.getGradient(), NeuronOCL.Activation())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return CNeuronSpikePatchStak::updateInputWeights(cState.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::Load(const int file_handle) { if(!CNeuronSpikePatchStak::Load(file_handle)) ReturnFalse; if(!cState.Init(0, 0, OpenCL, cBlock.GetWindow() * cBlock.GetUnits() * cBlock.GetVariables(), optimization, iBatch)) ReturnFalse; if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateFlow::Clear(void) { if(!CNeuronSpikePatchStak::Clear()) ReturnFalse; if(!cState.Clear() || !cState.getPrevOutput() || !cState.getPrevOutput().Fill(0)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCreateFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSpikePatchStak::SetOpenCL(obj); cState.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBiDirectCorrelation : public CNeuronTMAMFE { protected: CNeuronBatchNormOCL cInpNorm; CNeuronBaseOCL cRefFrame; CNeuronTransposeOCL cRefFrameT; CNeuronMultiWindowsConvOCL cWarp; CNeuronBatchNormOCL cWarpNorm; CNeuronBaseOCL cCorrelation; CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronBiDirectCorrelation(void) {}; ~CNeuronBiDirectCorrelation(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint variables, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronBiDirectCorrelation; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint variables, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch) { { uint windows[] = {dimension, 1}; if(!CNeuronTMAMFE::Init(numOutputs, myIndex, open_cl, units * variables, windows, window_out, optimization_type, batch)) ReturnFalse; } //--- uint index = 0; if(!cInpNorm.Init(0, index, OpenCL, variables * dimension * units, iBatch, optimization)) ReturnFalse; cInpNorm.SetActivationFunction(None); index++; if(!cRefFrame.Init(0, index, OpenCL, variables * dimension, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); index++; if(!cRefFrameT.Init(0, index, OpenCL, variables, dimension, optimization, iBatch)) ReturnFalse; cRefFrameT.SetActivationFunction(None); { uint windows[]; if(ArrayResize(windows, units) < (int)units) ReturnFalse; ArrayFill(windows, 0, units, dimension); index++; if(!cWarp.Init(0, index, OpenCL, windows, dimension, variables, 1, optimization, iBatch)) ReturnFalse; cWarp.SetActivationFunction(None); } index++; if(!cWarpNorm.Init(0, index, OpenCL, cWarp.Neurons(), iBatch, optimization)) ReturnFalse; cWarpNorm.SetActivationFunction(None); index++; if(!cCorrelation.Init(0, index, OpenCL, units * variables, optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); index++; if(!cConcat.Init(0, index, OpenCL, (dimension + 1)*units * variables, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cInpNorm.FeedForward(NeuronOCL)) ReturnFalse; //--- uint dimension = cRefFrameT.GetWindow(); uint variables = cRefFrameT.GetCount(); uint units = cCorrelation.Neurons() / variables; uint ref_position = (units + 1) / 2; if(!DeConcat(cConcat.getOutput(), cRefFrame.getOutput(), cConcat.getOutput(), cInpNorm.getOutput(), (ref_position - 1)*dimension, dimension, (units - ref_position)*dimension, variables)) ReturnFalse; if(!cRefFrameT.FeedForward(cRefFrame.AsObject())) ReturnFalse; if(!cWarp.FeedForward(cInpNorm.AsObject())) ReturnFalse; if(!cWarpNorm.FeedForward(cWarp.AsObject())) ReturnFalse; if(!MatMul(cWarpNorm.getOutput(), cRefFrameT.getOutput(), cCorrelation.getOutput(), units, dimension, 1, variables, true)) ReturnFalse; if(!Concat(cInpNorm.getOutput(), cCorrelation.getOutput(), cConcat.getOutput(), dimension, 1, units * variables)) ReturnFalse; if(!CNeuronTMAMFE::feedForward(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint dimension = cRefFrameT.GetWindow(); uint variables = cRefFrameT.GetCount(); uint units = cCorrelation.Neurons() / variables; uint ref_position = (units + 1) / 2; //--- if(!CNeuronTMAMFE::calcInputGradients(cConcat.AsObject())) ReturnFalse; if(!DeConcat(cInpNorm.getPrevOutput(), cCorrelation.getGradient(), cConcat.getGradient(), dimension, 1, units * variables)) ReturnFalse; if(cInpNorm.Activation() != None) if(!DeActivation(cInpNorm.getOutput(), cInpNorm.getPrevOutput(), cInpNorm.getPrevOutput(), cInpNorm.Activation())) ReturnFalse; Deactivation(cCorrelation) //--- if(!MatMulGrad(cWarpNorm.getOutput(), cWarpNorm.getGradient(), cRefFrameT.getOutput(), cRefFrameT.getGradient(), cCorrelation.getOutput(), units, dimension, 1, variables, true)) ReturnFalse; Deactivation(cWarpNorm) Deactivation(cRefFrameT) //--- if(!cWarp.CalcHiddenGradients(cWarpNorm.AsObject())) ReturnFalse; //--- if(!cInpNorm.CalcHiddenGradients(cWarp.AsObject())) ReturnFalse; if(!SumAndNormilize(cInpNorm.getPrevOutput(), cInpNorm.getGradient(), cInpNorm.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cInpNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cInpNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cWarp.UpdateInputWeights(cInpNorm.AsObject())) ReturnFalse; if(!cWarpNorm.UpdateInputWeights(cWarp.AsObject())) ReturnFalse; if(!CNeuronTMAMFE::updateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronTMAMFE::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronBiDirectCorrelation* Source = source; if(!cInpNorm.WeightsUpdate(Source.cInpNorm.AsObject(), tau)) ReturnFalse; if(!cWarp.WeightsUpdate(Source.cWarp.AsObject(), tau)) ReturnFalse; if(!cWarpNorm.WeightsUpdate(Source.cWarpNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::Save(const int file_handle) { if(!CNeuronTMAMFE::Save(file_handle)) ReturnFalse; //--- if(!cInpNorm.Save(file_handle)) ReturnFalse; if(!cRefFrameT.Save(file_handle)) ReturnFalse; if(!cWarp.Save(file_handle)) ReturnFalse; if(!cWarpNorm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBiDirectCorrelation::Load(const int file_handle) { if(!CNeuronTMAMFE::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cInpNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cRefFrameT.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWarp.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWarpNorm.AsObject())) ReturnFalse; //--- uint dimension = cRefFrameT.GetWindow(); uint variables = cRefFrameT.GetCount(); uint units = cCorrelation.Neurons() / variables; //--- uint index = 1; if(!cRefFrame.Init(0, index, OpenCL, variables * dimension, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); index += 4; if(!cCorrelation.Init(0, index, OpenCL, units * variables, optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); index++; if(!cConcat.Init(0, index, OpenCL, (dimension + 1)*units * variables, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBiDirectCorrelation::SetOpenCL(COpenCLMy *obj) { CNeuronTMAMFE::SetOpenCL(obj); //--- cInpNorm.SetOpenCL(OpenCL); cRefFrameT.SetOpenCL(OpenCL); cWarp.SetOpenCL(OpenCL); cWarpNorm.SetOpenCL(OpenCL); cRefFrame.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSATMA : public CNeuronBaseOCL { protected: CLayer cWarp; CNeuronBaseOCL cRefFrame; CNeuronRelativeCrossAttention cCrossAttention; CNeuronConvOCL cSpatialAttention; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSATMA(void) {}; ~CNeuronSATMA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint variables, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSATMA; } //--- virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint variables, uint window_key, uint heads, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units * variables, optimization_type, batch)) ReturnFalse; activation = None; //--- cWarp.Clear(); cWarp.SetOpenCL(OpenCL); CNeuronMultiWindowsConvOCL* mwconv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronBaseOCL* neuron = NULL; uint index = 0; { uint windows[]; if(ArrayResize(windows, units) < (int)units) ReturnFalse; ArrayFill(windows, 0, units, dimension); mwconv = new CNeuronMultiWindowsConvOCL(); if(!mwconv || !mwconv.Init(0, index, OpenCL, windows, dimension, variables, 1, optimization, iBatch) || !cWarp.Add(mwconv)) { DeleteObj(mwconv) ReturnFalse; } mwconv.SetActivationFunction(None); } index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, mwconv.Neurons(), iBatch, optimization) || !cWarp.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; if(!cRefFrame.Init(0, index, OpenCL, dimension * variables, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); index++; if(!cCrossAttention.Init(0, index, OpenCL, dimension, window_key, units * variables, heads, dimension, units * variables, optimization, iBatch)) ReturnFalse; index++; if(!cSpatialAttention.Init(0, index, OpenCL, dimension, dimension, dimension, units, variables, optimization, iBatch)) ReturnFalse; cSpatialAttention.SetActivationFunction(SIGMOID); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cWarp.Total(); i++) { curr = cWarp[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!cSpatialAttention.FeedForward(NeuronOCL)) ReturnFalse; //--- uint dimension = cSpatialAttention.GetWindow(); uint units = cSpatialAttention.GetUnits(); uint variables = cSpatialAttention.GetVariables(); if(!DeConcat(cRefFrame.getOutput(), prev.getPrevOutput(), NeuronOCL.getOutput(), dimension, dimension * (units - 1), variables)) ReturnFalse; if(!SumVecMatrix(cRefFrame.getOutput(), prev.getOutput(), prev.getOutput(), dimension, variables, 0, 0, 0, 1)) ReturnFalse; if(!cCrossAttention.FeedForward(NeuronOCL, prev.getOutput())) ReturnFalse; if(!ElementMult(cSpatialAttention.getOutput(), cCrossAttention.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!ElementMultGrad(cSpatialAttention.getOutput(), cSpatialAttention.getGradient(), cCrossAttention.getOutput(), cCrossAttention.getGradient(), Gradient, cSpatialAttention.Activation(), cCrossAttention.Activation())) ReturnFalse; //--- CNeuronBaseOCL* next = cWarp[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !NeuronOCL.CalcHiddenGradients(cCrossAttention.AsObject(), next.getOutput(), next.getGradient(), (ENUM_ACTIVATION)next.Activation())) ReturnFalse; //--- for(int i = cWarp.Total() - 2; i >= 0; i--) { curr = cWarp[i]; if(!curr || !curr.FeedForward(next)) ReturnFalse; next = curr; } //--- uint dimension = cSpatialAttention.GetWindow(); CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(cSpatialAttention.getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(next) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSpatialAttention.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cWarp.Total(); i++) { curr = cWarp[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!cSpatialAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cCrossAttention.UpdateInputWeights(NeuronOCL, prev.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSATMA* Source = source; if(!cWarp.WeightsUpdate(Source.cWarp.AsObject(), tau)) ReturnFalse; if(!cSpatialAttention.WeightsUpdate(Source.cSpatialAttention.AsObject(), tau)) ReturnFalse; if(!cCrossAttention.WeightsUpdate(Source.cCrossAttention.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cWarp.Save(file_handle)) ReturnFalse; if(!cSpatialAttention.Save(file_handle)) ReturnFalse; if(!cCrossAttention.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSATMA::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cWarp.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSpatialAttention.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossAttention.AsObject())) ReturnFalse; //--- uint dimension = cSpatialAttention.GetWindow(); uint variables = cSpatialAttention.GetVariables(); uint index = cWarp.Total(); if(!cRefFrame.Init(0, index, OpenCL, dimension * variables, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSATMA::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cWarp.SetOpenCL(OpenCL); cSpatialAttention.SetOpenCL(OpenCL); cCrossAttention.SetOpenCL(OpenCL); cRefFrame.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBAT : public CNeuronSpikeConvBlock { protected: CLayer cPrepare; CLayer cEncoder; CLayer cContext; CNeuronBaseOCL cConcatenated; CNeuronSpikeConvGRU2D cGRU; CNeuronTransposeRCDOCL cTranspose; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronBAT(void) {}; ~CNeuronBAT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronBAT; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; virtual CBufferFloat *getWeights(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, dimension, dimension, dimension, stack_size * variables, 1, optimization_type, batch)) ReturnFalse; //--- int index = 0; CNeuronBatchNormOCL* norm = NULL; CNeuronCreateFlow* stack = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMultiWindowsConvWPadOCL* mwconv = NULL; CNeuronMSRes* resblock = NULL; CNeuronBiDirectCorrelation* correlation = NULL; CNeuronSATMA* satma = NULL; //--- cPrepare.Clear(); cPrepare.SetOpenCL(OpenCL); norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, dimension * variables, iBatch, optimization) || !cPrepare.Add(norm)) { DeleteObj(norm) ReturnFalse; } index++; stack = new CNeuronCreateFlow(); if(!stack || !stack.Init(0, index, OpenCL, stack_size, dimension, embed_size, variables, optimization, iBatch) || !cPrepare.Add(stack)) { DeleteObj(stack) ReturnFalse; } //--- cEncoder.Clear(); cContext.Clear(); cEncoder.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); //--- uint windows[] = {embed_size, 3 * embed_size, 5 * embed_size, 7 * embed_size}; mwconv = new CNeuronMultiWindowsConvWPadOCL(); index++; if(!mwconv || !mwconv.Init(0, index, OpenCL, windows, embed_size, embed_size, stack_size, variables, optimization, iBatch) || !cEncoder.Add(mwconv)) { DeleteObj(mwconv) ReturnFalse; } mwconv.SetActivationFunction(None); mwconv = new CNeuronMultiWindowsConvWPadOCL(); index++; if(!mwconv || !mwconv.Init(0, index, OpenCL, windows, embed_size, embed_size, stack_size, variables, optimization, iBatch) || !cContext.Add(mwconv)) { DeleteObj(mwconv) ReturnFalse; } mwconv.SetActivationFunction(None); uint mult = windows.Size(); while(mult > 1) { index++; resblock = new CNeuronMSRes(); if(!resblock || !resblock.Init(0, index, OpenCL, mult * stack_size, embed_size, variables, optimization, iBatch) || !cEncoder.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } index++; resblock = new CNeuronMSRes(); if(!resblock || !resblock.Init(0, index, OpenCL, mult * stack_size, embed_size, variables, optimization, iBatch) || !cContext.Add(resblock)) { DeleteObj(resblock) ReturnFalse; } mult = (mult + 1) / 2; index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 2 * embed_size, 2 * embed_size, embed_size, mult * stack_size, variables, optimization, iBatch) || !cEncoder.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 2 * embed_size, 2 * embed_size, embed_size, mult * stack_size, variables, optimization, iBatch) || !cContext.Add(conv)) { DeleteObj(conv) ReturnFalse; } } //--- index++; correlation = new CNeuronBiDirectCorrelation(); if(!correlation || !correlation.Init(0, index, OpenCL, embed_size, stack_size, variables, embed_size, optimization, iBatch) || !cEncoder.Add(correlation)) { DeleteObj(correlation) ReturnFalse; } index++; satma = new CNeuronSATMA(); if(!satma || !satma.Init(0, index, OpenCL, embed_size, stack_size, variables, (embed_size + 3) / 4, 4, optimization, iBatch) || !cEncoder.Add(satma)) { DeleteObj(satma) ReturnFalse; } index++; if(!cConcatenated.Init(0, index, OpenCL, 2 * embed_size * stack_size * variables, optimization, iBatch)) ReturnFalse; index++; if(!cGRU.Init(0, index, OpenCL, stack_size * variables, 2 * embed_size, dimension, optimization, iBatch)) ReturnFalse; index++; if(!cTranspose.Init(0, index, OpenCL, variables, stack_size, dimension, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; //--- for(int i = 0; i < cPrepare.Total(); i++) { current = cPrepare[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- prev = cPrepare[-1]; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.FeedForward(prev)) ReturnFalse; prev = current; } //--- uint units = cGRU.GetUnits(); if(!Concat(cEncoder[-1].getOutput(), cContext[-1].getOutput(), cConcatenated.getOutput(), cEncoder[-1].Neurons() / units, cContext[-1].Neurons() / units, units)) ReturnFalse; if(!cGRU.FeedForward(cConcatenated.AsObject())) ReturnFalse; //--- if(!cTranspose.FeedForward(cGRU.AsObject())) ReturnFalse; if(!SumVecMatrix(cPrepare[0].getOutput(), cTranspose.getOutput(), cTranspose.getOutput(), cPrepare[0].Neurons(), 1, 0, 0, 0, 1)) ReturnFalse; //--- return CNeuronSpikeConvBlock::feedForward(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cTranspose.AsObject())) ReturnFalse; //--- if(!cGRU.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; if(!cConcatenated.CalcHiddenGradients(cGRU.AsObject())) ReturnFalse; uint units = cGRU.GetUnits(); if(!cEncoder[-1] || !cContext[-1] || !DeConcat(cEncoder[-1].getGradient(), cContext[-1].getGradient(), cConcatenated.getGradient(), cEncoder[-1].Neurons() / units, cContext[-1].Neurons() / units, units)) ReturnFalse; Deactivation(cEncoder[-1]) Deactivation(cContext[-1]) //--- CNeuronBaseOCL* current = NULL; for(int i = cEncoder.Total() - 2; i >= 0; i--) { current = cEncoder[i]; if(!current || !current.CalcHiddenGradients(cEncoder[i + 1])) ReturnFalse; } //--- for(int i = cContext.Total() - 2; i >= 0; i--) { current = cContext[i]; if(!current || !current.CalcHiddenGradients(cContext[i + 1])) ReturnFalse; } //--- current = cPrepare[-1]; if(!current || !current.CalcHiddenGradients(cEncoder[0])) ReturnFalse; CBufferFloat* temp = current.getGradient(); if(!current.SetGradient(current.getPrevOutput(), false)) ReturnFalse; if(!current.CalcHiddenGradients(cContext[0]) || !SumAndNormilize(temp, current.getGradient(), temp, 1, false, 0, 0, 0, 1) || !current.SetGradient(temp, false)) ReturnFalse; //--- for(int i = cPrepare.Total() - 2; i >= 0; i--) { current = cPrepare[i]; if(!current || !current.CalcHiddenGradients(cPrepare[i + 1])) ReturnFalse; } //--- if(!SumVecMatrixGrad(current.getPrevOutput(), cTranspose.getPrevOutput(), cTranspose.getGradient(), current.Neurons(), 1, 0, 0, 0, 1)) ReturnFalse; if(current.Activation() != None) if(!DeActivation(current.getOutput(), current.getPrevOutput(), current.getPrevOutput(), current.Activation())) ReturnFalse; if(!SumAndNormilize(current.getPrevOutput(), current.getGradient(), current.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(current)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* current = NULL; CNeuronBaseOCL* prev = NeuronOCL; //--- for(int i = 0; i < cPrepare.Total(); i++) { current = cPrepare[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- for(int i = 0; i < cEncoder.Total(); i++) { current = cEncoder[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- prev = cPrepare[-1]; for(int i = 0; i < cContext.Total(); i++) { current = cContext[i]; if(!current || !current.UpdateInputWeights(prev)) ReturnFalse; prev = current; } //--- return CNeuronSpikeConvBlock::updateInputWeights(cTranspose.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronBAT* Source = source; if(!cPrepare.WeightsUpdate(Source.cPrepare.AsObject(), tau)) ReturnFalse; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; if(!cContext.WeightsUpdate(Source.cContext.AsObject(), tau)) ReturnFalse; if(!cGRU.WeightsUpdate(Source.cGRU.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cGRU.Save(file_handle)) ReturnFalse; if(!cTranspose.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(!cPrepare.Load(file_handle)) ReturnFalse; if(!cEncoder.Load(file_handle)) ReturnFalse; if(!cContext.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGRU.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; //--- if(!cConcatenated.Init(0, 0, OpenCL, cContext[-1].Neurons() + cEncoder[-1].Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBAT::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cEncoder.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cConcatenated.SetOpenCL(OpenCL); cGRU.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronBAT::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cEncoder.TrainMode(bTrain); cContext.TrainMode(bTrain); cGRU.TrainMode(bTrain); cTranspose.TrainMode(bTrain); cConcatenated.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBAT::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; if(!cEncoder.ClearStates()) ReturnFalse; if(!cContext.ClearStates()) ReturnFalse; if(!cGRU.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronBAT::getWeights(void) { CBufferFloat* result = NULL; for(int layerNum = 0; (layerNum < cPrepare.Total() && !result && !IsStopped()); layerNum++) { CNeuronBaseOCL *temp = cPrepare[layerNum]; CNeuronConvOCL *conv = NULL; CNeuronConcatenate *conc = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; switch(temp.Type()) { case defNeuronConvOCL: conv = temp; result = conv.GetWeightsConv(); break; case defNeuronConcatenate: conc = temp; result = conc.getConcatWeights(); break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; result = batch.getBatchOptions(); break; case defNeuronLSTMOCL: lstm = temp; result = lstm.getLSTMWeights(); break; default: result = temp.getWeights(); break; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSCM : public CNeuronSpikeConvBlock { protected: CLayer cScale; CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeSCM(void) {}; ~CNeuronSpikeSCM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSCM; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return (!cScale[0] ? 0 : ((CNeuronSpikeConvBlock*)cScale[0]).GetWindow()); } //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, window + window_out, window + window_out, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; //--- uint index = 0; uint windows[1] = {window * 3}; CNeuronSpikeConvBlock* conv = NULL; CNeuronMultiWindowsConvWPadOCL* pad = NULL; cScale.Clear(); cScale.SetOpenCL(OpenCL); //--- pad = new CNeuronMultiWindowsConvWPadOCL(); if(!pad || !pad.Init(0, index, OpenCL, windows, window, (window_out + 3) / 4, units_count, variables, optimization, iBatch) || !cScale.Add(pad)) { DeleteObj(pad) ReturnFalse; } pad.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, pad.GetFilters(), pad.GetFilters(), (window_out + 1) / 2, units_count, variables, optimization, iBatch) || !cScale.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; windows[0] = conv.GetFilters() * 3; pad = new CNeuronMultiWindowsConvWPadOCL(); if(!pad || !pad.Init(0, index, OpenCL, windows, conv.GetFilters(), conv.GetFilters(), units_count, variables, optimization, iBatch) || !cScale.Add(pad)) { DeleteObj(pad) ReturnFalse; } pad.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, pad.GetFilters(), pad.GetFilters(), window_out, units_count, variables, optimization, iBatch) || !cScale.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- if(!cConcat.Init(0, index, OpenCL, (window + window_out)*units_count * variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScale.Total(); i++) { curr = cScale[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint units = GetUnits() * GetVariables(); uint window_in = NeuronOCL.Neurons() / units; uint window_prev = prev.Neurons() / units; if(!Concat(NeuronOCL.getOutput(), prev.getOutput(), cConcat.getOutput(), window_in, window_prev, units)) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cConcat.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* next = cScale[-1]; CNeuronBaseOCL* curr = NULL; uint units = GetUnits() * GetVariables(); uint window_in = NeuronOCL.Neurons() / units; uint window_prev = cConcat.Neurons() / units - window_in; if(!next || !DeConcat(cConcat.getPrevOutput(), next.getGradient(), cConcat.getGradient(), window_in, window_prev, units)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cConcat.getPrevOutput(), cConcat.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; //--- for(int i = cScale.Total() - 2; i >= 0; i--) { curr = cScale[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next) || !SumAndNormilize(NeuronOCL.getGradient(), cConcat.getPrevOutput(), NeuronOCL.getGradient(), window_in, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScale.Total(); i++) { curr = cScale[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeConvBlock::updateInputWeights(cConcat.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSCM* Source = source; if(!cScale.WeightsUpdate(Source.cScale.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cScale.Load(file_handle)) ReturnFalse; //--- if(!cConcat.Init(0, cScale.Total(), OpenCL, cConv.GetWindow() * GetUnits() * GetVariables(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSCM::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cScale.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSCM::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cScale.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSCM::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cScale.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeDepthWiseConv : public CNeuronSpikeConvBlock { protected: CNeuronTransposeVRCOCL cTranspose[2]; CNeuronMultiWindowsConvWPadOCL cDepthConv; CNeuronBatchNormOCL cNorm; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeDepthWiseConv(void) {}; ~CNeuronSpikeDepthWiseConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint depth_window, uint depth_step, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeDepthWiseConv; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint depth_window, uint depth_step, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, chanels_in, chanels_in, chanels_out, units, variables, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cTranspose[0].Init(0, index, OpenCL, variables, units * depth_step, chanels_in, optimization, iBatch)) ReturnFalse; cTranspose[0].SetActivationFunction(None); index++; if(!cTranspose[1].Init(0, index, OpenCL, variables, chanels_in, units, optimization, iBatch)) ReturnFalse; cTranspose[1].SetActivationFunction(None); index++; //--- uint windows[] = { depth_window }; if(!cDepthConv.Init(0, index, OpenCL, windows, depth_step, 1, units, chanels_in * variables, optimization, iBatch)) ReturnFalse; cDepthConv.SetActivationFunction(None); index++; if(!cNorm.Init(0, index, OpenCL, cDepthConv.Neurons(), iBatch, optimization)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::feedForward(CNeuronBaseOCL *NeuronOCL) { int total_in = NeuronOCL.Neurons(); uint units = total_in / cTranspose[0].GetVariables(); units = (units + cTranspose[0].GetWindow() - 1) / cTranspose[0].GetWindow(); if(units != cTranspose[0].GetCount()) { if(!cTranspose[0].Init(0, 0, OpenCL, cTranspose[0].GetVariables(), units, cTranspose[0].GetWindow(), optimization, iBatch)) ReturnFalse; cTranspose[0].SetActivationFunction(None); } if(!cTranspose[0].FeedForward(NeuronOCL)) ReturnFalse; #ifdef _DEBUG if(!cTranspose[0].getOutput().BufferRead()) ReturnFalse; #endif if(!cDepthConv.FeedForward(cTranspose[0].AsObject())) ReturnFalse; #ifdef _DEBUG if(!cDepthConv.getOutput().BufferRead()) ReturnFalse; #endif if(!cTranspose[1].FeedForward(cDepthConv.AsObject())) ReturnFalse; if(!cNorm.FeedForward(cTranspose[1].AsObject())) ReturnFalse; #ifdef _DEBUG if(!cNorm.getOutput().BufferRead()) ReturnFalse; #endif if(!CNeuronSpikeConvBlock::feedForward(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeConvBlock::calcInputGradients(cNorm.AsObject())) ReturnFalse; if(!cTranspose[1].CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; if(!cDepthConv.CalcHiddenGradients(cTranspose[1].AsObject())) ReturnFalse; if(!cTranspose[0].CalcHiddenGradients(cDepthConv.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cTranspose[0].AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cDepthConv.UpdateInputWeights(cTranspose[0].AsObject())) ReturnFalse; if(!cNorm.UpdateInputWeights(cTranspose[1].AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeDepthWiseConv* Source = source; if(!cDepthConv.WeightsUpdate(Source.cDepthConv.AsObject(), tau)) ReturnFalse; if(!cNorm.WeightsUpdate(Source.cNorm.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- for(uint i = 0; i < cTranspose.Size(); i++) if(!cTranspose[i].Save(file_handle)) ReturnFalse; if(!cDepthConv.Save(file_handle)) ReturnFalse; if(!cNorm.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseConv::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- for(uint i = 0; i < cTranspose.Size(); i++) if(!LoadInsideLayer(file_handle, cTranspose[i].AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cDepthConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeDepthWiseConv::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); //--- for(uint i = 0; i < cTranspose.Size(); i++) cTranspose[i].SetOpenCL(OpenCL); cDepthConv.SetOpenCL(OpenCL); cNorm.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeDepthWiseConv::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- for(uint i = 0; i < cTranspose.Size(); i++) cTranspose[i].TrainMode(bTrain); cDepthConv.TrainMode(bTrain); cNorm.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeFAM : public CNeuronBatchNormOCL { protected: CNeuronBaseOCL cProduct; CNeuronSpikeActivation cActivation; CNeuronSpikeDepthWiseConv cConv; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronSpikeFAM(void) {}; ~CNeuronSpikeFAM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeFAM; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cConv.GetWindow(); } virtual uint GetVariables(void) const { return cConv.GetVariables(); } virtual uint GetUnits(void) const { return cConv.GetUnits(); } //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, batch, optimization_type)) ReturnFalse; activation = None; if(!cProduct.Init(0, 0, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cProduct.SetActivationFunction(None); if(!cActivation.Init(0, 1, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; if(!cConv.Init(0, 2, OpenCL, window, window, 3, 1, units_count, variables, optimization, iBatch)) ReturnFalse; cConv.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(!ElementMult(NeuronOCL.getOutput(), SecondInput, cProduct.getOutput())) ReturnFalse; if(!cActivation.FeedForward(cProduct.AsObject())) ReturnFalse; if(!cConv.FeedForward(cActivation.AsObject())) ReturnFalse; if(!CNeuronBatchNormOCL::feedForward(cConv.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), Output, Output, GetWindow(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; if(!CNeuronBatchNormOCL::calcInputGradients(cConv.AsObject())) ReturnFalse; if(!cActivation.CalcHiddenGradients(cConv.AsObject())) ReturnFalse; if(!cProduct.CalcHiddenGradients(cActivation.AsObject())) ReturnFalse; //--- if(!ElementMultGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), SecondInput, SecondGradient, cProduct.getGradient(), NeuronOCL.Activation(), SecondActivation)) ReturnFalse; if(!DeActivation(NeuronOCL.getOutput(), cProduct.getPrevOutput(), Gradient, NeuronOCL.Activation()) || !SumAndNormilize(NeuronOCL.getGradient(), cProduct.getPrevOutput(), NeuronOCL.getGradient(), GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cActivation.UpdateInputWeights(cProduct.AsObject())) ReturnFalse; if(!cConv.UpdateInputWeights(cActivation.AsObject())) ReturnFalse; if(!CNeuronBatchNormOCL::updateInputWeights(cConv.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::Save(const int file_handle) { if(!CNeuronBatchNormOCL::Save(file_handle)) ReturnFalse; if(!cActivation.Save(file_handle)) ReturnFalse; if(!cConv.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::Load(const int file_handle) { if(!CNeuronBatchNormOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cActivation.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConv.AsObject())) ReturnFalse; if(!cProduct.Init(0, 0, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; cProduct.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::Clear(void) { if(!CNeuronBatchNormOCL::Clear()) ReturnFalse; if(!cActivation.Clear()) ReturnFalse; if(!cConv.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeFAM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBatchNormOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeFAM* Source = source; if(!cActivation.WeightsUpdate(Source.cActivation.AsObject(), tau)) ReturnFalse; if(!cConv.WeightsUpdate(Source.cConv.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeFAM::SetOpenCL(COpenCLMy *obj) { CNeuronBatchNormOCL::SetOpenCL(obj); cProduct.SetOpenCL(OpenCL); cActivation.SetOpenCL(OpenCL); cConv.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeFAM::TrainMode(bool flag) { CNeuronBatchNormOCL::TrainMode(flag); cActivation.TrainMode(bTrain); cConv.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeMDC : public CNeuronBatchNormOCL { protected: CLayer cSCMs; CLayer cFAMs; CLayer cAFFs; CLayer cProjects; CLayer cProjectsOut; CLayer cFeatExtracts; CNeuronBaseOCL cConcatOut; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeMDC(void) {}; ~CNeuronSpikeMDC(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeMDC; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { // 3 * 32 * 6 * 1 if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, 3 * window_out * units_count * variables, batch, optimization_type)) ReturnFalse; //--- cSCMs.Clear(); cFAMs.Clear(); cAFFs.Clear(); cProjects.Clear(); cProjectsOut.Clear(); cFeatExtracts.Clear(); cSCMs.SetOpenCL(OpenCL); cFAMs.SetOpenCL(OpenCL); cAFFs.SetOpenCL(OpenCL); cProjects.SetOpenCL(OpenCL); cProjectsOut.SetOpenCL(OpenCL); cFeatExtracts.SetOpenCL(OpenCL); //--- CNeuronSpikeConvBlock* conv = NULL; CNeuronSpikeDepthWiseConv* pad_conv = NULL; CNeuronSpikeSCM* scm = NULL; CNeuronSpikeFAM* fam = NULL; CNeuronBaseOCL* neuron = NULL; uint index = 0; //--- // 576 if(!cConcatOut.Init(0, index, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 32 * 6 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, window, window_out, 3, 1, units_count, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); uint u = (units_count + 1) / 2; // 64 * 3 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, window_out, 2 * window_out, 3, 2, u, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); u = (u + 1) / 2; // 128 * 2 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 2 * window_out, 4 * window_out, 3, 2, u, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 256 * 2 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 4 * window_out, 8 * window_out, 3, 1, u, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } u *= 2; index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 128 * 4 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 4 * window_out, 4 * window_out, 3, 1, u, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } u *= 2; index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 32 * 8 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, window_out, window_out, 5, 1, units_count, variables, optimization, iBatch) || !cFeatExtracts.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } //--- index++; neuron = new CNeuronBaseOCL(); // 5 * 32 * 6 * 1 if(!neuron || !neuron.Init(0, index, OpenCL, 5 * window_out * units_count * variables, optimization, iBatch) || !cProjects.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; // 64 * 6 * 1 conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 5 * window_out, 5 * window_out, 2 * window_out, units_count, variables, optimization, iBatch) || !cProjects.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; neuron = new CNeuronBaseOCL(); // 5 * 32 * 6 * 1 if(!neuron || !neuron.Init(0, index, OpenCL, 5 * window_out * units_count * variables, optimization, iBatch) || !cProjects.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); // 32 * 6 * 1 if(!conv || !conv.Init(0, index, OpenCL, 5 * window_out, 5 * window_out, window_out, units_count, variables, optimization, iBatch) || !cProjects.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 32 * 6 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 4 * window_out, window_out, 3, 2, units_count, variables, optimization, iBatch) || !cProjectsOut.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 32 * 6 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 2 * window_out, window_out, 3, 2, units_count, variables, optimization, iBatch) || !cProjectsOut.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } //--- index++; neuron = new CNeuronBaseOCL(); // 7 * 32 * 6 * 1 if(!neuron || !neuron.Init(0, index, OpenCL, 7 * window_out * units_count * variables, optimization, iBatch) || !cAFFs.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); // 32 * 6 * 1 if(!conv || !conv.Init(0, index, OpenCL, 7 * window_out, 7 * window_out, window_out, units_count, variables, optimization, iBatch) || !cAFFs.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 32 * 6 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, window_out, window_out, 3, 1, units_count, variables, optimization, iBatch) || !cAFFs.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); // 64 * 3 * 1 if(!conv || !conv.Init(0, index, OpenCL, 14 * window_out, 14 * window_out, 2 * window_out, (units_count + 1) / 2, variables, optimization, iBatch) || !cAFFs.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; pad_conv = new CNeuronSpikeDepthWiseConv(); // 64 * 3 * 1 if(!pad_conv || !pad_conv.Init(0, index, OpenCL, 2 * window_out, 2 * window_out, 3, 1, (units_count + 1) / 2, variables, optimization, iBatch) || !cAFFs.Add(pad_conv)) { DeleteObj(pad_conv) ReturnFalse; } pad_conv.SetActivationFunction(None); //--- index++; fam = new CNeuronSpikeFAM(); pad_conv = cFeatExtracts[2]; // 128 * 6 * 1 if(!fam || !fam.Init(0, index, OpenCL, pad_conv.GetFilters(), pad_conv.GetUnits(), variables, optimization, iBatch) || !cFAMs.Add(fam)) { DeleteObj(fam) ReturnFalse; } index++; fam = new CNeuronSpikeFAM(); pad_conv = cFeatExtracts[1]; // 64 * 6 * 1 if(!fam || !fam.Init(0, index, OpenCL, pad_conv.GetFilters(), pad_conv.GetUnits(), variables, optimization, iBatch) || !cFAMs.Add(fam)) { DeleteObj(fam) ReturnFalse; } //--- index++; scm = new CNeuronSpikeSCM(); // 64 * 3 * 1 if(!scm || !scm.Init(0, index, OpenCL, 2 * window, 2 * window_out, (units_count + 1) / 2, variables, optimization, iBatch) || !cSCMs.Add(scm)) { DeleteObj(scm) ReturnFalse; } u = scm.GetUnits(); index++; scm = new CNeuronSpikeSCM(); // 128 * 2 * 1 if(!scm || !scm.Init(0, index, OpenCL, 4 * window, 4 * window_out, (u + 1) / 2, variables, optimization, iBatch) || !cSCMs.Add(scm)) { DeleteObj(scm) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSCMs.Total(); i++) { curr = cSCMs[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!cFeatExtracts[0] || !cFeatExtracts[0].FeedForward(NeuronOCL)) ReturnFalse; if(!cFeatExtracts[1] || !cFeatExtracts[1].FeedForward(cFeatExtracts[0])) ReturnFalse; if(!cFAMs[1] || !cFAMs[1].FeedForward(cFeatExtracts[1], cSCMs[0].getOutput())) ReturnFalse; if(!cFeatExtracts[2] || !cFeatExtracts[2].FeedForward(cFAMs[1])) ReturnFalse; if(!cFAMs[0] || !cFAMs[0].FeedForward(cFeatExtracts[2], cSCMs[1].getOutput())) ReturnFalse; //--- AAFF CNeuronSpikeConvBlock* conv = cAFFs[1]; if(!conv) ReturnFalse; uint units = conv.GetUnits() * conv.GetVariables(); uint window1 = cFeatExtracts[0].Neurons() / units; uint window2 = cFAMs[1].Neurons() / units; uint window3 = cFAMs[0].Neurons() / units; prev = cAFFs[0]; if(!prev || !Concat(cFeatExtracts[0].getOutput(), cFAMs[1].getOutput(), cFAMs[0].getOutput(), prev.getOutput(), window1, window2, window3, units)) ReturnFalse; for(int i = 1; i < cAFFs.Total(); i += 2) { conv = cAFFs[i]; curr = cAFFs[i + 1]; if(!conv || !conv.FeedForward(prev)) ReturnFalse; if(!curr || !curr.FeedForward(conv)) ReturnFalse; } //--- prev = cProjectsOut[0]; curr = cFeatExtracts[3]; if(!prev || !prev.FeedForward(cFAMs[0])) ReturnFalse; if(!curr || !curr.FeedForward(cFAMs[0])) ReturnFalse; //--- window1 = curr.Neurons() / units; window2 = cAFFs[4].Neurons() / units; prev = cProjects[0]; if(!prev || !Concat(curr.getOutput(), cAFFs[4].getOutput(), prev.getOutput(), window1, window2, units)) ReturnFalse; if(!cProjects[1] || !cProjects[1].FeedForward(prev)) ReturnFalse; if(!cProjectsOut[1] || !cProjectsOut[1].FeedForward(cProjects[1])) ReturnFalse; if(!cFeatExtracts[4] || !cFeatExtracts[4].FeedForward(cProjects[1])) ReturnFalse; //--- window1 = cFeatExtracts[4].Neurons() / units; window2 = cAFFs[2].Neurons() / units; prev = cProjects[2]; if(!prev || !Concat(cFeatExtracts[4].getOutput(), cAFFs[2].getOutput(), prev.getOutput(), window1, window2, units)) ReturnFalse; if(!cProjects[3] || !cProjects[3].FeedForward(prev)) ReturnFalse; if(!cFeatExtracts[5] || !cFeatExtracts[5].FeedForward(cProjects[3])) ReturnFalse; window1 = cProjectsOut[0].Neurons() / units; window2 = cProjectsOut[1].Neurons() / units; window3 = cFeatExtracts[5].Neurons() / units; //--- if(!Concat(cProjectsOut[0].getOutput(), cProjectsOut[1].getOutput(), cFeatExtracts[5].getOutput(), cConcatOut.getOutput(), window1, window2, window3, units)) ReturnFalse; window1 = NeuronOCL.Neurons() / units; if(!SumVecMatrix(NeuronOCL.getOutput(), cConcatOut.getOutput(), cConcatOut.getOutput(), window1, units, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronBatchNormOCL::feedForward(cConcatOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronBatchNormOCL::calcInputGradients(cConcatOut.AsObject())) ReturnFalse; CNeuronSpikeConvBlock* conv = cAFFs[1]; if(!conv) ReturnFalse; uint units = conv.GetUnits() * conv.GetVariables(); uint window1 = NeuronOCL.Neurons() / units; if(!SumVecMatrixGrad(cConcatOut.getPrevOutput(), cConcatOut.getGradient(), cConcatOut.getGradient(), window1, units, 0, 0, 0, 1)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cConcatOut.getPrevOutput(), cConcatOut.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; //--- if(!cProjectsOut[0] || !cProjectsOut[1] || !cFeatExtracts[5]) ReturnFalse; window1 = cProjectsOut[0].Neurons() / units; uint window2 = cProjectsOut[1].Neurons() / units; uint window3 = cFeatExtracts[5].Neurons() / units; if(!DeConcat(cProjectsOut[0].getGradient(), cProjectsOut[1].getGradient(), cFeatExtracts[5].getGradient(), cConcatOut.getGradient(), window1, window2, window3, units)) ReturnFalse; Deactivation(cFeatExtracts[5]) for(int i = 0; i < 2; i++) { Deactivation(cProjectsOut[i]) } //--- if(!cProjects[3] || !cProjects[3].CalcHiddenGradients(cFeatExtracts[5])) ReturnFalse; if(!cProjects[2] || !cProjects[2].CalcHiddenGradients(cProjects[3])) ReturnFalse; if(!cFeatExtracts[4] || !cAFFs[2]) ReturnFalse; window1 = cFeatExtracts[4].Neurons() / units; window2 = cAFFs[2].Neurons() / units; if(!DeConcat(cFeatExtracts[4].getGradient(), cAFFs[2].getGradient(), cProjects[2].getGradient(), window1, window2, units)) ReturnFalse; Deactivation(cFeatExtracts[4]) Deactivation(cAFFs[2]) CNeuronBaseOCL* curr = cProjects[1]; if(!curr || !curr.CalcHiddenGradients(cFeatExtracts[4])) ReturnFalse; CBufferFloat* temp = curr.getGradient(); if(!curr.SetGradient(curr.getPrevOutput(), false) || !curr.CalcHiddenGradients(cProjectsOut[1]) || !SumAndNormilize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; if(!cProjects[0] || !cProjects[0].CalcHiddenGradients(curr)) ReturnFalse; //--- if(!cFeatExtracts[3] || !cAFFs[4]) ReturnFalse; window1 = cFeatExtracts[3].Neurons() / units; window2 = cAFFs[4].Neurons() / units; if(!DeConcat(cFeatExtracts[3].getGradient(), cAFFs[4].getGradient(), cProjects[0].getGradient(), window1, window2, units)) ReturnFalse; Deactivation(cFeatExtracts[3]) Deactivation(cAFFs[4]) //--- curr = cFAMs[0]; if(!curr || !curr.CalcHiddenGradients(cFeatExtracts[3])) ReturnFalse; temp = curr.getGradient(); if(!curr.SetGradient(curr.getPrevOutput(), false) || !curr.CalcHiddenGradients(cProjectsOut[0]) || !SumAndNormilize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; //--- curr = cAFFs[0]; if(!cAFFs[3] || !curr || !cAFFs[3].CalcHiddenGradients(cAFFs[4]) || !curr.CalcHiddenGradients(cAFFs[3])) ReturnFalse; temp = curr.getGradient(); if(!curr.SetGradient(curr.getPrevOutput(), false) || !cAFFs[1] || !cAFFs[1].CalcHiddenGradients(cAFFs[2]) || !curr.CalcHiddenGradients(cAFFs[1]) || !SumAndNormilize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; //--- window1 = cFeatExtracts[0].Neurons() / units; window2 = cFAMs[1].Neurons() / units; window3 = cFAMs[0].Neurons() / units; if(!DeConcat(cFeatExtracts[0].getPrevOutput(), cFAMs[1].getPrevOutput(), cFAMs[0].getPrevOutput(), curr.getGradient(), window1, window2, window3, units)) ReturnFalse; for(int i = 0; i < 2; i++) { curr = cFAMs[1]; if(curr.Activation() != None) if(!DeActivation(curr.getOutput(), curr.getPrevOutput(), curr.getPrevOutput(), curr.Activation())) ReturnFalse; } curr = cFeatExtracts[0]; if(curr.Activation() != None) if(!DeActivation(curr.getOutput(), curr.getPrevOutput(), curr.getPrevOutput(), curr.Activation())) ReturnFalse; if(!SumAndNormilize(cFAMs[0].getPrevOutput(), cFAMs[0].getGradient(), cFAMs[0].getGradient(), window3, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cFeatExtracts[2] || !cFeatExtracts[2].CalcHiddenGradients(cFAMs[0], cSCMs[1].getOutput(), cSCMs[1].getGradient(), (ENUM_ACTIVATION)cSCMs[1].Activation())) ReturnFalse; if(!cFAMs[1].CalcHiddenGradients(cFeatExtracts[2]) || !SumAndNormilize(cFAMs[1].getPrevOutput(), cFAMs[1].getGradient(), cFAMs[1].getGradient(), window2, false, 0, 0, 0, 1)) ReturnFalse; if(!cFeatExtracts[1] || !cFeatExtracts[1].CalcHiddenGradients(cFAMs[1], cSCMs[0].getOutput(), cSCMs[0].getPrevOutput(), (ENUM_ACTIVATION)cSCMs[0].Activation())) ReturnFalse; if(!cFeatExtracts[0] || !cFeatExtracts[0].CalcHiddenGradients(cFeatExtracts[1])) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFeatExtracts[0]) || !SumAndNormilize(NeuronOCL.getGradient(), cConcatOut.getPrevOutput(), cConcatOut.getPrevOutput(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cSCMs[0].CalcHiddenGradients(cSCMs[1]) || !SumAndNormilize(cSCMs[0].getPrevOutput(), cSCMs[0].getGradient(), cSCMs[0].getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSCMs[0]) || !SumAndNormilize(NeuronOCL.getGradient(), cConcatOut.getPrevOutput(), NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSCMs.Total(); i++) { curr = cSCMs[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cFeatExtracts[0] || !cFeatExtracts[0].UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFeatExtracts[1] || !cFeatExtracts[1].UpdateInputWeights(cFeatExtracts[0])) ReturnFalse; if(!cFAMs[1] || !cFAMs[1].UpdateInputWeights(cFeatExtracts[1], cSCMs[0].getOutput())) ReturnFalse; if(!cFeatExtracts[2] || !cFeatExtracts[2].UpdateInputWeights(cFAMs[1])) ReturnFalse; if(!cFAMs[0] || !cFAMs[0].UpdateInputWeights(cFeatExtracts[2], cSCMs[1].getOutput())) ReturnFalse; //--- AAFF prev = cAFFs[0]; CNeuronSpikeConvBlock* conv = NULL; for(int i = 1; i < cAFFs.Total(); i += 2) { conv = cAFFs[i]; curr = cAFFs[i + 1]; if(!conv || !conv.UpdateInputWeights(prev)) ReturnFalse; if(!curr || !curr.UpdateInputWeights(conv)) ReturnFalse; } //--- prev = cProjectsOut[0]; curr = cFeatExtracts[3]; if(!prev || !prev.UpdateInputWeights(cFAMs[0])) ReturnFalse; if(!curr || !curr.UpdateInputWeights(cFAMs[0])) ReturnFalse; //--- prev = cProjects[0]; if(!cProjects[1] || !cProjects[1].UpdateInputWeights(prev)) ReturnFalse; if(!cProjectsOut[1] || !cProjectsOut[1].UpdateInputWeights(cProjects[1])) ReturnFalse; if(!cFeatExtracts[4] || !cFeatExtracts[4].UpdateInputWeights(cProjects[1])) ReturnFalse; //--- prev = cProjects[2]; if(!cProjects[3] || !cProjects[3].UpdateInputWeights(prev)) ReturnFalse; if(!cFeatExtracts[5] || !cFeatExtracts[5].UpdateInputWeights(cProjects[3])) ReturnFalse; if(!CNeuronBatchNormOCL::updateInputWeights(cConcatOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::Save(const int file_handle) { if(!CNeuronBatchNormOCL::Save(file_handle)) ReturnFalse; if(!cSCMs.Save(file_handle)) ReturnFalse; if(!cFAMs.Save(file_handle)) ReturnFalse; if(!cAFFs.Save(file_handle)) ReturnFalse; if(!cProjects.Save(file_handle)) ReturnFalse; if(!cProjectsOut.Save(file_handle)) ReturnFalse; if(!cFeatExtracts.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::Load(const int file_handle) { if(!CNeuronBatchNormOCL::Load(file_handle)) ReturnFalse; if(!cSCMs.Load(file_handle)) ReturnFalse; if(!cFAMs.Load(file_handle)) ReturnFalse; if(!cAFFs.Load(file_handle)) ReturnFalse; if(!cProjects.Load(file_handle)) ReturnFalse; if(!cProjectsOut.Load(file_handle)) ReturnFalse; if(!cFeatExtracts.Load(file_handle)) ReturnFalse; //--- if(!cConcatOut.Init(0, 0, OpenCL, Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBatchNormOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeMDC* Source = source; dWeightsUpdate(cSCMs, Source, tau) dWeightsUpdate(cFAMs, Source, tau) dWeightsUpdate(cAFFs, Source, tau) dWeightsUpdate(cProjects, Source, tau) dWeightsUpdate(cProjectsOut, Source, tau) dWeightsUpdate(cFeatExtracts, Source, tau) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMDC::Clear(void) { if(!CNeuronBatchNormOCL::Clear()) ReturnFalse; if(!cSCMs.ClearStates()) ReturnFalse; if(!cFAMs.ClearStates()) ReturnFalse; if(!cAFFs.ClearStates()) ReturnFalse; if(!cProjects.ClearStates()) ReturnFalse; if(!cProjectsOut.ClearStates()) ReturnFalse; if(!cFeatExtracts.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMDC::SetOpenCL(COpenCLMy *obj) { CNeuronBatchNormOCL::SetOpenCL(obj); cSCMs.SetOpenCL(OpenCL); cFAMs.SetOpenCL(OpenCL); cAFFs.SetOpenCL(OpenCL); cProjects.SetOpenCL(OpenCL); cProjectsOut.SetOpenCL(OpenCL); cFeatExtracts.SetOpenCL(OpenCL); cConcatOut.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMDC::TrainMode(bool flag) { CNeuronBatchNormOCL::TrainMode(flag); cSCMs.TrainMode(bTrain); cFAMs.TrainMode(bTrain); cAFFs.TrainMode(bTrain); cProjects.TrainMode(bTrain); cProjectsOut.TrainMode(bTrain); cFeatExtracts.TrainMode(bTrain); cConcatOut.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMDS : public CNeuronBaseOCL { protected: uint iDimension; uint iUnits; uint iVariables; CNeuronBaseOCL cConcat; CLayer cScores; CNeuronTransposeVRCOCL cTranspose; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronMDS(void) {}; ~CNeuronMDS(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMDS; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window * units_count * variables, optimization_type, batch)) ReturnFalse; activation = None; //--- iDimension = window; iUnits = units_count; iVariables = variables; //--- uint index = 0; if(!cConcat.Init(0, index, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- CNeuronConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSoftMaxOCL* softmax = NULL; cScores.Clear(); cScores.SetOpenCL(OpenCL); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 4 * iDimension, 4 * iDimension, iDimension, iUnits, iVariables, optimization, iBatch) || !cScores.Add(conv)) ReturnFalse; conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cScores.Add(norm)) ReturnFalse; norm.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, iDimension, iDimension, 4, iUnits, iVariables, optimization, iBatch) || !cScores.Add(conv)) ReturnFalse; conv.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cScores.Add(norm)) ReturnFalse; norm.SetActivationFunction(SoftPlus); index++; softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, norm.Neurons(), optimization, iBatch) || !cScores.Add(softmax)) ReturnFalse; softmax.SetHeads(iUnits * iVariables); index++; if(!cTranspose.Init(0, index, OpenCL, iUnits * iVariables, 4, iDimension, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(!Concat(NeuronOCL.getOutput(), SecondInput, cConcat.getOutput(), iDimension, 3 * iDimension, iUnits * iVariables)) ReturnFalse; CNeuronBaseOCL* prev = cConcat.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScores.Total(); i++) { curr = cScores[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!cTranspose.FeedForward(cConcat.AsObject())) ReturnFalse; if(!MatMul(cTranspose.getOutput(), curr.getOutput(), Output, iDimension, 4, 1, iUnits * iVariables, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondInput || !SecondGradient) ReturnFalse; //--- CNeuronBaseOCL* curr = cScores[-1]; if(!curr || !MatMulGrad(cTranspose.getOutput(), cTranspose.getGradient(), curr.getOutput(), curr.getGradient(), Gradient, iDimension, 4, 1, iUnits * iVariables, true)) ReturnFalse; Deactivation(curr) Deactivation(cTranspose) if(!cConcat.CalcHiddenGradients(cTranspose.AsObject())) ReturnFalse; CBufferFloat* temp = cConcat.getGradient(); if(!cConcat.SetGradient(cTranspose.getPrevOutput(), false)) ReturnFalse; //--- for(int i = cScores.Total() - 2; i >= 0; i--) { curr = cScores[i]; if(!curr || !curr.CalcHiddenGradients(cScores[i + 1])) ReturnFalse; } if(!cConcat.CalcHiddenGradients(curr) || !SumAndNormilize(cConcat.getGradient(), temp, temp, iDimension, false, 0, 0, 0, 1) || !cConcat.SetGradient(temp, false)) ReturnFalse; //--- if(!DeConcat(NeuronOCL.getGradient(), SecondGradient, cConcat.getGradient(), iDimension, 3 * iDimension, iUnits * iVariables)) ReturnFalse; Deactivation(NeuronOCL) if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { CNeuronBaseOCL* prev = cConcat.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScores.Total(); i++) { curr = cScores[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMDS* Source = source; dWeightsUpdate(cScores, Source, tau) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMDS::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cScores.Load(file_handle)) ReturnFalse; //--- if(!cScores[0] || cScores[0].Type() != defNeuronConvOCL) ReturnFalse; CNeuronConvOCL* conv = cScores[0]; iDimension = conv.GetFilters(); iUnits = conv.GetUnits(); iVariables = conv.GetVariables(); //--- if(!cConcat.Init(0, 0, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); if(!cTranspose.Init(0, cScores.Total() + 1, OpenCL, iUnits * iVariables, 4, iDimension, optimization, iBatch)) ReturnFalse; cTranspose.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMDS::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cScores.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); cTranspose.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeADM : public CNeuronMDS { protected: CNeuronSpikeConvBlock cConv; CNeuronSpikeMDC cMDC; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override { return updateInputWeights(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } public: CNeuronSpikeADM(void) {}; ~CNeuronSpikeADM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeADM; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMDS::Init(numOutputs, myIndex, open_cl, window, units_count, variables, optimization_type, batch)) ReturnFalse; if(!cConv.Init(0, 0, OpenCL, window, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; if(!cMDC.Init(0, 1, OpenCL, window, window, units_count, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cConv.FeedForward(NeuronOCL)) ReturnFalse; if(!cMDC.FeedForward(cConv.AsObject())) ReturnFalse; if(!CNeuronMDS::feedForward(cConv.AsObject(), cMDC.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMDS::calcInputGradients(cConv.AsObject(), cMDC.getOutput(), cMDC.getGradient(), (ENUM_ACTIVATION)cMDC.Activation())) ReturnFalse; CBufferFloat* temp = cConv.getGradient(); if(!cConv.SetGradient(cConv.getPrevOutput(), false) || !cConv.CalcHiddenGradients(cMDC.AsObject()) || !SumAndNormilize(cConv.getGradient(), temp, temp, cConv.GetFilters(), false, 0, 0, 0, 1) || !cConv.SetGradient(temp, false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cConv.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cConv.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cMDC.UpdateInputWeights(cConv.AsObject())) ReturnFalse; if(!CNeuronMDS::updateInputWeights(cConv.AsObject(), cMDC.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMDS::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikeADM* Source = source; dWeightsUpdate(cConv, Source, tau) dWeightsUpdate(cMDC, Source, tau) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::Save(const int file_handle) { if(!CNeuronMDS::Save(file_handle)) ReturnFalse; if(!cConv.Save(file_handle)) ReturnFalse; if(!cMDC.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::Load(const int file_handle) { if(!CNeuronMDS::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cConv.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMDC.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeADM::Clear(void) { if(!CNeuronMDS::Clear()) ReturnFalse; if(!cConv.Clear()) ReturnFalse; if(!cMDC.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeADM::SetOpenCL(COpenCLMy *obj) { CNeuronMDS::SetOpenCL(obj); cConv.SetOpenCL(OpenCL); cMDC.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeADM::TrainMode(bool flag) { CNeuronMDS::TrainMode(flag); cConv.TrainMode(bTrain); cMDC.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronBATvADM : public CNeuronBAT { public: CNeuronBATvADM(void) {}; ~CNeuronBATvADM(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronBATvADM; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBATvADM::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBAT::Init(numOutputs, myIndex, open_cl, dimension, embed_size, stack_size, variables, optimization_type, batch)) ReturnFalse; //--- int index = cPrepare.Total(); CNeuronSpikeADM* amd = new CNeuronSpikeADM(); if(!amd || !amd.Init(0, index, OpenCL, embed_size, stack_size, variables, optimization, iBatch) || !cPrepare.Add(amd)) { DeleteObj(amd) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCDCSelfCorrector : public CNeuronBatchNormOCL { protected: uint iUnits; uint iVariables; CLayer cEstimator; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCDCSelfCorrector(void) {}; ~CNeuronCDCSelfCorrector(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint &inside_dimensions[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCDCSelfCorrector; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint &inside_dimensions[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { uint layers = inside_dimensions.Size(); if(layers <= 0) ReturnFalse; if(!CNeuronBatchNormOCL::Init(numOutputs, myIndex, open_cl, window_out * units_count * variables, batch, optimization_type)) ReturnFalse; activation = None; //--- iUnits = units_count; iVariables = variables; //--- cEstimator.Clear(); cEstimator.SetOpenCL(OpenCL); CNeuronMultiWindowsConvWPadOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronBaseOCL* concat = NULL; uint index = 0; uint inp_window = window; uint windows[1] = { 3 * inp_window }; for(uint i = 0; i < layers; i++) { conv = new CNeuronMultiWindowsConvWPadOCL(); if(!conv || !conv.Init(0, index, OpenCL, windows, inp_window, inside_dimensions[i], iUnits, iVariables, optimization, iBatch) || !cEstimator.Add(conv)) { DeleteObj(conv) ReturnFalse; } conv.SetActivationFunction(SoftPlus); index++; inp_window += conv.GetFilters(); windows[0] = 3 * inp_window; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cEstimator.Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; concat = new CNeuronBaseOCL(); if(!concat || !concat.Init(0, index, OpenCL, inp_window * iUnits * iVariables, optimization, iBatch) || !cEstimator.Add(concat)) { DeleteObj(concat) ReturnFalse; } index++; concat.SetActivationFunction(None); } //--- conv = new CNeuronMultiWindowsConvWPadOCL(); if(!conv || !conv.Init(0, index, OpenCL, windows, inp_window, window_out, iUnits, iVariables, optimization, iBatch) || !cEstimator.Add(conv)) { DeleteObj(conv) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* residual = NeuronOCL; CNeuronBaseOCL* prev = residual; CNeuronBaseOCL* curr = NULL; if(!residual) ReturnFalse; //--- uint count = iUnits * iVariables; uint res_window = residual.Neurons() / count; int total = cEstimator.Total(); //--- for(int i = 0; i < total; i++) { curr = cEstimator[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!Concat(residual.getOutput(), prev.getOutput(), curr.getOutput(), res_window, prev.Neurons() / count, count)) ReturnFalse; residual = curr; res_window = curr.Neurons() / count; } else if(!curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronBatchNormOCL::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronBatchNormOCL::calcInputGradients(cEstimator[-1])) ReturnFalse; //--- CNeuronBaseOCL* next = cEstimator[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* residual = NULL; uint count = iUnits * iVariables; //--- for(int i = cEstimator.Total() - 2; i >= 0; i--) { curr = cEstimator[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; if(curr.Type() == defNeuronBaseOCL && i > 0) { int concat_win = curr.Neurons() / (int)count; if(!!residual) if(!SumAndNormilize(curr.getGradient(), residual.getPrevOutput(), curr.getGradient(), concat_win, false, 0, 0, 0, 1)) ReturnFalse; next = cEstimator[i - 1]; if(!next) ReturnFalse; int prev_win = next.Neurons() / (int)count; if(!DeConcat(curr.getPrevOutput(), next.getGradient(), curr.getGradient(), concat_win - prev_win, prev_win, count)) ReturnFalse; residual = curr; Deactivation(next); i--; } else next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; if(!!residual) { if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), residual.getPrevOutput(), residual.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), residual.getPrevOutput(), NeuronOCL.getGradient(), NeuronOCL.Neurons() / count, false, 0, 0, 0, 1)) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- int total = cEstimator.Total(); //--- for(int i = 0; i < total; i++) { curr = cEstimator[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronBatchNormOCL::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBatchNormOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronCDCSelfCorrector* Source = source; if(!cEstimator.WeightsUpdate(Source.cEstimator.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::Save(const int file_handle) { if(!CNeuronBatchNormOCL::Save(file_handle)) ReturnFalse; if(!cEstimator.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::Load(const int file_handle) { if(!CNeuronBatchNormOCL::Load(file_handle)) ReturnFalse; if(!cEstimator.Load(file_handle)) ReturnFalse; //--- if(!cEstimator[0] || cEstimator[0].Type() != defNeuronMultiWindowsConvWPadOCL) ReturnFalse; CNeuronConvOCL* conv = cEstimator[0]; iUnits = conv.GetUnits(); iVariables = conv.GetVariables(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCDCSelfCorrector::Clear(void) { if(!CNeuronBatchNormOCL::Clear()) ReturnFalse; if(!cEstimator.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCDCSelfCorrector::SetOpenCL(COpenCLMy *obj) { CNeuronBatchNormOCL::SetOpenCL(obj); cEstimator.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCDCSelfCorrector::TrainMode(bool flag) { CNeuronBatchNormOCL::TrainMode(flag); cEstimator.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeCDC : public CNeuronSpikeConvBlock { protected: CNeuronCDCSelfCorrector cEsimator; CNeuronBaseOCL cFlowCorrector; CNeuronRelativeSelfAttention cFlowCorrelation; CNeuronBaseOCL cFlowMask; CNeuronBaseOCL cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeCDC(void) {}; ~CNeuronSpikeCDC(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint &inside_dimensions[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeCDC; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint &inside_dimensions[], uint window_out, uint units_count, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, window, window, window_out, units_count, variables, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cEsimator.Init(0, index, OpenCL, window, inside_dimensions, 2 * window, units_count, variables, optimization, iBatch)) ReturnFalse; index++; if(!cFlowCorrector.Init(0, index, OpenCL, cEsimator.Neurons() / 2, optimization, iBatch)) ReturnFalse; cFlowCorrector.SetActivationFunction(None); index++; if(!cFlowCorrelation.Init(0, index, OpenCL, window, (window + 3) / 4, units_count * variables, 4, optimization, iBatch)) ReturnFalse; index++; if(!cFlowMask.Init(0, index, OpenCL, cFlowCorrector.Neurons(), optimization, iBatch)) ReturnFalse; cFlowMask.SetActivationFunction(SIGMOID); index++; if(!cFlow.Init(0, index, OpenCL, cFlowCorrector.Neurons(), optimization, iBatch)) ReturnFalse; cFlow.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cEsimator.FeedForward(NeuronOCL)) ReturnFalse; //--- uint count = cConv.GetUnits() * cConv.GetVariables(); uint window = cConv.GetWindow(); if(!DeConcat(cFlowCorrector.getOutput(), cFlowMask.getOutput(), cEsimator.getOutput(), window, window, count)) ReturnFalse; dActivation(cFlowCorrector) dActivation(cFlowMask) if(!SumAndNormilize(cFlowCorrector.getOutput(), NeuronOCL.getOutput(), cFlowCorrector.getOutput(), window, false, 0, 0, 0, 1)) ReturnFalse; if(!cFlowCorrelation.FeedForward(cFlowCorrector.AsObject())) ReturnFalse; if(!GateElementMult(NeuronOCL.getOutput(), cFlowCorrelation.getOutput(), cFlowMask.getOutput(), cFlow.getOutput())) ReturnFalse; if(!CNeuronSpikeConvBlock::feedForward(cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow.AsObject())) ReturnFalse; if(!GateElementMultGrad(NeuronOCL.getOutput(), cFlowMask.getPrevOutput(), cFlowCorrelation.getOutput(), cFlowCorrelation.getGradient(), cFlowMask.getOutput(), cFlowMask.getGradient(), cFlow.getGradient(), NeuronOCL.Activation(), cFlowCorrelation.Activation(), cFlowMask.Activation())) ReturnFalse; if(!cFlowCorrector.CalcHiddenGradients(cFlowCorrelation.AsObject())) ReturnFalse; uint count = cConv.GetUnits() * cConv.GetVariables(); uint window = cFlowCorrector.Neurons() / count; if(!Concat(cFlowCorrector.getGradient(), cFlowMask.getGradient(), cEsimator.getGradient(), window, window, count)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cEsimator.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cFlowMask.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cEsimator.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cFlowCorrelation.UpdateInputWeights(cFlowCorrector.AsObject())) ReturnFalse; if(!CNeuronSpikeConvBlock::updateInputWeights(cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikeCDC* Source = source; if(!cEsimator.WeightsUpdate(Source.cEsimator.AsObject(), tau)) ReturnFalse; if(!cFlowCorrelation.WeightsUpdate(Source.cFlowCorrelation.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cEsimator.Save(file_handle)) ReturnFalse; if(!cFlowCorrelation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cEsimator.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFlowCorrelation.AsObject())) ReturnFalse; //--- uint index = 1; if(!cFlowCorrector.Init(0, index, OpenCL, cEsimator.Neurons() / 2, optimization, iBatch)) ReturnFalse; cFlowCorrector.SetActivationFunction(None); index += 2; if(!cFlowMask.Init(0, index, OpenCL, cFlowCorrector.Neurons(), optimization, iBatch)) ReturnFalse; cFlowMask.SetActivationFunction(SIGMOID); index++; if(!cFlow.Init(0, index, OpenCL, cFlowCorrector.Neurons(), optimization, iBatch)) ReturnFalse; cFlow.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeCDC::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cEsimator.Clear()) ReturnFalse; if(!cFlowCorrelation.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeCDC::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cEsimator.SetOpenCL(OpenCL); cFlowCorrector.SetOpenCL(OpenCL); cFlowCorrelation.SetOpenCL(OpenCL); cFlowMask.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeCDC::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cEsimator.TrainMode(bTrain); cFlowCorrector.TrainMode(bTrain); cFlowCorrelation.TrainMode(bTrain); cFlowMask.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeEEMFLow : public CNeuronSpikeCDC { protected: uint iMaxDisplacement; int ibShifts; CLayer cPrepare; CLayer cEncoder[3]; CNeuronBaseOCL cCorrelations[3]; CNeuronBaseOCL cConcat[3]; CLayer cDecoder[4]; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeEEMFLow(void) : ibShifts(INVALID_HANDLE) {}; ~CNeuronSpikeEEMFLow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeEEMFLow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual CBufferFloat *getWeights(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint embed_size, uint stack_size, uint variables, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch) { uint inside_dimensions[] = {32, 32, 32, 16, 8}; if(!CNeuronSpikeCDC::Init(numOutputs, myIndex, open_cl, embed_size, inside_dimensions, dimension, stack_size, variables, optimization_type, batch)) ReturnFalse; //--- CNeuronCreateFlow* stack = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeADM* adm = NULL; CNeuronMultiWindowsConvWPadOCL* padconv = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronSpikeCDC* cdc = NULL; //--- uint index = 0; cPrepare.Clear(); cPrepare.SetOpenCL(OpenCL); norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, dimension * variables, iBatch, optimization) || !cPrepare.Add(norm)) { DeleteObj(norm) ReturnFalse; } index++; stack = new CNeuronCreateFlow(); if(!stack || !stack.Init(0, index, OpenCL, stack_size, dimension, embed_size, variables, optimization, iBatch) || !cPrepare.Add(stack)) { DeleteObj(stack) ReturnFalse; } index++; adm = new CNeuronSpikeADM(); if(!adm || !adm.Init(0, index, OpenCL, embed_size, stack_size, variables, optimization, iBatch) || !cPrepare.Add(adm)) { DeleteObj(adm) ReturnFalse; } index++; //--- int shifts[]; if(ArrayResize(shifts, max_displacement) <= 0) ReturnFalse; uint count = 0; for(uint i = 0; i < max_displacement; i++) { if(i > 2 && bool(i & 1)) continue; shifts[count] = (int)(i + 1); count++; } if(ArrayResize(shifts, count) < (int)count) ReturnFalse; ibShifts = OpenCL.AddBufferFromArray(shifts, 0, count, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; iMaxDisplacement = max_displacement; //--- uint window_in = embed_size; uint window_out = 16; uint units_in = stack_size; uint units_out = (units_in + 1) / 2; uint windows[] = { 3 * window_in }; //--- for(int i = 0; i < 3; i++) { cEncoder[i].Clear(); cDecoder[i].Clear(); cEncoder[i].SetOpenCL(OpenCL); cDecoder[i].SetOpenCL(OpenCL); //--- Encoder padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, 2 * window_in, window_out, units_out, variables, optimization, iBatch) || !cEncoder[i].Add(padconv)) { DeleteObj(padconv) ReturnFalse; } padconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, padconv.Neurons(), iBatch, optimization) || !cEncoder[i].Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; windows[0] = 3 * window_out; for(int j = 0; j < 2; j++) { padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, window_out, window_out, units_out, variables, optimization, iBatch) || !cEncoder[i].Add(padconv)) { DeleteObj(padconv) ReturnFalse; } padconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, padconv.Neurons(), iBatch, optimization) || !cEncoder[i].Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; } conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, window_out, window_out, window_out, units_out, variables, optimization, iBatch) || !cEncoder[i].Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- if(!cCorrelations[i].Init(0, index, OpenCL, 2 * count * units_out * variables, optimization, iBatch)) ReturnFalse; cCorrelations[i].SetActivationFunction(None); index++; if(!cConcat[i].Init(0, index, OpenCL, 2 * (window_out + count) * units_out * variables, optimization, iBatch)) ReturnFalse; cConcat[i].SetActivationFunction(None); index++; //--- Decoder windows[0] = 2 * (window_out + count); padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, windows[0], window_out, units_out, variables, optimization, iBatch) || !cDecoder[i].Add(padconv)) { DeleteObj(padconv) ReturnFalse; } padconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, padconv.Neurons(), iBatch, optimization) || !cDecoder[i].Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; windows[0] = 3 * window_out; padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, window_out, 2 * window_in, units_out, variables, optimization, iBatch) || !cDecoder[i].Add(padconv)) { DeleteObj(padconv) ReturnFalse; } padconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, padconv.Neurons(), iBatch, optimization) || !cDecoder[i].Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; windows[0] = 3 * window_in; for(int j = 0; j < 2; j++) { padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, window_in, window_in, units_in, variables, optimization, iBatch) || !cDecoder[i].Add(padconv)) { DeleteObj(padconv) ReturnFalse; } padconv.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, padconv.Neurons(), iBatch, optimization) || !cDecoder[i].Add(norm)) { DeleteObj(norm) ReturnFalse; } norm.SetActivationFunction(None); index++; } conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, window_in, window_in, window_in, units_in, variables, optimization, iBatch) || !cDecoder[i].Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; if(i > 0) { cdc = new CNeuronSpikeCDC(); if(!cdc || !cdc.Init(0, index, OpenCL, window_in, inside_dimensions, window_in, units_in, variables, optimization, iBatch) || !cDecoder[i].Add(cdc)) { DeleteObj(cdc) ReturnFalse; } } index++; //--- window_in = window_out; window_out = 2 * window_in; units_in = units_out; units_out = (units_in + 1) / 2; windows[0] = 3 * window_in; } //--- cDecoder[3].Clear(); cDecoder[3].SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, window_in * units_in * variables, optimization, iBatch) || !cDecoder[3].Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); neuron.Clear(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronSpikeConvBlock* conv = NULL; //--- for(int l = 0; l < cPrepare.Total(); l++) { curr = cPrepare[l]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- for(uint i = 0; i < cEncoder.Size(); i++) { for(int l = 0; l < cEncoder[i].Total(); l++) { curr = cEncoder[i][l]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(prev.Type() != defNeuronSpikeConvBlock) ReturnFalse; conv = prev; if(!DilatedCorrelation(prev, cCorrelations[i].AsObject(), ibShifts, conv.GetFilters(), conv.GetUnits()*conv.GetVariables())) ReturnFalse; } //--- CNeuronMultiWindowsConvWPadOCL* padconv = NULL; for(int i = int(cDecoder.Size()) - 2; i >= 0; i--) { if(!cDecoder[i][0] || cDecoder[i][0].Type() != defNeuronMultiWindowsConvWPadOCL) ReturnFalse; if(!cEncoder[i][-1] || !cDecoder[i + 1][-1]) ReturnFalse; //--- padconv = cDecoder[i][0]; uint units = padconv.GetUnits() * padconv.GetVariables(); uint window1 = cEncoder[i][-1].Neurons() / units; uint window2 = cDecoder[i][-1].Neurons() / units; uint window3 = cCorrelations[i].Neurons() / units; if(!Concat(cEncoder[i][-1].getOutput(), cDecoder[i + 1][-1].getOutput(), cCorrelations[i].getOutput(), cConcat[i].getOutput(), window1, window2, window3, units)) ReturnFalse; prev = cConcat[i].AsObject(); for(int l = 0; l < cDecoder[i].Total(); l++) { curr = cDecoder[i][l]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronSpikeCDC) if(!SumAndNormilize(cEncoder[i - 1][-1].getOutput(), prev.getOutput(), prev.getOutput(), ((CNeuronSpikeCDC*)curr).GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!curr.FeedForward(prev)) ReturnFalse; prev = curr; } } //--- if(!SumAndNormilize(cPrepare[-1].getOutput(), prev.getOutput(), prev.getOutput(), GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronSpikeCDC::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeCDC::calcInputGradients(cDecoder[0][-1])) ReturnFalse; //--- Decoder CNeuronBaseOCL* next = cDecoder[0][-1]; CNeuronBaseOCL* curr = NULL; CNeuronMultiWindowsConvWPadOCL* padconv = NULL; for(uint i = 0; i < cDecoder.Size() - 1; i++) { for(int l = cDecoder[i].Total() - 2; l >= 0; l--) { curr = cDecoder[i][l]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!cConcat[i].CalcHiddenGradients(next)) ReturnFalse; if(next.Type() != defNeuronMultiWindowsConvWPadOCL) ReturnFalse; if(!cEncoder[i][-1] || !cDecoder[i + 1][-1]) ReturnFalse; padconv = next; next = cDecoder[i + 1][-1]; curr = cEncoder[i][-1]; uint units = padconv.GetUnits() * padconv.GetVariables(); uint window1 = curr.Neurons() / units; uint window2 = next.Neurons() / units; uint window3 = cCorrelations[i].Neurons() / units; if(!DeConcat(cCorrelations[i].getPrevOutput(), next.getGradient(), cCorrelations[i].getGradient(), cConcat[i].getGradient(), window1, window2, window3, units)) ReturnFalse; Deactivation(next); if(!DilatedCorrelationGrad(curr, cCorrelations[i].AsObject(), ibShifts, window1, units)) ReturnFalse; if(i == cEncoder.Size() - 1) { if(!SumAndNormilize(curr.getGradient(), cCorrelations[i].getPrevOutput(), curr.getGradient(), window1, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(curr) } else { if(!SumAndNormilize(curr.getGradient(), cCorrelations[i].getPrevOutput(), cCorrelations[i].getPrevOutput(), window1, false, 0, 0, 0, 1)) ReturnFalse; if(curr.Activation() != None) if(!DeActivation(curr.getOutput(), cCorrelations[i].getPrevOutput(), cCorrelations[i].getPrevOutput(), curr.Activation())) ReturnFalse; } } //--- Encoder for(int i = int(cEncoder.Size()) - 1; i >= 0; i--) { next = cEncoder[i][-1]; for(int l = cEncoder[i].Total() - 2; l >= 0; l--) { curr = cEncoder[i][l]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(i == 0) continue; curr = cEncoder[i - 1][-1]; if(!curr || !SumAndNormilize(curr.getGradient(), cCorrelations[i - 1].getPrevOutput(), curr.getGradient(), ((CNeuronSpikeConvBlock*)curr).GetFilters(), false, 0, 0, 0, 1)) ReturnFalse; } //--- Prepare for(int l = cPrepare.Total() - 1; l >= 0; l--) { curr = cPrepare[l]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int l = 0; l < cPrepare.Total(); l++) { curr = cPrepare[l]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- for(uint i = 0; i < cEncoder.Size(); i++) { for(int l = 0; l < cEncoder[i].Total(); l++) { curr = cEncoder[i][l]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } } //--- for(int i = int(cDecoder.Size()) - 2; i >= 0; i--) { prev = cConcat[i].AsObject(); for(int l = 0; l < cDecoder[i].Total(); l++) { curr = cDecoder[i][l]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } } //--- if(!CNeuronSpikeCDC::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DilatedCorrelation(CNeuronBaseOCL *feature, CNeuronBaseOCL *correlations, uint shifts_index, uint dimension, uint units) { if(!OpenCL || !feature || !correlations || units <= 0 || dimension <= 0) ReturnFalse; //--- uint corrs = correlations.Neurons() / units; uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(dimension, OpenCL.GetMaxLocalSize(1)), corrs }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_DilatedCorrelation; setBuffer(kernel, def_k_dcor_correlations, correlations.getOutputIndex()) setBuffer(kernel, def_k_dcor_feature, feature.getOutputIndex()) setBuffer(kernel, def_k_dcor_shifts, shifts_index) setArgument(kernel, def_k_dcor_dimension, dimension) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!correlations.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DilatedCorrelationGrad(CNeuronBaseOCL *feature, CNeuronBaseOCL *correlations, uint shifts_index, uint dimension, uint units) { if(!OpenCL || !feature || !correlations || units <= 0 || dimension <= 0) ReturnFalse; //--- uint corrs = correlations.Neurons() / units; uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(corrs, OpenCL.GetMaxLocalSize(1)), dimension }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_DilatedCorrelationGrad; setBuffer(kernel, def_k_dcorgr_corr_gr, correlations.getGradientIndex()) setBuffer(kernel, def_k_dcorgr_feature, feature.getOutputIndex()) setBuffer(kernel, def_k_dcorgr_feature_gr, feature.getGradientIndex()) setBuffer(kernel, def_k_dcorgr_shifts, shifts_index) setArgument(kernel, def_k_dcorgr_total_corr, corrs) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!feature.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeCDC::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeEEMFLow* Source = source; for(uint i = 0; i < cEncoder.Size(); i++) if(!cEncoder[i].WeightsUpdate(Source.cEncoder[i].AsObject(), tau)) ReturnFalse; for(uint i = 0; i < cDecoder.Size(); i++) if(!cDecoder[i].WeightsUpdate(Source.cDecoder[i].AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::Save(const int file_handle) { if(!CNeuronSpikeCDC::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; for(uint i = 0; i < cEncoder.Size(); i++) if(!cEncoder[i].Save(file_handle)) ReturnFalse; for(uint i = 0; i < cDecoder.Size(); i++) if(!cDecoder[i].Save(file_handle)) ReturnFalse; if(FileWriteInteger(file_handle, iMaxDisplacement, INT_VALUE) < INT_VALUE) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::Load(const int file_handle) { if(!CNeuronSpikeCDC::Load(file_handle)) ReturnFalse; //--- if(!cPrepare.Load(file_handle)) ReturnFalse; for(uint i = 0; i < cEncoder.Size(); i++) if(!cEncoder[i].Load(file_handle)) ReturnFalse; for(uint i = 0; i < cDecoder.Size(); i++) if(!cDecoder[i].Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iMaxDisplacement = (uint)FileReadInteger(file_handle); //--- OpenCL.BufferFree(ibShifts); //--- int shifts[]; if(ArrayResize(shifts, iMaxDisplacement) <= 0) ReturnFalse; uint count = 0; for(uint i = 0; i < iMaxDisplacement; i++) { if(i > 2 && bool(i & 1)) continue; shifts[count] = (int)(i + 1); count++; } if(ArrayResize(shifts, count) < (int)count) ReturnFalse; ibShifts = OpenCL.AddBufferFromArray(shifts, 0, count, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- int index = 0; CNeuronSpikeConvBlock* conv = NULL; for(uint i = 0; i < cEncoder.Size(); i++) { index += cEncoder[i].Total(); conv = cEncoder[i][-1]; uint units = conv.GetUnits(); uint variables = conv.GetVariables(); uint window_out = conv.GetFilters(); //--- if(!cCorrelations[i].Init(0, index, OpenCL, 2 * count * units * variables, optimization, iBatch)) ReturnFalse; cCorrelations[i].SetActivationFunction(None); index++; if(!cConcat[i].Init(0, index, OpenCL, 2 * (window_out + count) * units * variables, optimization, iBatch)) ReturnFalse; cConcat[i].SetActivationFunction(None); index += cDecoder[i].Total() + 1; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeEEMFLow::Clear(void) { if(!CNeuronSpikeCDC::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; for(uint i = 0; i < cEncoder.Size(); i++) if(!cEncoder[i].ClearStates()) ReturnFalse; for(uint i = 0; i < cDecoder.Size(); i++) if(!cDecoder[i].ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeEEMFLow::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); //--- CNeuronSpikeCDC::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); for(uint i = 0; i < cEncoder.Size(); i++) cEncoder[i].SetOpenCL(OpenCL); for(uint i = 0; i < cDecoder.Size(); i++) cDecoder[i].SetOpenCL(OpenCL); //--- int shifts[]; ArrayResize(shifts, iMaxDisplacement); uint count = 0; for(uint i = 0; i < iMaxDisplacement; i++) { if(i > 2 && bool(i & 1)) continue; shifts[count] = (int)(i + 1); count++; } ArrayResize(shifts, count); ibShifts = OpenCL.AddBufferFromArray(shifts, 0, count, CL_MEM_READ_ONLY); //--- for(uint i = 0; i < cEncoder.Size(); i++) { cCorrelations[i].SetOpenCL(OpenCL); cConcat[i].SetOpenCL(OpenCL); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeEEMFLow::TrainMode(bool flag) { CNeuronSpikeCDC::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); for(uint i = 0; i < cEncoder.Size(); i++) cEncoder[i].TrainMode(bTrain); for(uint i = 0; i < cDecoder.Size(); i++) cDecoder[i].TrainMode(bTrain); for(uint i = 0; i < cEncoder.Size(); i++) { cCorrelations[i].TrainMode(bTrain); cConcat[i].TrainMode(bTrain); } } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronSpikeEEMFLow::getWeights(void) { CBufferFloat* result = NULL; for(int layerNum = 0; (layerNum < cPrepare.Total() && !result && !IsStopped()); layerNum++) { CNeuronBaseOCL *temp = cPrepare[layerNum]; CNeuronConvOCL *conv = NULL; CNeuronConcatenate *conc = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; switch(temp.Type()) { case defNeuronConvOCL: conv = temp; result = conv.GetWeightsConv(); break; case defNeuronConcatenate: conc = temp; result = conc.getConcatWeights(); break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; result = batch.getBatchOptions(); break; case defNeuronLSTMOCL: lstm = temp; result = lstm.getLSTMWeights(); break; default: result = temp.getWeights(); break; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleExcitation : public CNeuronBaseOCL { protected: CLayer cExcitation; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultiScaleExcitation(void) {}; ~CNeuronMultiScaleExcitation(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint scales, uint bottleneck, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultiScaleExcitation; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint scales, uint bottleneck, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units * scales, optimization_type, batch)) ReturnFalse; //--- cExcitation.Clear(); cExcitation.SetOpenCL(OpenCL); CNeuronSpikeConvBlock* conv = NULL; CNeuronTransposeOCL* transp = NULL; CNeuronSoftMaxOCL* softmax = NULL; uint index = 0; //--- [Scale, Unit, Dimension] transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, scales, units * dimension, optimization, iBatch) || !cExcitation.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; //--- [Unit, Dimension, Scale] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, scales, scales, 1, dimension * units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Dimension] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension, dimension, bottleneck, units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Bottleneck] conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, scales * dimension, units, 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- [Unit, Dimension, Scale] softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) || !cExcitation.Add(softmax)) { DeleteObj(softmax) ReturnFalse; } softmax.SetHeads(dimension * units); index++; //--- [Unit, Dimension, Scale] transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units * dimension, scales, optimization, iBatch) || !cExcitation.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; //--- [Scale, Unit, Dimension] return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!ElementMult(NeuronOCL.getOutput(), prev.getOutput(), Output)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* next = cExcitation[-1]; CNeuronBaseOCL* curr = NULL; //--- if(!NeuronOCL || !next) ReturnFalse; if(!ElementMultGrad(NeuronOCL.getOutput(), PrevOutput, next.getOutput(), next.getGradient(), Gradient, NeuronOCL.Activation(), next.Activation())) ReturnFalse; //--- for(int i = cExcitation.Total() - 2; i >= 0; i--) { curr = cExcitation[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMultiScaleExcitation* Source = source; if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cExcitation.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cExcitation.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleExcitation::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; if(!cExcitation.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleExcitation::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cExcitation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleExcitation::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cExcitation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DilatedDifference(CNeuronBaseOCL *feature, CNeuronBaseOCL *difference, uint shifts_index, uint dimension, uint units) { if(!OpenCL || !feature || !difference || units <= 0 || dimension <= 0) ReturnFalse; //--- uint shifts = difference.Neurons() / feature.Neurons(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, shifts, dimension }; uint kernel = def_k_DilatedDifference; setBuffer(kernel, def_k_ddiff_differences, difference.getOutputIndex()) setBuffer(kernel, def_k_ddiff_feature, feature.getOutputIndex()) setBuffer(kernel, def_k_ddiff_shifts, shifts_index) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!difference.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::DilatedDifferenceGrad(CNeuronBaseOCL *feature, CNeuronBaseOCL *difference, uint shifts_index, uint dimension, uint units) { if(!OpenCL || !feature || !difference || units <= 0 || dimension <= 0) ReturnFalse; //--- uint shifts = difference.Neurons() / feature.Neurons(); uint global_work_offset[3] = {0}; uint global_work_size[] = { units, (uint)MathMin(shifts, OpenCL.GetMaxLocalSize(1)), dimension }; uint local_work_size[] = { 1, global_work_size[1], 1 }; uint kernel = def_k_DilatedDifferenceGrad; setBuffer(kernel, def_k_ddiffgr_differences_gr, difference.getGradientIndex()) setBuffer(kernel, def_k_ddiffgr_feature, feature.getOutputIndex()) setBuffer(kernel, def_k_ddiffgr_feature_gr, feature.getGradientIndex()) setBuffer(kernel, def_k_ddiffgr_shifts, shifts_index) setArgument(kernel, def_k_ddiffgr_total_shifts, shifts) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!feature.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiScaleDifference : public CNeuronSpikeConvBlock { protected: uint iaShifts[]; int ibShifts; //--- CLayer cBottleneck; CLayer cExcitation; CLayer cProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultiScaleDifference(void) {}; ~CNeuronMultiScaleDifference(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint bottleneck, uint &shifts[], ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronMultiScaleDifference; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint bottleneck, uint &shifts[], ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck, bottleneck, dimension, units, 1, optimization_type, batch)) ReturnFalse; //--- int scales = int(shifts.Size()); if(scales == 0 || ArrayResize(iaShifts, scales) < scales || ArrayCopy(iaShifts, shifts, 0, 0, scales) < scales) ReturnFalse; ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, scales, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- cBottleneck.Clear(); cExcitation.Clear(); cProjection.Clear(); cBottleneck.SetOpenCL(OpenCL); cExcitation.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); //--- CNeuronMultiWindowsConvWPadOCL* padconv = NULL; CNeuronSpikeDepthWiseConv* dwconv = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronMultiScaleExcitation* excitation = NULL; CNeuronTransposeOCL* transpose = NULL; uint index = 0; //--- Bottleneck uint units_out = (units + 1) / 2; dwconv = new CNeuronSpikeDepthWiseConv(); if(!dwconv || !dwconv.Init(0, index, OpenCL, dimension, bottleneck, 5, 2, units_out, 1, optimization, iBatch) || !cBottleneck.Add(dwconv)) { DeleteObj(dwconv); ReturnFalse; } index++; uint windows[] = { 3 * bottleneck}; padconv = new CNeuronMultiWindowsConvWPadOCL(); if(!padconv || !padconv.Init(0, index, OpenCL, windows, bottleneck, bottleneck, units_out, 1, optimization, iBatch) || !cBottleneck.Add(padconv)) { DeleteObj(padconv) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out, 1, optimization, iBatch) || !cBottleneck.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- Excitation neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, conv.Neurons()*scales, optimization, iBatch) || !cExcitation.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; excitation = new CNeuronMultiScaleExcitation(); if(!excitation || !excitation.Init(0, index, OpenCL, bottleneck, units_out, scales, (bottleneck + 1) / 2, optimization, iBatch) || !cExcitation.Add(excitation)) { DeleteObj(excitation) ReturnFalse; } index++; transpose = new CNeuronTransposeOCL(); if(!transpose || !transpose.Init(0, index, OpenCL, bottleneck * units_out, scales, optimization, iBatch) || !cExcitation.Add(transpose)) { DeleteObj(transpose) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, scales, scales, 1, transpose.GetCount(), 1, optimization, iBatch) || !cExcitation.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; //--- Projection neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, conv.Neurons(), optimization, iBatch) || !cProjection.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); if(!neuron.SetGradient(conv.getGradient(), true)) ReturnFalse; index++; dwconv = new CNeuronSpikeDepthWiseConv(); if(!dwconv || !dwconv.Init(0, index, OpenCL, bottleneck, 2 * bottleneck, 5, 1, units_out, 1, optimization, iBatch) || !cProjection.Add(dwconv)) { DeleteObj(dwconv); ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Bottleneck for(int i = 0; i < cBottleneck.Total(); i++) { curr = cBottleneck[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Excitation curr = cExcitation[0]; if(!curr) ReturnFalse; uint dimension = cConv.GetWindow(); uint units = prev.Neurons() / dimension; if(!DilatedDifference(prev, curr, ibShifts, dimension, units)) ReturnFalse; prev = curr; for(int i = 1; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Projection curr = cProjection[0]; if(!curr || !SumAndNormilize(cBottleneck[-1].getOutput(), prev.getOutput(), curr.getOutput(), dimension, true, 0, 0, 0, 1)) ReturnFalse; prev = curr; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), Output, Output, cConv.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Projection CNeuronBaseOCL* next = cProjection[-1]; CNeuronBaseOCL* curr = NULL; if(!CNeuronSpikeConvBlock::calcInputGradients(next)) ReturnFalse; for(int i = cProjection.Total() - 2; i >= 0; i--) { curr = cProjection[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Excitation next = cExcitation[-1]; if(!next) ReturnFalse; Deactivation(next) for(int i = cExcitation.Total() - 2; i >= 0; i--) { curr = cExcitation[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Bottleneck curr = cBottleneck[-1]; if(!curr) ReturnFalse; uint dimension = cConv.GetWindow(); uint units = curr.Neurons() / dimension; if(!DilatedDifferenceGrad(curr, next, ibShifts, dimension, units)) ReturnFalse; Deactivation(curr) next = curr; for(int i = cBottleneck.Total() - 2; i >= 0; i--) { curr = cBottleneck[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Bottleneck for(int i = 0; i < cBottleneck.Total(); i++) { curr = cBottleneck[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Excitation prev = cExcitation[0]; for(int i = 1; i < cExcitation.Total(); i++) { curr = cExcitation[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Projection prev = cProjection[0]; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- uint size = iaShifts.Size(); if(FileWriteInteger(file_handle, size, INT_VALUE) < INT_VALUE) ReturnFalse; for(uint i = 0; i < size; i++) { if(FileWriteInteger(file_handle, iaShifts[i], INT_VALUE) < INT_VALUE) ReturnFalse; } //--- if(!cBottleneck.Save(file_handle)) ReturnFalse; if(!cExcitation.Save(file_handle)) ReturnFalse; if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; uint size = FileReadInteger(file_handle); if(ArrayResize(iaShifts, size) < (int)size) ReturnFalse; for(uint i = 0; i < size; i++) { if(FileIsEnding(file_handle)) ReturnFalse; iaShifts[i] = FileReadInteger(file_handle); } OpenCL.BufferFree(ibShifts); ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, size, CL_MEM_READ_ONLY); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- if(!cBottleneck.Load(file_handle)) ReturnFalse; if(!cExcitation.Load(file_handle)) ReturnFalse; if(!cProjection.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMultiScaleDifference* Source = source; if(!cBottleneck.WeightsUpdate(Source.cBottleneck.AsObject(), tau)) ReturnFalse; if(!cExcitation.WeightsUpdate(Source.cExcitation.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiScaleDifference::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cBottleneck.ClearStates()) ReturnFalse; if(!cExcitation.ClearStates()) ReturnFalse; if(!cProjection.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleDifference::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); CNeuronSpikeConvBlock::SetOpenCL(obj); ibShifts = OpenCL.AddBufferFromArray(iaShifts, 0, iaShifts.Size(), CL_MEM_READ_ONLY); cBottleneck.SetOpenCL(OpenCL); cExcitation.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiScaleDifference::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cBottleneck.TrainMode(bTrain); cExcitation.TrainMode(bTrain); cProjection.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCorrelationEncoder : public CNeuronSpikeConvBlock { protected: uint iMaxDisplacement; int ibShifts; CNeuronBaseOCL cCorrelation; CNeuronSpikeConvBlock cWarpCorrs; CNeuronSpikeConvBlock cWarpInps; CLayer cProjection; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCorrelationEncoder(void) {}; ~CNeuronCorrelationEncoder(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint bottleneck, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCorrelationEncoder; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ int CNeuronBaseOCL::CreateShifts(const uint max_displacement, int &count) { if(!OpenCL || max_displacement < 1) return INVALID_HANDLE; //--- int shifts[]; if(ArrayResize(shifts, max_displacement) <= 0) return INVALID_HANDLE; //--- count = 0; for(uint i = 0; i < max_displacement; i++) { if(i > 2 && bool(i & 1)) continue; shifts[count] = (int)(i + 1); count++; } if(ArrayResize(shifts, count) < (int)count) return INVALID_HANDLE; //--- return OpenCL.AddBufferFromArray(shifts, 0, count, CL_MEM_READ_ONLY); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint bottleneck, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, 2 * bottleneck, 2 * bottleneck, dimension, units, 1, optimization_type, batch)) ReturnFalse; //--- int count = 0; ibShifts = CreateShifts(max_displacement, count); if(ibShifts == INVALID_HANDLE) ReturnFalse; iMaxDisplacement = max_displacement; //--- uint index = 0; if(!cCorrelation.Init(0, index, OpenCL, units * count, optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); index++; if(!cWarpCorrs.Init(0, index, OpenCL, count, count, bottleneck, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cWarpInps.Init(0, index, OpenCL, dimension, dimension, bottleneck, units, 1, optimization, iBatch)) ReturnFalse; index++; //--- cProjection.Clear(); cProjection.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronMSRes* res_block = NULL; //--- neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, cWarpCorrs.Neurons() + cWarpInps.Neurons(), optimization, iBatch) || !cProjection.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; res_block = new CNeuronMSRes(); if(!res_block || !res_block.Init(0, index, OpenCL, units, 2 * bottleneck, 1, optimization, iBatch) || !cProjection.Add(res_block)) { DeleteObj(res_block) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cWarpInps.FeedForward(NeuronOCL)) ReturnFalse; if(!DilatedCorrelation(NeuronOCL, cCorrelation.AsObject(), ibShifts, cWarpInps.GetWindow(), cWarpInps.GetUnits())) ReturnFalse; if(!cWarpCorrs.FeedForward(cCorrelation.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* curr = cProjection[0]; CNeuronBaseOCL* prev = NULL; //--- if(!curr || !Concat(cWarpInps.getOutput(), cWarpCorrs.getOutput(), curr.getOutput(), cWarpInps.GetFilters(), cWarpCorrs.GetFilters(), cWarpInps.GetUnits())) ReturnFalse; prev = curr; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), Output, Output, cConv.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cProjection[-1])) ReturnFalse; //--- CNeuronBaseOCL* next = cProjection[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cProjection.Total() - 2; i >= 0; i--) { curr = cProjection[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!DeConcat(cWarpInps.getGradient(), cWarpCorrs.getGradient(), next.getGradient(), cWarpInps.GetFilters(), cWarpCorrs.GetFilters(), cWarpInps.GetUnits())) ReturnFalse; if(!cCorrelation.CalcHiddenGradients(cWarpCorrs.AsObject())) ReturnFalse; if(!DilatedCorrelationGrad(NeuronOCL, cCorrelation.AsObject(), ibShifts, cWarpInps.GetWindow(), cWarpInps.GetUnits())) ReturnFalse; //--- CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(PrevOutput, false) || !NeuronOCL.CalcHiddenGradients(cWarpInps.AsObject()) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, cWarpInps.GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cWarpInps.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cWarpCorrs.UpdateInputWeights(cCorrelation.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* curr = cProjection[0]; CNeuronBaseOCL* prev = NULL; //--- prev = curr; for(int i = 1; i < cProjection.Total(); i++) { curr = cProjection[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, iMaxDisplacement, INT_VALUE) < INT_VALUE) ReturnFalse; //--- if(!cWarpCorrs.Save(file_handle)) ReturnFalse; if(!cWarpInps.Save(file_handle)) ReturnFalse; //--- if(!cProjection.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::Load(const int file_handle) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(FileIsEnding(file_handle)) ReturnFalse; iMaxDisplacement = FileReadInteger(file_handle); //--- if(!LoadInsideLayer(file_handle, cWarpCorrs.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cWarpInps.AsObject())) ReturnFalse; //--- if(!cProjection.Load(file_handle)) ReturnFalse; //--- int count = 0; ibShifts = CreateShifts(iMaxDisplacement, count); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- uint index = 0; if(!cCorrelation.Init(0, index, OpenCL, cWarpCorrs.GetUnits() * count, optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronCorrelationEncoder* Source = source; //--- if(!cWarpCorrs.WeightsUpdate(Source.cWarpCorrs.AsObject(), tau)) ReturnFalse; if(!cWarpInps.WeightsUpdate(Source.cWarpInps.AsObject(), tau)) ReturnFalse; if(!cProjection.WeightsUpdate(Source.cProjection.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCorrelationEncoder::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cWarpCorrs.Clear()) ReturnFalse; if(!cWarpInps.Clear()) ReturnFalse; if(!cProjection.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCorrelationEncoder::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); CNeuronSpikeConvBlock::SetOpenCL(obj); cWarpCorrs.SetOpenCL(OpenCL); cWarpInps.SetOpenCL(OpenCL); cProjection.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); //--- int count = 0; ibShifts = CreateShifts(iMaxDisplacement, count); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCorrelationEncoder::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cWarpCorrs.TrainMode(bTrain); cWarpInps.TrainMode(bTrain); cProjection.TrainMode(bTrain); cCorrelation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronEDCFlow : public CNeuronSpikeConvBlock { protected: CLayer cPrepare; CLayer cFeatureEncoder[2]; CLayer cConextEncoder; CNeuronMultiScaleDifference cDiffEncoder; CNeuronCorrelationEncoder cCorrelation; CLayer cCorrelationUpsampling; CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronEDCFlow(void) {}; ~CNeuronEDCFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint units_in, uint bottleneck, uint dimension_out, uint units_out, uint &shifts[], uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronEDCFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual CBufferFloat *getWeights(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint units_in, uint bottleneck, uint dimension_out, uint units_out, uint &shifts[], uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck, bottleneck, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- for(uint i = 0; i < cFeatureEncoder.Size(); i++) { cFeatureEncoder[i].Clear(); cFeatureEncoder[i].SetOpenCL(OpenCL); } cPrepare.Clear(); cConextEncoder.Clear(); cCorrelationUpsampling.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cConextEncoder.SetOpenCL(OpenCL); cCorrelationUpsampling.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronCreateFlow* stack = NULL; CNeuronSpikeADM* adm = NULL; CNeuronSpikeDepthWiseConv* dw_conv = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronTransposeOCL* transp = NULL; CNeuronMultiScaleExcitation* excitation = NULL; CNeuronSpikeConvGRU2D* gru = NULL; uint index = 0; //--- Prepare norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, dimension_in, iBatch, optimization) || !cPrepare.Add(norm)) { DeleteObj(norm) ReturnFalse; } index++; stack = new CNeuronCreateFlow(); if(!stack || !stack.Init(0, index, OpenCL, units_in, dimension_in, dimension_in, 1, optimization, iBatch) || !cPrepare.Add(stack)) { DeleteObj(stack) ReturnFalse; } index++; adm = new CNeuronSpikeADM(); if(!adm || !adm.Init(0, index, OpenCL, dimension_in, units_in, 1, optimization, iBatch) || !cPrepare.Add(adm)) { DeleteObj(adm) ReturnFalse; } index++; //--- Feature Encoder HD uint units = (units_in + 1) / 2; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, dimension_in, dimension_in, 5, 2, units, 1, optimization, iBatch) || !cFeatureEncoder[0].Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; units = (units + 1) / 2; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, dimension_in, bottleneck, 5, 2, units, 1, optimization, iBatch) || !cFeatureEncoder[0].Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; if(!cDiffEncoder.Init(0, index, OpenCL, bottleneck, units, bottleneck, shifts, optimization, iBatch)) ReturnFalse; index++; //--- Feature Encoder LD units = (units + 1) / 2; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, bottleneck, 2 * bottleneck, 5, 2, units, 1, optimization, iBatch) || !cFeatureEncoder[1].Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; if(!cCorrelation.Init(0, index, OpenCL, 2 * bottleneck, units, bottleneck, max_displacement, optimization, iBatch)) ReturnFalse; index++; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, 2 * bottleneck, 2 * bottleneck, 5, 1, units, 1, optimization, iBatch) || !cCorrelationUpsampling.Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; //--- Conext Encoder units = (units_in + 1) / 2; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, dimension_in, dimension_in, 10, 2, units, 1, optimization, iBatch) || !cConextEncoder.Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; units = (units + 1) / 2; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, dimension_in, bottleneck, 10, 2, units, 1, optimization, iBatch) || !cConextEncoder.Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; //--- Flow neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 4 * bottleneck, optimization, iBatch) || !cFlow.Add(neuron)) { DeleteObj(neuron) ReturnFalse; } neuron.SetActivationFunction(None); index++; excitation = new CNeuronMultiScaleExcitation(); if(!excitation || !excitation.Init(0, index, OpenCL, bottleneck, units, 4, (bottleneck + 1) / 2, optimization, iBatch) || !cFlow.Add(excitation)) { DeleteObj(excitation) ReturnFalse; } index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 4 * bottleneck, 4 * bottleneck, bottleneck, units, 1, optimization, iBatch) || !cFlow.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, bottleneck, bottleneck, 3, 1, units, 1, optimization, iBatch) || !cFlow.Add(dw_conv)) { DeleteObj(dw_conv) ReturnFalse; } index++; gru = new CNeuronSpikeConvGRU2D(); if(!gru || !gru.Init(0, index, OpenCL, units, bottleneck, bottleneck, optimization, iBatch) || !cFlow.Add(gru)) { DeleteObj(gru) ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units, bottleneck, optimization, iBatch) || !cFlow.Add(transp)) { DeleteObj(transp) ReturnFalse; } index++; conv = CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, units, units, units_out, bottleneck, 1, optimization, iBatch) || !cFlow.Add(conv)) { DeleteObj(conv) ReturnFalse; } index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, bottleneck, units_out, optimization, iBatch) || !cFlow.Add(transp)) { DeleteObj(transp) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Prepare for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) { for(int i = 0; i < cFeatureEncoder[fe].Total(); i++) { curr = cFeatureEncoder[fe][i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } } //--- Context Encoder prev = cPrepare[-1]; for(int i = 0; i < cConextEncoder.Total(); i++) { curr = cConextEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Moution Encoder if(!cDiffEncoder.FeedForward(cFeatureEncoder[0][-1].AsObject())) ReturnFalse; if(!cCorrelation.FeedForward(cFeatureEncoder[1][-1].AsObject())) ReturnFalse; prev = cCorrelation.AsObject(); for(int i = 0; i < cCorrelationUpsampling.Total(); i++) { curr = cCorrelationUpsampling[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } curr = cFlow[0]; uint units = cDiffEncoder.GetUnits(); uint bottleneck = cDiffEncoder.GetFilters(); if(!curr || !Concat(cFeatureEncoder[0][-1].getOutput(), cConextEncoder[-1].getOutput(), cDiffEncoder.getOutput(), prev.getOutput(), curr.getOutput(), bottleneck, bottleneck, bottleneck, bottleneck, units)) ReturnFalse; prev = curr; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- Flow CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- uint units = cDiffEncoder.GetUnits(); uint bottleneck = cDiffEncoder.GetFilters(); if(!cConextEncoder[-1] || !cCorrelationUpsampling[-1]) ReturnFalse; if(!DeConcat(cDiffEncoder.getPrevOutput(), cConextEncoder[-1].getGradient(), cDiffEncoder.getGradient(), cCorrelationUpsampling[-1].getGradient(), next.getGradient(), bottleneck, bottleneck, bottleneck, bottleneck, units)) ReturnFalse; Deactivation(cConextEncoder[-1]) Deactivation(cCorrelationUpsampling[-1]) //--- Correlation Encoder next = cCorrelationUpsampling[-1]; for(int i = cCorrelationUpsampling.Total() - 2; i >= 0; i--) { curr = cCorrelationUpsampling[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!cCorrelation.CalcHiddenGradients(next)) ReturnFalse; next = cCorrelation.AsObject(); //--- Feature Encoder LD for(int i = cFeatureEncoder[1].Total() - 1; i >= 0; i--) { curr = cFeatureEncoder[1][i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Feature Encoder HD curr = cFeatureEncoder[0][-1]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; if(curr.Activation() != None) if(!DeActivation(curr.getOutput(), cDiffEncoder.getPrevOutput(), cDiffEncoder.getPrevOutput(), curr.Activation())) ReturnFalse; if(!SumAndNormilize(curr.getGradient(), cDiffEncoder.getPrevOutput(), cDiffEncoder.getPrevOutput(), bottleneck, false, 0, 0, 0, 1)) ReturnFalse; if(!curr.CalcHiddenGradients(cDiffEncoder.AsObject()) || !SumAndNormilize(curr.getGradient(), cDiffEncoder.getPrevOutput(), curr.getPrevOutput(), bottleneck, false, 0, 0, 0, 1)) ReturnFalse; next = curr; for(int i = cFeatureEncoder[0].Total() - 2; i >= 0; i--) { curr = cFeatureEncoder[0][i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Context Encoder next = cConextEncoder[-1]; for(int i = cConextEncoder.Total() - 2; i >= 0; i--) { curr = cConextEncoder[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Prepare curr = cPrepare[-1]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; CBufferFloat* temp = curr.getGradient(); if(!curr.SetGradient(curr.getPrevOutput(), false) || !curr.CalcHiddenGradients(cFeatureEncoder[0][0]) || !SumAndNormilize(temp, curr.getGradient(), temp, 1, false, 0, 0, 0, 1) || !curr.SetGradient(temp, false)) ReturnFalse; next = curr; for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Prepare for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) { for(int i = 0; i < cFeatureEncoder[fe].Total(); i++) { curr = cFeatureEncoder[fe][i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } } //--- Context Encoder prev = cPrepare[-1]; for(int i = 0; i < cConextEncoder.Total(); i++) { curr = cConextEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Moution Encoder if(!cDiffEncoder.UpdateInputWeights(cFeatureEncoder[0][-1].AsObject())) ReturnFalse; if(!cCorrelation.UpdateInputWeights(cFeatureEncoder[1][-1].AsObject())) ReturnFalse; prev = cCorrelation.AsObject(); for(int i = 0; i < cCorrelationUpsampling.Total(); i++) { curr = cCorrelationUpsampling[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- Prepare if(!cPrepare.Save(file_handle)) ReturnFalse; //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) if(!cFeatureEncoder[fe].Save(file_handle)) ReturnFalse; //--- Context Encoder if(!cConextEncoder.Save(file_handle)) ReturnFalse; //--- Moution Encoder if(!cDiffEncoder.Save(file_handle)) ReturnFalse; if(!cCorrelation.Save(file_handle)) ReturnFalse; if(!cCorrelationUpsampling.Save(file_handle)) ReturnFalse; //--- Flow if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- Prepare if(!cPrepare.Load(file_handle)) ReturnFalse; //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) if(!cFeatureEncoder[fe].Load(file_handle)) ReturnFalse; //--- Context Encoder if(!cConextEncoder.Load(file_handle)) ReturnFalse; //--- Moution Encoder if(!LoadInsideLayer(file_handle, cDiffEncoder.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCorrelation.AsObject())) ReturnFalse; if(!cCorrelationUpsampling.Load(file_handle)) ReturnFalse; //--- Flow if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronEDCFlow* Source = source; //--- Prepare if(!cPrepare.WeightsUpdate(Source.cPrepare.AsObject(), tau)) ReturnFalse; //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) if(!cFeatureEncoder[fe].WeightsUpdate(Source.cFeatureEncoder[fe].AsObject(), tau)) ReturnFalse; //--- Context Encoder if(!cConextEncoder.WeightsUpdate(Source.cConextEncoder.AsObject(), tau)) ReturnFalse; //--- Moution Encoder if(!cDiffEncoder.WeightsUpdate(Source.cDiffEncoder.AsObject(), tau)) ReturnFalse; if(!cCorrelation.WeightsUpdate(Source.cCorrelation.AsObject(), tau)) ReturnFalse; if(!cCorrelationUpsampling.WeightsUpdate(Source.cCorrelationUpsampling.AsObject(), tau)) ReturnFalse; //--- Flow if(!cFlow.WeightsUpdate(Source.cFlow.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEDCFlow::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- Prepare if(!cPrepare.ClearStates()) ReturnFalse; //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) if(!cFeatureEncoder[fe].ClearStates()) ReturnFalse; //--- Context Encoder if(!cConextEncoder.ClearStates()) ReturnFalse; //--- Moution Encoder if(!cDiffEncoder.Clear()) ReturnFalse; if(!cCorrelation.Clear()) ReturnFalse; if(!cCorrelationUpsampling.ClearStates()) ReturnFalse; //--- Flow if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEDCFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); //--- Prepare cPrepare.SetOpenCL(OpenCL); //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) cFeatureEncoder[fe].SetOpenCL(OpenCL); //--- Context Encoder cConextEncoder.SetOpenCL(OpenCL); //--- Moution Encoder cDiffEncoder.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); cCorrelationUpsampling.SetOpenCL(OpenCL); //--- Flow cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEDCFlow::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- Prepare cPrepare.TrainMode(bTrain); //--- Feature Encoder for(uint fe = 0; fe < cFeatureEncoder.Size(); fe++) cFeatureEncoder[fe].TrainMode(bTrain); //--- Context Encoder cConextEncoder.TrainMode(bTrain); //--- Moution Encoder cDiffEncoder.TrainMode(bTrain); cCorrelation.TrainMode(bTrain); cCorrelationUpsampling.TrainMode(bTrain); //--- Flow cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ CBufferFloat* CNeuronEDCFlow::getWeights(void) { CBufferFloat* result = NULL; for(int layerNum = 0; (layerNum < cPrepare.Total() && !result && !IsStopped()); layerNum++) { CNeuronBaseOCL *temp = cPrepare[layerNum]; CNeuronConvOCL *conv = NULL; CNeuronConcatenate *conc = NULL; CNeuronBatchNormOCL *batch = NULL; CNeuronLSTMOCL *lstm = NULL; if(!temp) continue; switch(temp.Type()) { case defNeuronConvOCL: conv = temp; result = conv.GetWeightsConv(); break; case defNeuronConcatenate: conc = temp; result = conc.getConcatWeights(); break; case defNeuronBatchNormOCL: case defNeuronBatchNormWithNoise: batch = temp; result = batch.getBatchOptions(); break; case defNeuronLSTMOCL: lstm = temp; result = lstm.getLSTMWeights(); break; default: result = temp.getWeights(); break; } } //--- return result; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronCreateICEFlow : public CNeuronCreateFlow { protected: CNeuronBaseOCL cConcat; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronCreateICEFlow(void) {}; ~CNeuronCreateICEFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronCreateICEFlow; } //--- methods for working with files virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikePatchStak::Init(numOutputs, myIndex, open_cl, stack_size, 2 * dimension, patch_dimension, variables, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cState.Init(0, index, OpenCL, dimension * variables, optimization, iBatch)) ReturnFalse; cState.SetActivationFunction(None); index++; if(!cConcat.Init(0, index, OpenCL, 2 * dimension * variables, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- if(!Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!CalcFlow(NeuronOCL)) ReturnFalse; uint dimension = cBlock.GetWindow() / 2; uint units = cBlock.GetUnits() * cBlock.GetVariables(); if(!Concat(NeuronOCL.getOutput(), cState.getOutput(), cConcat.getOutput(), dimension, dimension, units)) ReturnFalse; if(!Normilize(cConcat.getOutput(), dimension)) ReturnFalse; //--- return CNeuronSpikePatchStak::feedForward(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikePatchStak::calcInputGradients(cConcat.AsObject())) ReturnFalse; uint dimension = cBlock.GetWindow() / 2; uint units = cBlock.GetUnits() * cBlock.GetVariables(); if(!DeConcat(NeuronOCL.getGradient(), cState.getGradient(), cConcat.getGradient(), dimension, dimension, units)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cState.getGradient(), NeuronOCL.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return CNeuronSpikePatchStak::updateInputWeights(cConcat.AsObject()); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::Load(const int file_handle) { if(!CNeuronCreateFlow::Load(file_handle)) ReturnFalse; if(!cConcat.Init(0, 0, OpenCL, 2 * cState.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronCreateICEFlow::Clear(void) { if(!CNeuronCreateFlow::Clear()) ReturnFalse; if(!cConcat.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronCreateICEFlow::SetOpenCL(COpenCLMy *obj) { CNeuronCreateFlow::SetOpenCL(obj); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeMFE : public CNeuronSpikeConvBlock { protected: CLayer cEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeMFE(void) {}; ~CNeuronSpikeMFE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeMFE; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint &windows[], uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch) { uint wsize = windows.Size(); if(wsize < 2 || wsize % 2 != 0) ReturnFalse; uint modals = wsize / 2; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, 2 * window_out, 2 * window_out, window_out, modals * units, 1, optimization_type, batch)) ReturnFalse; //--- uint index = 0; cEncoder.Clear(); cEncoder.SetOpenCL(OpenCL); CNeuronMultiWindowsConvOCL* mw_conv = NULL; CNeuronBatchNormOCL* normal = NULL; CNeuronSpikeActivation* activat = NULL; //--- mw_conv = new CNeuronMultiWindowsConvOCL(); if(!mw_conv || !mw_conv.Init(0, index, OpenCL, windows, 2 * window_out, units, 1, optimization, iBatch) || !cEncoder.Add(mw_conv)) { DeleteObj(mw_conv) ReturnFalse; } mw_conv.SetActivationFunction(None); index++; normal = new CNeuronBatchNormOCL(); if(!normal || !normal.Init(0, index, OpenCL, mw_conv.Neurons(), iBatch, optimization) || !cEncoder.Add(normal)) { DeleteObj(normal) ReturnFalse; } index++; activat = new CNeuronSpikeActivation(); if(!activat || !activat.Init(0, index, OpenCL, normal.Neurons(), optimization, iBatch) || !cEncoder.Add(activat)) { DeleteObj(activat) ReturnFalse; } index++; //--- uint temp[]; if(ArrayResize(temp, wsize) < int(wsize)) ReturnFalse; ArrayFill(temp, 0, wsize, 2 * window_out); mw_conv = new CNeuronMultiWindowsConvOCL(); if(!mw_conv || !mw_conv.Init(0, index, OpenCL, temp, window_out, units, 1, optimization, iBatch) || !cEncoder.Add(mw_conv)) { DeleteObj(mw_conv) ReturnFalse; } mw_conv.SetActivationFunction(None); index++; normal = new CNeuronBatchNormOCL(); if(!normal || !normal.Init(0, index, OpenCL, mw_conv.Neurons(), iBatch, optimization) || !cEncoder.Add(normal)) { DeleteObj(normal) ReturnFalse; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cEncoder[-1])) ReturnFalse; CNeuronBaseOCL* next = cEncoder[-1]; CNeuronBaseOCL* curr = NULL; //--- for(int i = cEncoder.Total() - 2; i >= 0; i--) { curr = cEncoder[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cEncoder.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMFE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikeMFE* Source = source; if(!cEncoder.WeightsUpdate(Source.cEncoder.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMFE::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cEncoder.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMFE::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cEncoder.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeMHCrossAttention : public CNeuronMSRes { protected: uint iHeads; //--- CNeuronSpikeConvBlock cQ_Embedding; CNeuronSpikeConvBlock cKV_Embedding; int ibScoreIndex; CNeuronBaseOCL cMHAttentionOut; CNeuronSpikeConvBlock cW0; CNeuronBaseOCL cAttentionOut; CNeuronBaseOCL cContext; //--- virtual bool Attention(void); virtual bool AttentionGradients(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; public: CNeuronSpikeMHCrossAttention(void) {}; ~CNeuronSpikeMHCrossAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronSpikeMHCrossAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint units_count, uint window_k, uint units_k, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, units_count, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window, iHeads * window_key, units_count, 1, optimization, iBatch)) ReturnFalse; index++; if(!cKV_Embedding.Init(0, index, OpenCL, window_k, window_k, 2 * iHeads * window_key, units_k, 1, optimization, iBatch)) ReturnFalse; index++; if(!cContext.Init(0, index, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetUnits()*cKV_Embedding.GetVariables(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); index++; ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cQ_Embedding.GetUnits() * cKV_Embedding.GetUnits() * iHeads, CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.GetFilters()*cQ_Embedding.GetUnits()*cQ_Embedding.GetVariables(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, cQ_Embedding.GetFilters(), cQ_Embedding.GetFilters(), cQ_Embedding.GetWindow(), cQ_Embedding.GetUnits(), cQ_Embedding.GetVariables(), optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(cContext.getOutput() != Context) if(!cContext.SetOutput(Context)) ReturnFalse; //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(cContext.AsObject())) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::Attention(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {cQ_Embedding.GetUnits(), cKV_Embedding.GetUnits(), iHeads}; uint local_work_size[3] = {1, global_work_size[1], 1}; ResetLastError(); uint kernel = def_k_MH2AttentionOut; setBuffer(kernel, def_k_mh2ao_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mh2ao_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mh2ao_score, ibScoreIndex) setBuffer(kernel, def_k_mh2ao_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_mh2ao_dimension, (int)(cQ_Embedding.GetFilters() / iHeads)) setArgument(kernel, def_k_mh2ao_heads_kv, (int)iHeads) setArgument(kernel, def_k_mh2ao_mask, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::AttentionGradients(void) { if(!OpenCL) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[3] = {cQ_Embedding.GetUnits(), cQ_Embedding.GetFilters() / iHeads, iHeads}; ResetLastError(); uint kernel = def_k_MH2AttentionInsideGradients; setBuffer(kernel, def_k_mh2aig_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mh2aig_qg, cQ_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mh2aig_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mh2aig_kvg, cKV_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mh2aig_score, ibScoreIndex) setBuffer(kernel, def_k_mh2aig_outg, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_mh2aig_kunits, cKV_Embedding.GetUnits()) setArgument(kernel, def_k_mh2aig_heads_kv, (int)iHeads) kernelExecute(kernel, global_work_offset, global_work_size) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer) ReturnFalse; if(cContext.getGradient() != SecondGradient) if(!cContext.SetGradient(SecondGradient)) ReturnFalse; cContext.SetActivationFunction(SecondActivation); //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; Deactivation(cQ_Embedding) Deactivation(cKV_Embedding) //--- if(!cContext.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; Deactivation(cContext) //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getPrevOutput(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), cAttentionOut.getPrevOutput(), prevLayer.getGradient(), cQ_Embedding.GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iHeads), INT_VALUE) < INT_VALUE) ReturnFalse; if(!cQ_Embedding.Save(file_handle)) ReturnFalse; if(!cKV_Embedding.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(!LoadInsideLayer(file_handle, cQ_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- uint index = 2; if(!cContext.Init(0, index, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetUnits()*cKV_Embedding.GetVariables(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); index++; ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cQ_Embedding.GetUnits() * cKV_Embedding.GetUnits() * iHeads, CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.GetFilters()*cQ_Embedding.GetUnits()*cQ_Embedding.GetVariables(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index += 2; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeMHCrossAttention* Source = source; if(!cQ_Embedding.WeightsUpdate(Source.cQ_Embedding.AsObject(), tau)) ReturnFalse; if(!cKV_Embedding.WeightsUpdate(Source.cKV_Embedding.AsObject(), tau)) ReturnFalse; if(!cW0.WeightsUpdate(Source.cW0.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMHCrossAttention::Clear(void) { if(!CNeuronMSRes::Clear()) ReturnFalse; //--- if(!cQ_Embedding.Clear()) ReturnFalse; if(!cKV_Embedding.Clear()) ReturnFalse; if(!cW0.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMHCrossAttention::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibScoreIndex); CNeuronMSRes::SetOpenCL(obj); cQ_Embedding.SetOpenCL(OpenCL); cKV_Embedding.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cQ_Embedding.GetUnits() * cKV_Embedding.GetUnits() * iHeads, CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMHCrossAttention::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); cQ_Embedding.TrainMode(bTrain); cKV_Embedding.TrainMode(bTrain); cW0.TrainMode(bTrain); cContext.TrainMode(bTrain); cMHAttentionOut.TrainMode(bTrain); cAttentionOut.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeMixFusion : public CNeuronSpikeConvBlock { protected: CLayer cMain; CNeuronConvOCL cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeMixFusion(void) {}; ~CNeuronSpikeMixFusion(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint dimension_out, uint units_count, uint depth, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeMixFusion; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cResidual.GetWindow(); } //--- virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint dimension_out, uint units_count, uint depth, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, dimension_out, dimension_out, dimension_out, units_count, 1, optimization_type, batch)) ReturnFalse; //--- Residual uint index = 0; if(!cResidual.Init(0, index, OpenCL, dimension_in, dimension_in, dimension_out, units_count, 1, optimization, iBatch)) ReturnFalse; cResidual.SetActivationFunction(None); //--- Main cMain.Clear(); cMain.SetOpenCL(OpenCL); CNeuronSpikeConvBlock* conv = NULL; CNeuronSpikeDepthWiseConv* dw_conv = NULL; index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_in, dimension_in, dimension_out, units_count, 1, optimization, iBatch) || !cMain.Add(conv)) DeleteObjAndFalse(conv); index++; dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, dimension_out, dimension_out, depth, 1, units_count, 1, optimization, iBatch) || !cMain.Add(dw_conv)) DeleteObjAndFalse(dw_conv); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- if(!cResidual.FeedForward(prev)) ReturnFalse; for(int i = 0; i < cMain.Total(); i++) { curr = cMain[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; if(!SumAndNormilize(Output, cResidual.getOutput(), Output, GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* next = cMain[-1]; CNeuronBaseOCL* curr = NULL; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(next)) ReturnFalse; if(!DeActivation(cResidual.getOutput(), cResidual.getGradient(), Gradient, cResidual.Activation())) ReturnFalse; //--- for(int i = cMain.Total() - 2; i >= 0; i--) { curr = cMain[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cResidual.AsObject()) || !SumAndNormilize(NeuronOCL.getGradient(), temp, temp, GetWindow(), false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- if(!cResidual.UpdateInputWeights(prev)) ReturnFalse; for(int i = 0; i < cMain.Total(); i++) { curr = cMain[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeMixFusion* Source = source; if(!cResidual.WeightsUpdate(Source.cResidual.AsObject(), tau)) ReturnFalse; if(!cMain.WeightsUpdate(Source.cMain.AsObject(), tau)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(!cResidual.Save(file_handle)) ReturnFalse; if(!cMain.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; if(!cMain.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeMixFusion::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- if(!cResidual.Clear()) ReturnFalse; if(!cMain.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMixFusion::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); //--- cResidual.SetOpenCL(OpenCL); cMain.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeMixFusion::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- cResidual.TrainMode(bTrain); cMain.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTFlow : public CNeuronSpikeConvBlock { protected: uint iMaxDisplacement; int ibShifts; //--- CNeuronBatchNormOCL cNorm; CNeuronCreateICEFlow cICE; CNeuronSpikePatchStak cEvents; CLayer cEncoder; CNeuronBaseOCL cICEEmbedding; CNeuronBaseOCL cEventsEmbedding; CNeuronBaseOCL cContext; CNeuronBaseOCL cICECorrelation; CNeuronBaseOCL cEventsCorrelation; CNeuronBaseOCL cFeatureVsCorrelation; CNeuronSpikeMFE cMoutionEncoder; CNeuronTransposeRCDOCL cTranspose; CNeuronSpikeMHCrossAttention cCrossModalAggregation; CNeuronSpikeMixFusion cMixFusion; CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTFlow(void) {}; ~CNeuronSTFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint units_in, uint bottleneck, uint dimension_out, uint units_out, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSTFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_in, uint units_in, uint bottleneck, uint dimension_out, uint units_out, uint max_displacement, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck, bottleneck, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Shifts int count = 0; ibShifts = CreateShifts(max_displacement, count); if(ibShifts == INVALID_HANDLE) ReturnFalse; iMaxDisplacement = max_displacement; //--- Create Flow uint index = 0; if(!cNorm.Init(0, index, OpenCL, dimension_in, iBatch, optimization)) ReturnFalse; SetActivationFunction(None); index++; if(!cICE.Init(0, index, OpenCL, units_in, dimension_in, bottleneck, 1, optimization, iBatch)) ReturnFalse; index++; if(!cEvents.Init(0, index, OpenCL, units_in, dimension_in, bottleneck, 1, optimization, iBatch)) ReturnFalse; index++; //--- cEncoder.Clear(); cEncoder.SetOpenCL(OpenCL); cFlow.Clear(); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvWPadOCL* mw_conv = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronTransposeOCL* transp = NULL; CNeuronSpikeConvGRU2D* gru = NULL; //--- Feature Encoder neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 2 * bottleneck * units_in, optimization, iBatch) || !cEncoder.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, 2 * units_in, 1, optimization, iBatch) || !cEncoder.Add(conv)) DeleteObjAndFalse(conv); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, 2 * bottleneck, optimization, iBatch) || !cEncoder.Add(transp)) DeleteObjAndFalse(transp); index++; uint windows_depth[] = { iMaxDisplacement }; for(int i = 0; i < 2; i++) { mw_conv = new CNeuronMultiWindowsConvWPadOCL(); if(!mw_conv || !mw_conv.Init(0, index, OpenCL, windows_depth, 1, 1, units_in, 2 * bottleneck, optimization, iBatch) || !cEncoder.Add(mw_conv)) DeleteObjAndFalse(mw_conv); mw_conv.SetActivationFunction(SoftPlus); index++; } transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, 2 * bottleneck, optimization, iBatch) || !cEncoder.Add(transp)) DeleteObjAndFalse(transp); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, 2 * units_in, 1, optimization, iBatch) || !cEncoder.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, bottleneck, bottleneck, 2 * bottleneck, 2 * units_in, 1, optimization, iBatch) || !cEncoder.Add(conv)) DeleteObjAndFalse(conv); index++; //--- Moution Encoder if(!cICEEmbedding.Init(0, index, OpenCL, bottleneck * units_in, optimization, iBatch)) ReturnFalse; cICEEmbedding.SetActivationFunction(None); index++; if(!cEventsEmbedding.Init(0, index, OpenCL, bottleneck * units_in, optimization, iBatch)) ReturnFalse; cEventsEmbedding.SetActivationFunction(None); index++; if(!cContext.Init(0, index, OpenCL, 2 * bottleneck * units_in, optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); index++; if(!cICECorrelation.Init(0, index, OpenCL, count * units_in, optimization, iBatch)) ReturnFalse; cICECorrelation.SetActivationFunction(None); index++; if(!cEventsCorrelation.Init(0, index, OpenCL, count * units_in, optimization, iBatch)) ReturnFalse; cEventsCorrelation.SetActivationFunction(None); index++; if(!cFeatureVsCorrelation.Init(0, index, OpenCL, 2 * (bottleneck + count)*units_in, optimization, iBatch)) ReturnFalse; cFeatureVsCorrelation.SetActivationFunction(None); index++; uint windows[] = { bottleneck, count, bottleneck, count }; if(!cMoutionEncoder.Init(0, index, OpenCL, units_in, windows, bottleneck, optimization, iBatch)) ReturnFalse; index++; //--- Aggregation if(!cTranspose.Init(0, index, OpenCL, units_in, 2, bottleneck, optimization, iBatch)) ReturnFalse; index++; if(!cCrossModalAggregation.Init(0, index, OpenCL, bottleneck, (bottleneck + 3) / 4, 4, 2 * units_in, bottleneck, units_in, optimization, iBatch)) ReturnFalse; index++; if(!cMixFusion.Init(0, index, OpenCL, 2 * bottleneck, bottleneck, units_in, iMaxDisplacement, optimization, iBatch)) ReturnFalse; index++; //--- Flow neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 3 * bottleneck * units_in, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, 3 * bottleneck, 3 * bottleneck, bottleneck, units_in, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; gru = new CNeuronSpikeConvGRU2D(); if(!gru || !gru.Init(0, index, OpenCL, units_in, bottleneck, bottleneck, optimization, iBatch) || !cFlow.Add(gru)) DeleteObjAndFalse(gru); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, units_in, bottleneck, optimization, iBatch) || !cFlow.Add(transp)) DeleteObjAndFalse(transp); index++; conv = CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, units_in, units_in, units_out, bottleneck, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; transp = new CNeuronTransposeOCL(); if(!transp || !transp.Init(0, index, OpenCL, bottleneck, units_out, optimization, iBatch) || !cFlow.Add(transp)) DeleteObjAndFalse(transp); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- Create Flow if(!cNorm.FeedForward(NeuronOCL)) ReturnFalse; if(!cICE.FeedForward(cNorm.AsObject())) ReturnFalse; if(!cEvents.FeedForward(cICE.GetState())) ReturnFalse; //--- CNeuronBaseOCL* prev = cEncoder[0]; CNeuronBaseOCL* curr = NULL; uint bottleneck = cICE.GetDimension(); uint units = cICE.GetStackSize(); uint correl = cICECorrelation.Neurons() / units; //--- Feature Encoder if(!prev || !Concat(cICE.getOutput(), cEvents.getOutput(), prev.getOutput(), bottleneck, bottleneck, units)) ReturnFalse; for(int i = 1; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Moution Encoder if(!DeConcat(cICEEmbedding.getOutput(), cContext.getOutput(), cEventsEmbedding.getOutput(), prev.getOutput(), bottleneck, 2 * bottleneck, bottleneck, units)) ReturnFalse; if(!DilatedCorrelation(cICEEmbedding.AsObject(), cICECorrelation.AsObject(), ibShifts, bottleneck, units)) ReturnFalse; if(!DilatedCorrelation(cEventsEmbedding.AsObject(), cEventsCorrelation.AsObject(), ibShifts, bottleneck, units)) ReturnFalse; if(!Concat(cICEEmbedding.getOutput(), cICECorrelation.getOutput(), cEventsEmbedding.getOutput(), cEventsCorrelation.getOutput(), cFeatureVsCorrelation.getOutput(), bottleneck, correl, bottleneck, correl, units)) ReturnFalse; if(!cMoutionEncoder.FeedForward(cFeatureVsCorrelation.AsObject())) ReturnFalse; //--- Aggregation if(!cTranspose.FeedForward(cMoutionEncoder.AsObject())) ReturnFalse; if(!cCrossModalAggregation.FeedForward(cMoutionEncoder.AsObject(), cTranspose.getOutput())) ReturnFalse; if(!cMixFusion.FeedForward(cContext.AsObject())) ReturnFalse; //--- Flow prev = cFlow[0]; if(!prev || !Concat(cCrossModalAggregation.getOutput(), cMixFusion.getOutput(), prev.getOutput(), 2 * bottleneck, bottleneck, units)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeConvBlock::calcInputGradients(cFlow[-1])) ReturnFalse; //--- Flow CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL*curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Aggregation uint bottleneck = cICE.GetDimension(); uint units = cICE.GetStackSize(); uint correl = cICECorrelation.Neurons() / units; //--- if(!DeConcat(cCrossModalAggregation.getGradient(), cMixFusion.getGradient(), next.getGradient(), 2 * bottleneck, bottleneck, units)) ReturnFalse; Deactivation(cCrossModalAggregation); Deactivation(cMixFusion); if(!cContext.CalcHiddenGradients(cMixFusion.AsObject())) ReturnFalse; //--- Moution Encoder if(!cMoutionEncoder.CalcHiddenGradients(cCrossModalAggregation.AsObject(), cTranspose.getOutput(), cTranspose.getGradient(), (ENUM_ACTIVATION)cTranspose.Activation())) ReturnFalse; CBufferFloat* temp = cMoutionEncoder.getGradient(); if(!cMoutionEncoder.SetGradient(cMoutionEncoder.getPrevOutput(), false) || !cMoutionEncoder.CalcHiddenGradients(cTranspose.AsObject()) || !SumAndNormilize(cMoutionEncoder.getGradient(), temp, temp, bottleneck, false, 0, 0, 0, 1) || !cMoutionEncoder.SetGradient(temp, false)) ReturnFalse; if(!cFeatureVsCorrelation.CalcHiddenGradients(cMoutionEncoder.AsObject())) ReturnFalse; if(!DeConcat(cICEEmbedding.getPrevOutput(), cICECorrelation.getGradient(), cEventsEmbedding.getPrevOutput(), cEventsCorrelation.getGradient(), cFeatureVsCorrelation.getGradient(), bottleneck, correl, bottleneck, correl, units)) ReturnFalse; if(!DilatedCorrelationGrad(cICEEmbedding.AsObject(), cICECorrelation.AsObject(), ibShifts, bottleneck, units)) ReturnFalse; if(!DilatedCorrelationGrad(cEventsEmbedding.AsObject(), cEventsCorrelation.AsObject(), ibShifts, bottleneck, units)) ReturnFalse; if(!SumAndNormilize(cICEEmbedding.getGradient(), cICEEmbedding.getPrevOutput(), cICEEmbedding.getGradient(), bottleneck, false, 0, 0, 0, 1) || !SumAndNormilize(cEventsEmbedding.getGradient(), cEventsEmbedding.getPrevOutput(), cEventsEmbedding.getGradient(), bottleneck, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cICEEmbedding); Deactivation(cEventsEmbedding); //--- Feature Encoder next = cEncoder[-1]; if(!next || !Concat(cICEEmbedding.getGradient(), cContext.getGradient(), cEventsEmbedding.getGradient(), next.getGradient(), bottleneck, 2 * bottleneck, bottleneck, units)) ReturnFalse; Deactivation(next); for(int i = cEncoder.Total() - 2; i >= 0; i--) { curr = cEncoder[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Create Flow if(!DeConcat(cICE.getGradient(), cEvents.getGradient(), next.getGradient(), bottleneck, bottleneck, units)) ReturnFalse; Deactivation(cICE); Deactivation(cEvents); curr = cICE.GetState(); if(!curr || !curr.CalcHiddenGradients(cEvents.AsObject())) ReturnFalse; if(!cNorm.CalcHiddenGradients(cICE.AsObject())) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cNorm.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- Create Flow if(!cNorm.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cICE.UpdateInputWeights(cNorm.AsObject())) ReturnFalse; if(!cEvents.UpdateInputWeights(cICE.GetState())) ReturnFalse; //--- CNeuronBaseOCL* prev = cEncoder[0]; CNeuronBaseOCL* curr = NULL; //--- Feature Encoder for(int i = 1; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Moution Encoder if(!cMoutionEncoder.UpdateInputWeights(cFeatureVsCorrelation.AsObject())) ReturnFalse; //--- Aggregation if(!cCrossModalAggregation.UpdateInputWeights(cMoutionEncoder.AsObject(), cTranspose.getOutput())) ReturnFalse; if(!cMixFusion.UpdateInputWeights(cContext.AsObject())) ReturnFalse; //--- Flow for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iMaxDisplacement), INT_VALUE) < INT_VALUE) ReturnFalse; //--- Create Flow if(!cNorm.Save(file_handle)) ReturnFalse; if(!cICE.Save(file_handle)) ReturnFalse; if(!cEvents.Save(file_handle)) ReturnFalse; //--- Feature Encoder if(!cEncoder.Save(file_handle)) ReturnFalse; //--- Moution Encoder if(!cMoutionEncoder.Save(file_handle)) ReturnFalse; //--- Aggregation if(!cTranspose.Save(file_handle)) ReturnFalse; if(!cCrossModalAggregation.Save(file_handle)) ReturnFalse; if(!cMixFusion.Save(file_handle)) ReturnFalse; //--- Flow if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iMaxDisplacement = uint(FileReadInteger(file_handle, INT_VALUE)); //--- Shifts int count = 0; ibShifts = CreateShifts(iMaxDisplacement, count); if(ibShifts == INVALID_HANDLE) ReturnFalse; //--- Create Flow if(!LoadInsideLayer(file_handle, cNorm.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cICE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cEvents.AsObject())) ReturnFalse; //--- Feature Encoder if(!cEncoder.Load(file_handle)) ReturnFalse; //--- Moution Encoder if(!LoadInsideLayer(file_handle, cMoutionEncoder.AsObject())) ReturnFalse; //--- Aggregation if(!LoadInsideLayer(file_handle, cTranspose.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCrossModalAggregation.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cMixFusion.AsObject())) ReturnFalse; //--- Flow if(!cFlow.Load(file_handle)) ReturnFalse; //--- uint index = 3 + cEncoder.Total(); uint bottleneck = cMoutionEncoder.GetFilters(); uint units = cICE.GetStackSize(); if(!cICEEmbedding.Init(0, index, OpenCL, bottleneck * units, optimization, iBatch)) ReturnFalse; cICEEmbedding.SetActivationFunction(None); index++; if(!cEventsEmbedding.Init(0, index, OpenCL, bottleneck * units, optimization, iBatch)) ReturnFalse; cEventsEmbedding.SetActivationFunction(None); index++; if(!cContext.Init(0, index, OpenCL, 2 * bottleneck * units, optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); index++; if(!cICECorrelation.Init(0, index, OpenCL, count * units, optimization, iBatch)) ReturnFalse; cICECorrelation.SetActivationFunction(None); index++; if(!cEventsCorrelation.Init(0, index, OpenCL, count * units, optimization, iBatch)) ReturnFalse; cEventsCorrelation.SetActivationFunction(None); index++; if(!cFeatureVsCorrelation.Init(0, index, OpenCL, 2 * (bottleneck + count)*units, optimization, iBatch)) ReturnFalse; cFeatureVsCorrelation.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSTFlow* Source = source; //--- Create Flow dWeightsUpdate(cNorm, Source, tau); dWeightsUpdate(cICE, Source, tau); dWeightsUpdate(cEvents, Source, tau); //--- Feature Encoder dWeightsUpdate(cEncoder, Source, tau); //--- Moution Encoder dWeightsUpdate(cMoutionEncoder, Source, tau); //--- Aggregation dWeightsUpdate(cCrossModalAggregation, Source, tau); dWeightsUpdate(cMixFusion, Source, tau); //--- Flow dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTFlow::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; //--- Create Flow if(!cNorm.Clear()) ReturnFalse; if(!cICE.Clear()) ReturnFalse; if(!cEvents.Clear()) ReturnFalse; //--- Feature Encoder if(!cEncoder.ClearStates()) ReturnFalse; //--- if(!cMoutionEncoder.Clear()) ReturnFalse; if(!cTranspose.Clear()) ReturnFalse; if(!cCrossModalAggregation.Clear()) ReturnFalse; if(!cMixFusion.Clear()) ReturnFalse; //--- Flow if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTFlow::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL) OpenCL.BufferFree(ibShifts); //--- CNeuronSpikeConvBlock::SetOpenCL(obj); //--- Create Flow cNorm.SetOpenCL(OpenCL); cICE.SetOpenCL(OpenCL); cEvents.SetOpenCL(OpenCL); //--- Feature Encoder cEncoder.SetOpenCL(OpenCL); //--- Moution Encoder cMoutionEncoder.SetOpenCL(OpenCL); //--- Aggregation cTranspose.SetOpenCL(OpenCL); cCrossModalAggregation.SetOpenCL(OpenCL); cMixFusion.SetOpenCL(OpenCL); //--- Flow cFlow.SetOpenCL(OpenCL); //--- cICEEmbedding.SetOpenCL(OpenCL); cEventsEmbedding.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cICECorrelation.SetOpenCL(OpenCL); cEventsCorrelation.SetOpenCL(OpenCL); cFeatureVsCorrelation.SetOpenCL(OpenCL); cFeatureVsCorrelation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTFlow::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); //--- Create Flow cNorm.TrainMode(bTrain); cICE.TrainMode(bTrain); cEvents.TrainMode(bTrain); //--- Feature Encoder cEncoder.TrainMode(bTrain); //--- Moution Encoder cMoutionEncoder.TrainMode(bTrain); //--- Aggregation cTranspose.TrainMode(bTrain); cCrossModalAggregation.TrainMode(bTrain); cMixFusion.TrainMode(bTrain); //--- Flow cFlow.TrainMode(bTrain); //--- cICEEmbedding.TrainMode(bTrain); cEventsEmbedding.TrainMode(bTrain); cContext.TrainMode(bTrain); cICECorrelation.TrainMode(bTrain); cEventsCorrelation.TrainMode(bTrain); cFeatureVsCorrelation.TrainMode(bTrain); cFeatureVsCorrelation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::PerturbedMatrix(CNeuronBaseOCL *inputs, CNeuronBaseOCL *perturb, CNeuronBaseOCL *outputs, float perturb_mult, uint dimension, uint units) { if(!OpenCL || !inputs || !perturb || !outputs || units <= 0 || dimension <= 0) ReturnFalse; //--- uint global_work_offset[2] = {0}; uint global_work_size[] = { dimension, units }; uint kernel = def_k_PerturbedMatrix; setBuffer(kernel, def_k_pm_inputs, inputs.getOutputIndex()) setBuffer(kernel, def_k_pm_perturb, perturb.getOutputIndex()) setBuffer(kernel, def_k_pm_output, outputs.getOutputIndex()) setArgument(kernel, def_k_pm_perturb_mult, perturb_mult) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!outputs.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::PerturbedMatrixGrad(CNeuronBaseOCL *inputs, CNeuronBaseOCL *perturb, CNeuronBaseOCL *outputs, float perturb_mult, uint dimension, uint units) { if(!OpenCL || !inputs || !perturb || !outputs || units <= 0 || dimension <= 0) ReturnFalse; //--- uint global_work_offset[2] = { 0 }; uint global_work_size[] = { dimension, units }; uint kernel = def_k_PerturbedMatrixGrad; setBuffer(kernel, def_k_pm_inputs, inputs.getGradientIndex()) setBuffer(kernel, def_k_pm_perturb, perturb.getGradientIndex()) setBuffer(kernel, def_k_pm_output, outputs.getGradientIndex()) setArgument(kernel, def_k_pm_perturb_mult, perturb_mult) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!inputs.getGradient().BufferRead() || !perturb.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPSSE : public CNeuronAddToStack { protected: CLayer cPrepare; //--- CParams cA; CNeuronSpikeConvBlock cBCDxE; //--- CNeuronBaseOCL cB; CNeuronBaseOCL cC; CNeuronBaseOCL cDx; CNeuronBaseOCL cE; CNeuronBaseOCL cH; //--- CNeuronBaseOCL cAE; CNeuronTransposeVRCOCL cAET; CNeuronBaseOCL cAED; //--- CNeuronBaseOCL cAH; CNeuronBaseOCL cBX; CNeuronBaseOCL cHc; CNeuronBaseOCL cCHc; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPSSE(void) {}; ~CNeuronPSSE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronPSSE; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAddToStack::Init(numOutputs, myIndex, open_cl, stack_size, patch_dimension, variables, optimization_type, batch)) ReturnFalse; //--- cPrepare.Clear(); cPrepare.SetOpenCL(OpenCL); CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeQKAttention* att = NULL; CNeuronSpikeConvBlock* conv = NULL; uint index = 0; uint units = MathMax(dimension - 2, 1); //--- norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, dimension * variables, iBatch, optimization) || !cPrepare.Add(norm)) ReturnFalse; norm.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, MathMin(dimension, 3), 1, patch_dimension, units, variables, optimization, iBatch) || !cPrepare.Add(conv)) DeleteObjAndFalse(conv); index++; att = new CNeuronSpikeQKAttention(); if(!att || !att.Init(0, index, OpenCL, units * variables, patch_dimension, 4, (patch_dimension + 3) / 4, optimization, iBatch) || !cPrepare.Add(att)) DeleteObjAndFalse(att); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, patch_dimension * units, patch_dimension * units, patch_dimension, variables, 1, optimization, iBatch) || !cPrepare.Add(conv)) DeleteObjAndFalse(conv); index++; //--- if(!cA.Init(0, index, OpenCL, patch_dimension * (patch_dimension + 3) / 4 * variables, optimization, iBatch)) ReturnFalse; cA.SetActivationFunction(None); CBufferFloat* temp = cA.getWeightsParams(); if(!temp || !temp.Fill(0)) ReturnFalse; index++; if(!cBCDxE.Init(0, index, OpenCL, patch_dimension, patch_dimension, patch_dimension * (2 * patch_dimension + 1 + (patch_dimension + 3) / 4), variables, 1, optimization, iBatch)) ReturnFalse; index++; if(!cB.Init(0, index, OpenCL, patch_dimension * patch_dimension * variables, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, patch_dimension * patch_dimension * variables, optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cDx.Init(0, index, OpenCL, patch_dimension * variables, optimization, iBatch)) ReturnFalse; cDx.SetActivationFunction(None); index++; if(!cE.Init(0, index, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; cE.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, patch_dimension * variables, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); if(!cH.Clear()) ReturnFalse; index++; //--- if(!cAE.Init(0, index, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; cAE.SetActivationFunction(None); index++; if(!cAET.Init(0, index, OpenCL, variables, patch_dimension, (patch_dimension + 3) / 4, optimization, iBatch)) ReturnFalse; index++; if(!cAED.Init(0, index, OpenCL, patch_dimension * patch_dimension * variables, optimization, iBatch)) ReturnFalse; cAED.SetActivationFunction(None); index++; //--- if(!cAH.Init(0, index, OpenCL, patch_dimension * variables, optimization, iBatch)) ReturnFalse; cAH.SetActivationFunction(None); if(!cBX.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cBX.SetActivationFunction(None); index++; if(!cHc.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cHc.SetActivationFunction(None); index++; if(!cCHc.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cCHc.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) { if(!cA.FeedForward()) ReturnFalse; } //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!cBCDxE.FeedForward(prev)) ReturnFalse; if(!DeConcat(cB.getOutput(), cC.getOutput(), cDx.getOutput(), cE.getOutput(), cBCDxE.getOutput(), cB.Neurons() / iVariables, cC.Neurons() / iVariables, cDx.Neurons() / iVariables, cE.Neurons() / iVariables, iVariables)) ReturnFalse; if(!PerturbedMatrix(cA.AsObject(), cE.AsObject(), cAE.AsObject(), 0.01f, cE.Neurons() / iVariables, iVariables)) ReturnFalse; uint AE_dimension = cAE.Neurons() / (iDimension * iVariables); if(!Normilize(cAE.getOutput(), AE_dimension)) ReturnFalse; if(!cAET.FeedForward(cAE.AsObject())) ReturnFalse; if(!MatMul(cAE.getOutput(), cAET.getOutput(), cAED.getOutput(), iDimension, AE_dimension, iDimension, iVariables, true)) ReturnFalse; //--- if(!cH.SwapOutputs()) ReturnFalse; if(!MatMul(cAED.getOutput(), cH.getPrevOutput(), cAH.getOutput(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; if(!MatMul(cB.getOutput(), prev.getOutput(), cBX.getOutput(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; if(!SumAndNormilize(cAH.getOutput(), cBX.getOutput(), cHc.getOutput(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; if(!MatMul(cC.getOutput(), cHc.getOutput(), cCHc.getOutput(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; if(!SumAndNormilize(cCHc.getOutput(), cDx.getOutput(), cH.getOutput(), iDimension, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronAddToStack::feedForward(cH.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronAddToStack::calcInputGradients(cH.AsObject())) ReturnFalse; //--- if(!DeActivation(cCHc.getOutput(), cCHc.getGradient(), cH.getGradient(), cCHc.Activation())) ReturnFalse; if(!DeActivation(cDx.getOutput(), cDx.getGradient(), cH.getGradient(), cDx.Activation())) ReturnFalse; if(!MatMulGrad(cC.getOutput(), cC.getGradient(), cHc.getOutput(), cHc.getGradient(), cCHc.getGradient(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; Deactivation(cC); Deactivation(cHc); //--- if(!DeActivation(cAH.getOutput(), cAH.getGradient(), cHc.getGradient(), cAH.Activation())) ReturnFalse; if(!DeActivation(cBX.getOutput(), cBX.getGradient(), cHc.getGradient(), cBX.Activation())) ReturnFalse; //--- CNeuronBaseOCL* next = cPrepare[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !MatMulGrad(cB.getOutput(), cB.getGradient(), next.getOutput(), cBX.getPrevOutput(), cBX.getGradient(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; if(!MatMulGrad(cAED.getOutput(), cAED.getGradient(), cH.getPrevOutput(), cAH.getPrevOutput(), cAH.getGradient(), iDimension, iDimension, 1, iVariables, true)) ReturnFalse; uint AE_dimension = cAE.Neurons() / (iDimension * iVariables); if(!MatMulGrad(cAE.getOutput(), cAE.getPrevOutput(), cAET.getOutput(), cAET.getGradient(), cAED.getGradient(), iDimension, AE_dimension, iDimension, iVariables, true)) ReturnFalse; if(!cAE.CalcHiddenGradients(cAET.AsObject()) || !SumAndNormilize(cAE.getGradient(), cAE.getPrevOutput(), cAE.getGradient(), AE_dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!PerturbedMatrixGrad(cA.AsObject(), cE.AsObject(), cAE.AsObject(), 0.01f, cE.Neurons() / iVariables, iVariables)) ReturnFalse; Deactivation(cA); Deactivation(cE); if(!Concat(cB.getGradient(), cC.getGradient(), cDx.getGradient(), cE.getGradient(), cBCDxE.getGradient(), cB.Neurons() / iVariables, cC.Neurons() / iVariables, cDx.Neurons() / iVariables, cE.Neurons() / iVariables, iVariables)) ReturnFalse; if(!next.CalcHiddenGradients(cBCDxE.AsObject())) ReturnFalse; if(!next.Activation() != None) if(!DeActivation(next.getOutput(), next.getPrevOutput(), next.getPrevOutput(), next.Activation())) ReturnFalse; if(!SumAndNormilize(next.getGradient(), next.getPrevOutput(), next.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cA.UpdateInputWeights()) ReturnFalse; //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cBCDxE.UpdateInputWeights(prev)) ReturnFalse; if(!CNeuronAddToStack::updateInputWeights(cH.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronAddToStack::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronPSSE* Source = source; dWeightsUpdate(cA, Source, tau); dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cBCDxE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::Save(const int file_handle) { if(!CNeuronAddToStack::Save(file_handle)) ReturnFalse; if(!cA.Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cBCDxE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::Load(const int file_handle) { if(!CNeuronAddToStack::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cA.AsObject())) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cBCDxE.AsObject())) ReturnFalse; //--- uint index = cPrepare.Total() + 2; if(!cB.Init(0, index, OpenCL, iDimension * iDimension * iVariables, optimization, iBatch)) ReturnFalse; cB.SetActivationFunction(None); index++; if(!cC.Init(0, index, OpenCL, cB.Neurons(), optimization, iBatch)) ReturnFalse; cC.SetActivationFunction(None); index++; if(!cDx.Init(0, index, OpenCL, iDimension * iVariables, optimization, iBatch)) ReturnFalse; cDx.SetActivationFunction(None); index++; if(!cE.Init(0, index, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; cE.SetActivationFunction(None); index++; if(!cH.Init(0, index, OpenCL, iDimension * iVariables, optimization, iBatch)) ReturnFalse; cH.SetActivationFunction(None); if(!cH.Clear()) ReturnFalse; index++; //--- if(!cAE.Init(0, index, OpenCL, cA.Neurons(), optimization, iBatch)) ReturnFalse; cAE.SetActivationFunction(None); index++; uint AE_dimension = cAE.Neurons() / (iDimension * iVariables); if(!cAET.Init(0, index, OpenCL, iVariables, iDimension, AE_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cAED.Init(0, index, OpenCL, cB.Neurons(), optimization, iBatch)) ReturnFalse; cAED.SetActivationFunction(None); index++; //--- if(!cAH.Init(0, index, OpenCL, iDimension * iVariables, optimization, iBatch)) ReturnFalse; cAH.SetActivationFunction(None); if(!cBX.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cBX.SetActivationFunction(None); index++; if(!cHc.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cHc.SetActivationFunction(None); index++; if(!cCHc.Init(0, index, OpenCL, cAH.Neurons(), optimization, iBatch)) ReturnFalse; cCHc.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSE::Clear(void) { if(!CNeuronAddToStack::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; if(!cBCDxE.Clear()) ReturnFalse; if(!cH.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSSE::SetOpenCL(COpenCLMy *obj) { CNeuronAddToStack::SetOpenCL(obj); //--- cA.SetOpenCL(OpenCL); cPrepare.SetOpenCL(OpenCL); cBCDxE.SetOpenCL(OpenCL); cB.SetOpenCL(OpenCL); cC.SetOpenCL(OpenCL); cDx.SetOpenCL(OpenCL); cE.SetOpenCL(OpenCL); cH.SetOpenCL(OpenCL); cAE.SetOpenCL(OpenCL); cAET.SetOpenCL(OpenCL); cAED.SetOpenCL(OpenCL); cAH.SetOpenCL(OpenCL); cBX.SetOpenCL(OpenCL); cHc.SetOpenCL(OpenCL); cCHc.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSSE::TrainMode(bool flag) { CNeuronAddToStack::TrainMode(flag); //--- cA.TrainMode(bTrain); cPrepare.TrainMode(bTrain); cBCDxE.TrainMode(bTrain); cB.TrainMode(bTrain); cC.TrainMode(bTrain); cDx.TrainMode(bTrain); cE.TrainMode(bTrain); cH.TrainMode(bTrain); cAE.TrainMode(bTrain); cAET.TrainMode(bTrain); cAED.TrainMode(bTrain); cAH.TrainMode(bTrain); cBX.TrainMode(bTrain); cHc.TrainMode(bTrain); cCHc.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeDepthWiseResidual : public CNeuronSpikeDepthWiseConv { protected: CNeuronSpikeConvBlock cResidual; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeDepthWiseResidual(void) {}; ~CNeuronSpikeDepthWiseResidual(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint depth_window, uint depth_step, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeDepthWiseResidual; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint depth_window, uint depth_step, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeDepthWiseConv::Init(numOutputs, myIndex, open_cl, chanels_in, chanels_out, depth_window, depth_step, units, variables, optimization_type, batch)) ReturnFalse; if(!cResidual.Init(0, 0, OpenCL, chanels_in * depth_window, chanels_in * depth_step, chanels_out, units, variables, optimization, iBatch)) ReturnFalse; if(!cResidual.SetGradient(Gradient, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!CNeuronSpikeDepthWiseConv::feedForward(NeuronOCL)) ReturnFalse; if(!cResidual.FeedForward(NeuronOCL)) ReturnFalse; if(!SumAndNormilize(Output, cResidual.getOutput(), Output, GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeDepthWiseConv::calcInputGradients(NeuronOCL)) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cResidual.AsObject())) ReturnFalse; if(!SumAndNormilize(temp, NeuronOCL.getGradient(), temp, GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!CNeuronSpikeDepthWiseConv::updateInputWeights(NeuronOCL)) ReturnFalse; if(!cResidual.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeDepthWiseConv::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeDepthWiseResidual* Source = source; dWeightsUpdate(cResidual, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::Save(const int file_handle) { if(!CNeuronSpikeDepthWiseConv::Save(file_handle)) ReturnFalse; if(!cResidual.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeDepthWiseResidual::Load(const int file_handle) { if(!CNeuronSpikeDepthWiseConv::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cResidual.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeDepthWiseResidual::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeDepthWiseConv::SetOpenCL(obj); cResidual.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeDepthWiseResidual::TrainMode(bool flag) { CNeuronSpikeDepthWiseConv::TrainMode(flag); cResidual.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSuperKernelBlock : public CNeuronMSRes { protected: CLayer cSKFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeSuperKernelBlock(void) {}; ~CNeuronSpikeSuperKernelBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint &depth_window[], uint &depth_step[], uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSuperKernelBlock; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels_in, uint chanels_out, uint &depth_window[], uint &depth_step[], uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, units, chanels_out, variables, optimization_type, batch)) ReturnFalse; //--- uint layers = depth_window.Size(); if(layers == 0 || layers != depth_step.Size()) ReturnFalse; uint units_dw[]; if(ArrayResize(units_dw, layers) < (int)layers) ReturnFalse; units_dw[layers - 1] = units; for(int i = (int)layers - 2; i >= 0; i--) units_dw[i] = units_dw[i + 1] * depth_step[i + 1]; uint chanels_dw_in = chanels_in; CNeuronSpikeDepthWiseResidual* conv = NULL; cSKFlow.Clear(); cSKFlow.SetOpenCL(OpenCL); for(uint i = 0; i < layers; i++) { conv = new CNeuronSpikeDepthWiseResidual(); if(!conv || !conv.Init(0, i, OpenCL, chanels_dw_in, chanels_out, depth_window[i], depth_step[i], units_dw[i], variables, optimization, iBatch) || !cSKFlow.Add(conv)) DeleteObjAndFalse(conv); chanels_dw_in = chanels_out; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSKFlow.Total(); i++) { curr = cSKFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!CNeuronMSRes::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronMSRes::calcInputGradients(cSKFlow[-1])) ReturnFalse; //--- CNeuronBaseOCL* next = cSKFlow[-1]; CNeuronBaseOCL* curr = NULL; for(int i = cSKFlow.Total() - 2; i >= 0; i--) { curr = cSKFlow[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cSKFlow.Total(); i++) { curr = cSKFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!CNeuronMSRes::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(!cSKFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(!cSKFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSuperKernelBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSuperKernelBlock* Source = source; dWeightsUpdate(cSKFlow, Source, tau) //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSuperKernelBlock::SetOpenCL(COpenCLMy *obj) { CNeuronMSRes::SetOpenCL(obj); cSKFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSuperKernelBlock::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); cSKFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeGMA : public CNeuronBaseOCL { protected: uint iChanels; uint iUnits; uint iVariables; //--- CLayer cQKV; CNeuronBaseOCL cQ; CNeuronBaseOCL cK; CNeuronTransposeVRCOCL cKT; CNeuronBaseOCL cV; CLayer cScore; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeGMA(void) {}; ~CNeuronSpikeGMA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeGMA; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint chanels, uint units, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(chanels < 2) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, chanels * units * variables, optimization_type, batch)) ReturnFalse; activation = None; //--- iChanels = chanels; iUnits = units; iVariables = variables; //--- cQKV.Clear(); cQKV.SetOpenCL(OpenCL); CNeuronSpikeActivation* act = NULL; CNeuronMultiWindowsConvOCL* conv = NULL; CNeuronBatchNormOCL* norm = NULL; uint index = 0; //--- act = new CNeuronSpikeActivation(); if(!act || !act.Init(0, index, OpenCL, 2 * chanels * units * variables, optimization, iBatch) || !cQKV.Add(act)) DeleteObjAndFalse(act); //--- index++; uint windows[] = {chanels, chanels}; conv = new CNeuronMultiWindowsConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, windows, chanels, units, variables, optimization, iBatch) || !cQKV.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(None); //--- index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, conv.Neurons(), iBatch, optimization) || !cQKV.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); //--- index++; if(!cQ.Init(0, index, OpenCL, (chanels + 1) / 2 * units * variables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); cQ.Clear(); index++; if(!cK.Init(0, index, OpenCL, (chanels + 1) / 2 * units * variables, optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); cK.Clear(); index++; if(!cKT.Init(0, index, OpenCL, variables, units, (chanels + 1) / 2, optimization, iBatch)) ReturnFalse; cKT.SetActivationFunction(None); index++; if(!cV.Init(0, index, OpenCL, chanels * units * variables, optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); //--- cScore.Clear(); cScore.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronSoftMaxOCL* softmax = NULL; //--- index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, units * units * variables, optimization, iBatch) || !cScore.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cScore.Add(softmax)) DeleteObjAndFalse(softmax); softmax.SetHeads(units * variables); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cQKV.Total(); i++) { curr = cQKV[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!DeConcat(cV.getOutput(), cQ.getOutput(), cK.getOutput(), prev.getOutput(), iChanels, iChanels / 2, iChanels / 2, iUnits * iVariables)) ReturnFalse; if(!cKT.FeedForward(cK.AsObject())) ReturnFalse; prev = cScore[0]; if(!MatMul(cQ.getOutput(), cKT.getOutput(), prev.getOutput(), iUnits, iChanels / 2, iUnits, iVariables, true)) ReturnFalse; for(int i = 1; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!DeConcat(PrevOutput, Output, NeuronOCL.getOutput(), iChanels, iChanels, iUnits * iVariables)) ReturnFalse; if(!MatMul(prev.getOutput(), cV.getOutput(), Output, iUnits, iUnits, iChanels, iVariables, true)) ReturnFalse; if(!SumAndNormilize(PrevOutput, Output, Output, iChanels, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* next = cScore[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !MatMulGrad(next.getOutput(), next.getGradient(), cV.getOutput(), cV.getGradient(), Gradient, iUnits, iUnits, iChanels, iVariables, true)) ReturnFalse; for(int i = cScore.Total() - 2; i >= 0; i--) { curr = cScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!MatMulGrad(cQ.getOutput(), cQ.getGradient(), cKT.getOutput(), cKT.getGradient(), next.getGradient(), iUnits, iChanels / 2, iUnits, iVariables, true)) ReturnFalse; if(!cK.CalcHiddenGradients(cKT.AsObject())) ReturnFalse; next = cQKV[-1]; if(!next || !Concat(cV.getGradient(), cQ.getGradient(), cK.getGradient(), next.getGradient(), iChanels, iChanels / 2, iChanels / 2, iUnits * iVariables)) ReturnFalse; for(int i = cQKV.Total() - 1; i >= 0; i--) { curr = cQKV[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; if(!PrevOutput.Fill(0) || !Concat(PrevOutput, Gradient, next.getPrevOutput(), iChanels, iChanels, iUnits * iVariables)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), next.getPrevOutput(), next.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(next.getPrevOutput(), NeuronOCL.getGradient(), NeuronOCL.getGradient(), iChanels, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cQKV.Total(); i++) { curr = cQKV[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } for(int i = 1; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::Save(int const file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cQKV.Save(file_handle)) ReturnFalse; if(!cKT.Save(file_handle)) ReturnFalse; if(!cScore.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::Load(int const file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!cQKV.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cKT.AsObject())) ReturnFalse; if(!cScore.Load(file_handle)) ReturnFalse; //--- iUnits = cKT.GetCount(); iVariables = cKT.GetVariables(); iChanels = Neurons() / (iUnits * iVariables); //--- uint index = cQKV.Total(); if(!cQ.Init(0, index, OpenCL, (iChanels + 1) / 2 * iUnits * iVariables, optimization, iBatch)) ReturnFalse; cQ.SetActivationFunction(None); cQ.Clear(); index++; if(!cK.Init(0, index, OpenCL, (iChanels + 1) / 2 * iUnits * iVariables, optimization, iBatch)) ReturnFalse; cK.SetActivationFunction(None); cK.Clear(); index += 2; if(!cV.Init(0, index, OpenCL, iChanels * iUnits * iVariables, optimization, iBatch)) ReturnFalse; cV.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeGMA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSpikeGMA* Source = source; dWeightsUpdate(cQKV, Source, tau); dWeightsUpdate(cKT, Source, tau); dWeightsUpdate(cScore, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeGMA::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cQKV.SetOpenCL(OpenCL); cKT.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cQ.SetOpenCL(OpenCL); cK.SetOpenCL(OpenCL); cV.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeGMA::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cQKV.TrainMode(bTrain); cKT.TrainMode(bTrain); cScore.TrainMode(bTrain); cQ.TrainMode(bTrain); cK.TrainMode(bTrain); cV.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronETROF : public CNeuronAddToStack { protected: CNeuronPSSE cPSSE; CNeuronBaseOCL cRefFrame; CNeuronTransposeOCL cRefFrameT; CNeuronBaseOCL cForwardBackward; CNeuronBaseOCL cCorrelation; CLayer cMixFusion; CNeuronSpikeSuperKernelBlock cContext; CLayer cMotionEncoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronETROF(void) {}; ~CNeuronETROF(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension_in, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronETROF; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension_in, uint dimension_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronAddToStack::Init(numOutputs, myIndex, open_cl, stack_size, dimension_out, 1, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cPSSE.Init(0, index, OpenCL, 3, dimension_in, 5 * dimension_out, 1, optimization, iBatch)) ReturnFalse; index++; if(!cRefFrame.Init(0, index, OpenCL, 5 * dimension_out, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); index++; if(!cRefFrameT.Init(0, index, OpenCL, 5, dimension_out, optimization, iBatch)) ReturnFalse; index++; if(!cForwardBackward.Init(0, index, OpenCL, 10 * dimension_out, optimization, iBatch)) ReturnFalse; cForwardBackward.SetActivationFunction(None); index++; if(!cCorrelation.Init(0, index, OpenCL, 10 * 5, optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); index++; //--- cMixFusion.Clear(); cMixFusion.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronTransposeRCDOCL* transp = NULL; CNeuronSpikeSuperKernelBlock* sk_block = NULL; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, cForwardBackward.Neurons() + cCorrelation.Neurons(), optimization, iBatch) || !cMixFusion.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; transp = new CNeuronTransposeRCDOCL(); if(!transp || !transp.Init(0, index, OpenCL, 2, 5, neuron.Neurons() / 10, optimization, iBatch) || !cMixFusion.Add(transp)) DeleteObjAndFalse(transp); index++; uint windows[] = {3, transp.GetWindow()}; uint steps[] = {1, 1}; sk_block = new CNeuronSpikeSuperKernelBlock(); if(!sk_block || !sk_block.Init(0, index, OpenCL, transp.GetCount()*transp.GetDimension(), dimension_out, windows, steps, transp.GetWindow(), 1, optimization, iBatch) || !cMixFusion.Add(sk_block)) DeleteObjAndFalse(sk_block); index++; //--- if(!cContext.Init(0, index, OpenCL, cRefFrame.Neurons() / transp.GetWindow(), dimension_out, windows, steps, transp.GetWindow(), 1, optimization, iBatch)) ReturnFalse; //--- cMotionEncoder.Clear(); cMotionEncoder.SetOpenCL(OpenCL); CNeuronSpikeGMA* gma = NULL; //--- index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 2 * dimension_out * transp.GetWindow(), optimization, iBatch) || !cMotionEncoder.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; gma = new CNeuronSpikeGMA(); if(!gma || !gma.Init(0, index, OpenCL, dimension_out, cRefFrameT.GetCount(), 1, optimization, iBatch) || !cMotionEncoder.Add(gma)) DeleteObjAndFalse(gma); index++; windows[0] = cRefFrameT.GetCount(); windows[1] = 2 * windows[0]; steps[0] = 1; steps[1] = windows[0]; sk_block = new CNeuronSpikeSuperKernelBlock(); if(!sk_block || !sk_block.Init(0, index, OpenCL, dimension_out, dimension_out, windows, steps, 1, 1, optimization, iBatch) || !cMotionEncoder.Add(sk_block)) DeleteObjAndFalse(sk_block); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPSSE.FeedForward(NeuronOCL)) ReturnFalse; uint units = cRefFrameT.GetCount(); if(!DeConcat(cRefFrame.getPrevOutput(), cRefFrame.getOutput(), cForwardBackward.getPrevOutput(), cPSSE.getOutput(), units * iDimension, units * iDimension, units * iDimension, 1)) ReturnFalse; if(!Concat(cRefFrame.getPrevOutput(), cForwardBackward.getPrevOutput(), cForwardBackward.getOutput(), units * iDimension, units * iDimension, 1)) ReturnFalse; if(!cRefFrameT.FeedForward(cRefFrame.AsObject())) ReturnFalse; if(!MatMul(cForwardBackward.getOutput(), cRefFrameT.getOutput(), cCorrelation.getOutput(), units, iDimension, units, 2, false)) ReturnFalse; if(!cContext.FeedForward(cRefFrame.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* prev = cMixFusion[0]; CNeuronBaseOCL* curr = NULL; if(!prev || !Concat(cForwardBackward.getOutput(), cCorrelation.getOutput(), prev.getOutput(), iDimension, units, 2 * units)) ReturnFalse; for(int i = 1; i < cMixFusion.Total(); i++) { curr = cMixFusion[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- curr = cMotionEncoder[0]; if(!curr || !Concat(cContext.getOutput(), prev.getOutput(), curr.getOutput(), iDimension, iDimension, units)) ReturnFalse; prev = curr; for(int i = 1; i < cMotionEncoder.Total(); i++) { curr = cMotionEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronAddToStack::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronAddToStack::feedForward(cMotionEncoder[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cMotionEncoder.Total() - 2; i >= 0; i--) { curr = cMotionEncoder[i]; if(!curr || !curr.CalcHiddenGradients(cMotionEncoder[i + 1])) ReturnFalse; } //--- uint units = cRefFrameT.GetCount(); curr = cMixFusion[-1]; if(!curr || !DeConcat(cContext.getGradient(), curr.getGradient(), cMotionEncoder[0].getGradient(), iDimension, iDimension, units)) ReturnFalse; for(int i = cMixFusion.Total() - 2; i >= 0; i--) { curr = cMixFusion[i]; if(!curr || !curr.CalcHiddenGradients(cMixFusion[i + 1])) ReturnFalse; } if(!DeConcat(cForwardBackward.getGradient(), cCorrelation.getGradient(), curr.getGradient(), iDimension, units, 2 * units)) ReturnFalse; if(!MatMulGrad(cForwardBackward.getOutput(), cForwardBackward.getPrevOutput(), cRefFrameT.getOutput(), cRefFrameT.getGradient(), cCorrelation.getGradient(), units, iDimension, units, 2, false)) ReturnFalse; if(!SumAndNormilize(cForwardBackward.getGradient(), cForwardBackward.getPrevOutput(), cForwardBackward.getGradient(), iDimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!cRefFrame.CalcHiddenGradients(cContext.AsObject())) ReturnFalse; CBufferFloat* temp = cRefFrame.getGradient(); if(!cRefFrame.SetGradient(cRefFrame.getPrevOutput(), false) || !cRefFrame.CalcHiddenGradients(cRefFrameT.AsObject()) || !SumAndNormilize(cRefFrame.getGradient(), temp, temp, iDimension, false, 0, 0, 0, 1) || !cRefFrame.SetGradient(temp, false)) ReturnFalse; if(!DeConcat(cRefFrameT.getPrevOutput(), cForwardBackward.getPrevOutput(), cForwardBackward.getGradient(), units * iDimension, units * iDimension, 1)) ReturnFalse; if(!Concat(cRefFrameT.getPrevOutput(), cRefFrame.getGradient(), cForwardBackward.getPrevOutput(), cPSSE.getGradient(), units * iDimension, units * iDimension, units * iDimension, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cPSSE.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPSSE.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cContext.UpdateInputWeights(cRefFrame.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* prev = cMixFusion[0]; CNeuronBaseOCL* curr = NULL; for(int i = 1; i < cMixFusion.Total(); i++) { curr = cMixFusion[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cMotionEncoder[0]; for(int i = 1; i < cMotionEncoder.Total(); i++) { curr = cMotionEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronAddToStack::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronAddToStack::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronETROF* Source = source; dWeightsUpdate(cPSSE, Source, tau); dWeightsUpdate(cContext, Source, tau); dWeightsUpdate(cMixFusion, Source, tau); dWeightsUpdate(cMotionEncoder, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::Save(const int file_handle) { if(!CNeuronAddToStack::Save(file_handle)) ReturnFalse; //--- if(!cPSSE.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cMixFusion.Save(file_handle)) ReturnFalse; if(!cMotionEncoder.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::Load(const int file_handle) { if(!CNeuronAddToStack::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cPSSE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cContext.AsObject())) ReturnFalse; if(!cMixFusion.Load(file_handle)) ReturnFalse; if(!cMotionEncoder.Load(file_handle)) ReturnFalse; //--- uint index = 1; if(!cRefFrame.Init(0, index, OpenCL, cPSSE.Neurons() / 3, optimization, iBatch)) ReturnFalse; cRefFrame.SetActivationFunction(None); index++; if(!cRefFrameT.Init(0, index, OpenCL, cRefFrame.Neurons() / iDimension, iDimension, optimization, iBatch)) ReturnFalse; index++; if(!cForwardBackward.Init(0, index, OpenCL, 2 * cRefFrame.Neurons(), optimization, iBatch)) ReturnFalse; cForwardBackward.SetActivationFunction(None); index++; if(!cCorrelation.Init(0, index, OpenCL, 2 * cRefFrameT.GetCount()*cRefFrameT.GetCount(), optimization, iBatch)) ReturnFalse; cCorrelation.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronETROF::Clear(void) { if(!CNeuronAddToStack::Clear()) ReturnFalse; //--- if(!cPSSE.Clear()) ReturnFalse; if(!cContext.Clear()) ReturnFalse; if(!cMixFusion.ClearStates()) ReturnFalse; if(!cMotionEncoder.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronETROF::SetOpenCL(COpenCLMy *obj) { CNeuronAddToStack::SetOpenCL(obj); //--- cPSSE.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cMixFusion.SetOpenCL(OpenCL); cMotionEncoder.SetOpenCL(OpenCL); cRefFrame.SetOpenCL(OpenCL); cRefFrameT.SetOpenCL(OpenCL); cForwardBackward.SetOpenCL(OpenCL); cCorrelation.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronETROF::TrainMode(bool flag) { CNeuronAddToStack::TrainMode(flag); //--- cPSSE.TrainMode(bTrain); cContext.TrainMode(bTrain); cMixFusion.TrainMode(bTrain); cMotionEncoder.TrainMode(bTrain); cRefFrame.TrainMode(bTrain); cRefFrameT.TrainMode(bTrain); cForwardBackward.TrainMode(bTrain); cCorrelation.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronPSSEFlow : public CNeuronSpikeSuperKernelBlock { protected: CNeuronETROF cETROF; CNeuronSpikeSuperKernelBlock cLocal; CLayer cGlobal; CLayer cUpdateFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronPSSEFlow(void) {}; ~CNeuronPSSEFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint forecast, uint dimension, uint hidden_dimension, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronPSSEFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint forecast, uint dimension, uint hidden_dimension, ENUM_OPTIMIZATION optimization_type, uint batch) { uint windows[] = {3, 5}; uint steps[] = {1, 1}; if(!CNeuronSpikeSuperKernelBlock::Init(numOutputs, myIndex, open_cl, hidden_dimension, dimension, windows, steps, forecast, 1, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cETROF.Init(0, index, OpenCL, forecast, dimension, hidden_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cLocal.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, windows, steps, forecast, 1, optimization, iBatch)) ReturnFalse; index++; //--- cGlobal.Clear(); cGlobal.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronSpikeGMA* gma = NULL; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 2 * hidden_dimension * forecast, optimization, iBatch) || !cGlobal.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; gma = new CNeuronSpikeGMA(); if(!gma || !gma.Init(0, index, OpenCL, hidden_dimension, forecast, 1, optimization, iBatch) || !cGlobal.Add(gma)) DeleteObjAndFalse(gma); index++; //--- cUpdateFlow.Clear(); cUpdateFlow.SetOpenCL(OpenCL); CNeuronSpikeSuperKernelBlock* sk_blok = NULL; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, 4 * hidden_dimension * forecast, optimization, iBatch) || !cUpdateFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; sk_blok = new CNeuronSpikeSuperKernelBlock(); if(!sk_blok || !sk_blok.Init(0, index, OpenCL, 4 * hidden_dimension, hidden_dimension, windows, steps, forecast, 1, optimization, iBatch) || !cUpdateFlow.Add(sk_blok)) DeleteObjAndFalse(sk_blok); if(!sk_blok.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cETROF.FeedForward(NeuronOCL)) ReturnFalse; if(!cLocal.FeedForward(cETROF.AsObject())) ReturnFalse; //--- CNeuronSpikeDepthWiseResidual* resid = cSKFlow[0]; uint hidden_dimension = resid.GetWindow(); uint units = cLocal.Neurons() / hidden_dimension; uint dimension = Neurons() / units; //--- CNeuronBaseOCL* prev = cGlobal[0]; CNeuronBaseOCL* curr = NULL; if(!prev || !Concat(cLocal.getOutput(), cETROF.getOutput(), prev.getOutput(), hidden_dimension, hidden_dimension, units)) ReturnFalse; for(int i = 1; i < cGlobal.Total(); i++) { curr = cGlobal[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- curr = cUpdateFlow[0]; prev = cUpdateFlow[-1]; if(!prev || !curr || !Concat(cLocal.getOutput(), cETROF.getOutput(), cGlobal[-1].getOutput(), prev.getOutput(), curr.getOutput(), hidden_dimension, hidden_dimension, hidden_dimension, hidden_dimension, units)) ReturnFalse; prev = curr; for(int i = 1; i < cUpdateFlow.Total(); i++) { curr = cUpdateFlow[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeSuperKernelBlock::feedForward(prev)) ReturnFalse; if(!SumVecMatrix(NeuronOCL.getOutput(), Output, Output, dimension, 1, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeSuperKernelBlock::calcInputGradients(cUpdateFlow[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cUpdateFlow.Total() - 2; i >= 0; i--) { curr = cUpdateFlow[i]; if(!curr || !curr.CalcHiddenGradients(cUpdateFlow[i + 1])) ReturnFalse; } //--- CNeuronSpikeDepthWiseResidual* resid = cSKFlow[0]; uint hidden_dimension = resid.GetWindow(); uint units = cLocal.Neurons() / hidden_dimension; uint dimension = Neurons() / units; //--- curr = cGlobal[-1]; if(!curr || !DeConcat(cLocal.getPrevOutput(), cETROF.getPrevOutput(), curr.getGradient(), cUpdateFlow[0].getPrevOutput(), cUpdateFlow[0].getGradient(), hidden_dimension, hidden_dimension, hidden_dimension, hidden_dimension, units)) ReturnFalse; Deactivation(curr); for(int i = cGlobal.Total() - 2; i >= 0; i--) { curr = cGlobal[i]; if(!curr || !curr.CalcHiddenGradients(cGlobal[i + 1])) ReturnFalse; } //--- if(!DeConcat(cLocal.getGradient(), cETROF.getGradient(), curr.getGradient(), hidden_dimension, hidden_dimension, units)) ReturnFalse; if(!SumAndNormilize(cLocal.getGradient(), cLocal.getPrevOutput(), cLocal.getGradient(), hidden_dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!SumAndNormilize(cETROF.getGradient(), cETROF.getPrevOutput(), cETROF.getPrevOutput(), hidden_dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(cLocal); //--- if(!cETROF.CalcHiddenGradients(cLocal.AsObject())) ReturnFalse; if(cETROF.Activation() != None) if(!DeActivation(cETROF.getOutput(), cETROF.getPrevOutput(), cETROF.getPrevOutput(), cETROF.Activation())) ReturnFalse; if(!SumAndNormilize(cETROF.getGradient(), cETROF.getPrevOutput(), cETROF.getGradient(), hidden_dimension, false, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cETROF.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cETROF.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cLocal.UpdateInputWeights(cETROF.AsObject())) ReturnFalse; //--- CNeuronBaseOCL* prev = cGlobal[0]; CNeuronBaseOCL* curr = NULL; for(int i = 1; i < cGlobal.Total(); i++) { curr = cGlobal[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cUpdateFlow[0]; for(int i = 1; i < cUpdateFlow.Total(); i++) { curr = cUpdateFlow[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!CNeuronSpikeSuperKernelBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::Save(const int file_handle) { if(!CNeuronSpikeSuperKernelBlock::Save(file_handle)) ReturnFalse; if(!cETROF.Save(file_handle)) ReturnFalse; if(!cLocal.Save(file_handle)) ReturnFalse; //--- if(!cGlobal.Save(file_handle)) ReturnFalse; if(!cUpdateFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::Load(const int file_handle) { if(!CNeuronSpikeSuperKernelBlock::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cETROF.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLocal.AsObject())) ReturnFalse; //--- if(!cGlobal.Load(file_handle)) ReturnFalse; if(!cUpdateFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeSuperKernelBlock::WeightsUpdate(source, tau)) ReturnFalse //--- CNeuronPSSEFlow* Source = source; dWeightsUpdate(cETROF, Source, tau); dWeightsUpdate(cLocal, Source, tau); dWeightsUpdate(cGlobal, Source, tau); dWeightsUpdate(cUpdateFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronPSSEFlow::Clear(void) { if(!CNeuronSpikeSuperKernelBlock::Clear()) ReturnFalse; if(!cETROF.Clear()) ReturnFalse; if(!cLocal.Clear()) ReturnFalse; //--- if(!cGlobal.ClearStates()) ReturnFalse; if(!cUpdateFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSSEFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeSuperKernelBlock::SetOpenCL(obj); cETROF.SetOpenCL(OpenCL); cLocal.SetOpenCL(OpenCL); cGlobal.SetOpenCL(OpenCL); cUpdateFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronPSSEFlow::TrainMode(bool flag) { CNeuronSpikeSuperKernelBlock::TrainMode(flag); cETROF.TrainMode(bTrain); cLocal.TrainMode(bTrain); cGlobal.TrainMode(bTrain); cUpdateFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeUSR : public CNeuronSpikeADM { protected: CNeuronCreateICEFlow cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSpikeUSR(void) {}; ~CNeuronSpikeUSR(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeUSR; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint stack_size, uint dimension, uint patch_dimension, uint variables, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeADM::Init(numOutputs, myIndex, open_cl, patch_dimension, stack_size, variables, optimization_type, batch)) ReturnFalse; if(!cFlow.Init(0, 0, OpenCL, stack_size, dimension, patch_dimension, variables, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cFlow.FeedForward(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeADM::feedForward(cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; if(!CNeuronSpikeADM::calcInputGradients(cFlow.AsObject())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cFlow.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!CNeuronSpikeADM::updateInputWeights(cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::Save(const int file_handle) { if(!CNeuronSpikeADM::Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::Load(const int file_handle) { if(!CNeuronSpikeADM::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeADM::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeUSR* Source = source; dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeUSR::Clear(void) { if(!CNeuronSpikeADM::Clear()) ReturnFalse; if(!cFlow.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeUSR::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeADM::SetOpenCL(obj); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeUSR::TrainMode(bool flag) { CNeuronSpikeADM::TrainMode(flag); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronInitialFlow : public CNeuronBaseOCL { protected: CParams cCandidates; CLayer cScores; CNeuronAddToStack cStack; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronInitialFlow(void) {}; ~CNeuronInitialFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint window_out, uint candidates, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronInitialFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint units, uint window_out, uint candidates, uint stack_size, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * window_out, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cCandidates.Init(0, index, OpenCL, units * window_out * candidates, optimization, iBatch)) ReturnFalse; cCandidates.SetActivationFunction(None); CBufferFloat* temp = cCandidates.getWeightsParams(); if(!temp || !temp.Fill(0)) ReturnFalse; index++; //--- cScores.Clear(); cScores.SetOpenCL(OpenCL); CNeuronSpikeDepthWiseConv* dw_conv = NULL; CNeuronSoftMaxOCL* softmax = NULL; uint window_in = window; for(int i = 0; i < 2; i++) { dw_conv = new CNeuronSpikeDepthWiseConv(); if(!dw_conv || !dw_conv.Init(0, index, OpenCL, window_in, candidates, 3, 1, units, 1, optimization, iBatch) || !cScores.Add(dw_conv)) DeleteObjAndFalse(dw_conv); index++; window_in = candidates; } softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, units * candidates, optimization, iBatch) || !cScores.Add(softmax)) DeleteObjAndFalse(softmax); index++; if(!cStack.Init(0, index, OpenCL, stack_size, softmax.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) if(!cCandidates.FeedForward()) ReturnFalse; //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScores.Total(); i++) { curr = cScores[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- CNeuronSoftMaxOCL* softmax = prev; uint units = softmax.Heads(); uint candidates = softmax.Neurons() / units; uint dimension = Neurons() / units; if(!MatMul(cCandidates.getOutput(), softmax.getOutput(), Output, dimension, candidates, 1, units, true)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronSoftMaxOCL* softmax = cScores[-1]; if(!cStack.FeedForward(softmax)) ReturnFalse; //--- uint units = softmax.Heads(); uint candidates = softmax.Neurons() / units; uint dimension = Neurons() / units; if(!DiversityLoss(cStack.AsObject(), cStack.GetStackSize(), softmax.Neurons(), false)) ReturnFalse; //--- if(!MatMulGrad(cCandidates.getOutput(), cCandidates.getGradient(), softmax.getOutput(), softmax.getGradient(), Gradient, dimension, candidates, 1, units, true)) ReturnFalse; if(!SumAndNormilize(softmax.getGradient(), cStack.getGradient(), softmax.getGradient(), candidates, false, 0, 0, 0, 1)) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cScores.Total() - 2; i >= 0; i--) { curr = cScores[i]; if(!curr || !curr.CalcHiddenGradients(cScores[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cCandidates.UpdateInputWeights()) ReturnFalse; //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScores.Total(); i++) { curr = cScores[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { //--- if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronInitialFlow* Source = source; dWeightsUpdate(cCandidates, Source, tau); dWeightsUpdate(cScores, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::Save(const int file_handle) { //--- if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cCandidates.Save(file_handle)) ReturnFalse; if(!cScores.Save(file_handle)) ReturnFalse; if(!cStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::Load(const int file_handle) { //--- if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cCandidates.AsObject())) ReturnFalse; if(!cScores.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cStack.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronInitialFlow::Clear(void) { //--- if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cCandidates.Clear()) ReturnFalse; if(!cScores.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInitialFlow::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cCandidates.SetOpenCL(OpenCL); cScores.SetOpenCL(OpenCL); cStack.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronInitialFlow::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cCandidates.TrainMode(bTrain); cScores.TrainMode(bTrain); cStack.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSpikeSMR : public CNeuronMSRes { protected: CNeuronSpikeSuperKernelBlock cUpsampleFlow; CNeuronBaseOCL cConcat; CNeuronSpikeConvGRU2D cConvGRU; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronSpikeSMR(void) {}; ~CNeuronSpikeSMR(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint second_window, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronSpikeSMR; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; virtual CBufferFloat* GetHidden() const { return cConvGRU.getOutput(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint window, uint second_window, uint window_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, units, window_out, 1, optimization_type, batch)) ReturnFalse; //--- uint windows[] = {3, 5}; uint steps[] = {1, 1}; uint index = 0; if(!cUpsampleFlow.Init(0, index, OpenCL, window, window_out, windows, steps, units, 1, optimization, iBatch)) ReturnFalse; index++; if(!cConcat.Init(0, index, OpenCL, (window + second_window)*units, optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); index++; if(!cConvGRU.Init(0, index, OpenCL, units, (window + second_window), window_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { if(!NeuronOCL || !SecondInput) ReturnFalse; if(!cUpsampleFlow.FeedForward(NeuronOCL)) ReturnFalse; //--- uint units = GetUnits(); uint window_flow = NeuronOCL.Neurons() / units;; uint second_window = SecondInput.Total() / units; if(!Concat(NeuronOCL.getOutput(), SecondInput, cConcat.getOutput(), window_flow, second_window, units)) ReturnFalse; if(!cConvGRU.FeedForward(cConcat.AsObject())) ReturnFalse; if(!CNeuronMSRes::feedForward(cConvGRU.AsObject())) ReturnFalse; if(!SumAndNormilize(cUpsampleFlow.getOutput(), Output, Output, GetWindow(), false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; //--- if(!CNeuronMSRes::calcInputGradients(cConvGRU.AsObject())) ReturnFalse; if(!cConcat.CalcHiddenGradients(cConvGRU.AsObject())) ReturnFalse; //--- if(!DeActivation(cUpsampleFlow.getOutput(), cUpsampleFlow.getGradient(), Gradient, cUpsampleFlow.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cUpsampleFlow.AsObject())) ReturnFalse; //--- uint units = GetUnits(); uint window_flow = NeuronOCL.Neurons() / units;; uint second_window = SecondInput.Total() / units; if(!DeConcat(cConcat.getPrevOutput(), SecondGradient, cConcat.getGradient(), window_flow, second_window, units)) ReturnFalse; if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), cConcat.getPrevOutput(), cConcat.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), cConcat.getPrevOutput(), NeuronOCL.getGradient(), window_flow, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { if(!cUpsampleFlow.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- if(!cConvGRU.UpdateInputWeights(cConcat.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cConvGRU.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronSpikeSMR* Source = source; dWeightsUpdate(cUpsampleFlow, Source, tau); dWeightsUpdate(cConvGRU, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; if(!cUpsampleFlow.Save(file_handle)) ReturnFalse; if(!cConvGRU.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cUpsampleFlow.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cConvGRU.AsObject())) ReturnFalse; //--- if(!cConcat.Init(0, 1, OpenCL, cConvGRU.GetChanelsIn()*cConvGRU.GetUnits(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSpikeSMR::Clear(void) { if(!CNeuronMSRes::Clear()) ReturnFalse; if(!cUpsampleFlow.Clear()) ReturnFalse; if(!cConvGRU.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSMR::SetOpenCL(COpenCLMy *obj) { CNeuronMSRes::SetOpenCL(obj); cUpsampleFlow.SetOpenCL(OpenCL); cConvGRU.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSpikeSMR::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); cUpsampleFlow.TrainMode(bTrain); cConvGRU.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronEVAFlow : public CNeuronSpikeSMR { protected: CLayer cPrepare; CLayer cEncoder; CNeuronInitialFlow cInitFlow; CLayer cDecoder; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override { return feedForward(NeuronOCL); } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override { return updateInputWeights(NeuronOCL); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override { return calcInputGradients(NeuronOCL); } public: CNeuronEVAFlow(void) {}; ~CNeuronEVAFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint forecast, uint dimension, uint hidden_dimension, uint layers, uint candidates, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronEVAFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual void TrainMode(bool flag) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint forecast, uint dimension, uint hidden_dimension, uint layers, uint candidates, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronSpikeSMR::Init(numOutputs, myIndex, open_cl, (forecast + 1) / 2, hidden_dimension, 2 * hidden_dimension, 2 * dimension, optimization_type, batch)) ReturnFalse; //--- Prepare cPrepare.Clear(); cPrepare.SetOpenCL(OpenCL); CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeUSR* usr = NULL; uint index = 0; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, dimension, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; usr = new CNeuronSpikeUSR(); if(!usr || !usr.Init(0, index, OpenCL, forecast, dimension, hidden_dimension, 1, optimization, iBatch) || !cPrepare.Add(usr)) DeleteObjAndFalse(usr); //--- Encoder / Decoder cEncoder.Clear(); cDecoder.Clear(); cEncoder.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronSpikeSuperKernelBlock* block = NULL; CNeuronSpikeSMR* smr = NULL; uint windows[] = { 3, 5}; uint steps_enc[] = {1, 2}; uint steps_dec[] = {1, 1}; uint units_in = forecast; uint units_out = MathMax((units_in + 1) / 2, 1); for(uint i = 0; i < layers; i++) { index++; block = new CNeuronSpikeSuperKernelBlock(); if(!block || !block.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, windows, steps_enc, units_out, 1, optimization, iBatch) || !cEncoder.Add(block)) DeleteObjAndFalse(block); if(i > 0) { index++; smr = new CNeuronSpikeSMR(); if(!smr || !smr.Init(0, index, OpenCL, units_out, hidden_dimension, (1 + int(i != (layers - 1))) * hidden_dimension, 2 * hidden_dimension, optimization, iBatch) || !cDecoder.Add(smr)) DeleteObjAndFalse(smr); } if(i < (layers - 1)) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, units_out * hidden_dimension * 2, optimization, iBatch) || !cDecoder.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); } units_in = units_out; units_out = MathMax((units_in + 1) / 2, 1); } index++; if(!cInitFlow.Init(0, index, OpenCL, hidden_dimension, units_in, hidden_dimension, candidates, 2 * forecast, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!cInitFlow.FeedForward(prev)) ReturnFalse; CNeuronBaseOCL* flow = cInitFlow.AsObject(); for(int i = cDecoder.Total() - 1; i >= 0; i -= 2) { curr = cDecoder[i]; if(!curr || !curr.FeedForward(flow, prev.getOutput())) ReturnFalse; flow = curr; if(i <= 0) continue; prev = cDecoder[i - 1]; if(!prev) ReturnFalse; CNeuronSpikeSMR* smr = flow; uint dimension = smr.GetWindow() / 2; uint units = prev.Neurons() / (2 * dimension); if(!cEncoder[i / 2] || !Concat(smr.GetHidden(), cEncoder[i / 2].getOutput(), prev.getOutput(), dimension, dimension, units)) ReturnFalse; } if(!CNeuronSpikeSMR::feedForward(flow, prev.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- CNeuronBaseOCL* prev = (cDecoder.Total() >= 1 ? cDecoder[0] : cEncoder[-1]); CNeuronBaseOCL* flow = (cDecoder.Total() >= 2 ? cDecoder[1] : cInitFlow.AsObject()); CNeuronBaseOCL* curr = NULL; //--- if(!CNeuronSpikeSMR::calcInputGradients(flow, prev.getOutput(), prev.getGradient(), (ENUM_ACTIVATION) prev.Activation())) ReturnFalse; //--- for(int i = 1; i < cDecoder.Total(); i += 2) { flow = cDecoder[i]; prev = ((i + 1) < cDecoder.Total() ? cDecoder[i + 1] : cEncoder[-1]); curr = ((i + 2) < cDecoder.Total() ? cDecoder[i + 2] : cInitFlow.AsObject()); if(!curr || !prev || !curr.CalcHiddenGradients(flow, prev.getOutput(), prev.getGradient(), (ENUM_ACTIVATION) prev.Activation())) ReturnFalse; } //--- CBufferFloat* temp = prev.getGradient(); if(!prev.SetGradient(prev.getPrevOutput(), false) || !prev.CalcHiddenGradients(cInitFlow.AsObject()) || !SumAndNormilize(temp, prev.getGradient(), temp, 1, false, 0, 0, 0, 1) || !prev.SetGradient(temp, false)) ReturnFalse; curr = prev; //--- for(int i = cEncoder.Total() - 2; i >= 0; i--) { prev = cEncoder[i]; if(!prev || !prev.CalcHiddenGradients(curr)) ReturnFalse; curr = cDecoder[i * 2]; uint dimension = ((CNeuronSpikeSuperKernelBlock*)prev).GetWindow(); uint units = ((CNeuronSpikeSuperKernelBlock*)prev).GetUnits(); if(!curr || !DeConcat(curr.getPrevOutput(), prev.getPrevOutput(), curr.getGradient(), dimension, dimension, units)) ReturnFalse; if(prev.Activation() != None) if(!DeActivation(prev.getOutput(), prev.getPrevOutput(), prev.getPrevOutput(), prev.Activation())) ReturnFalse; if(!SumAndNormilize(prev.getGradient(), prev.getPrevOutput(), prev.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; curr = prev; } //--- for(int i = cPrepare.Total() - 1; i >= 0; i--) { prev = cPrepare[i]; if(!prev || !prev.CalcHiddenGradients(curr)) ReturnFalse; curr = prev; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- for(int i = 0; i < cEncoder.Total(); i++) { curr = cEncoder[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } if(!cInitFlow.UpdateInputWeights(prev)) ReturnFalse; CNeuronBaseOCL* flow = cInitFlow.AsObject(); for(int i = cDecoder.Total() - 1; i >= 0; i -= 2) { curr = cDecoder[i]; if(!curr || !curr.UpdateInputWeights(flow, prev.getOutput())) ReturnFalse; flow = curr; if(i <= 0) continue; prev = cDecoder[i - 1]; } if(!CNeuronSpikeSMR::updateInputWeights(flow, prev.getOutput())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::Save(const int file_handle) { if(!CNeuronSpikeSMR::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cEncoder.Save(file_handle)) ReturnFalse; if(!cDecoder.Save(file_handle)) ReturnFalse; if(!cInitFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::Load(const int file_handle) { if(!CNeuronSpikeSMR::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!cEncoder.Load(file_handle)) ReturnFalse; if(!cDecoder.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cInitFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeSMR::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronEVAFlow* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cEncoder, Source, tau); dWeightsUpdate(cDecoder, Source, tau); dWeightsUpdate(cInitFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronEVAFlow::Clear(void) { if(!CNeuronSpikeSMR::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cEncoder.ClearStates()) ReturnFalse; if(!cDecoder.ClearStates()) ReturnFalse; if(!cInitFlow.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEVAFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeSMR::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cEncoder.SetOpenCL(OpenCL); cDecoder.SetOpenCL(OpenCL); cInitFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronEVAFlow::TrainMode(bool flag) { CNeuronSpikeSMR::TrainMode(flag); cPrepare.TrainMode(bTrain); cEncoder.TrainMode(bTrain); cDecoder.TrainMode(bTrain); cInitFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResFlowHTR : public CNeuronMSRes { protected: CLayer cContextProjection; CLayer cMFProjection; CNeuronBaseOCL cContextVsMF; CNeuronSpikeConvGRU2D cGRU; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; public: CNeuronResFlowHTR(void) {}; ~CNeuronResFlowHTR(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint mf_dimension, uint context_size, uint inside_dimension, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronResFlowHTR; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint units, uint mf_dimension, uint context_size, uint inside_dimension, uint chanels_out, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, units, chanels_out, 1, optimization_type, batch)) ReturnFalse; //--- cContextProjection.Clear(); cMFProjection.Clear(); cContextProjection.SetOpenCL(OpenCL); cMFProjection.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronSpikeActivation* activat = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSpikeSuperKernelBlock* block = NULL; //--- Context Projection uint index = 0; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, context_size, optimization, iBatch) || !cContextProjection.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; activat = new CNeuronSpikeActivation(); if(!activat || !activat.Init(inside_dimension, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cContextProjection.Add(activat)) DeleteObjAndFalse(activat); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, inside_dimension, optimization, iBatch) || !cContextProjection.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, neuron.Neurons(), iBatch, optimization) || !cContextProjection.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; //--- Motion Feature uint windows[] = {3, 5}; uint steps[] = {1, 1}; uint chan_in = mf_dimension; for(int i = 0; i < 3; i++) { block = new CNeuronSpikeSuperKernelBlock(); if(!block || !block.Init(0, index, OpenCL, chan_in, inside_dimension, windows, steps, units, 1, optimization, iBatch) || !cMFProjection.Add(block)) DeleteObjAndFalse(block); index++; chan_in = inside_dimension; } //--- if(!cContextVsMF.Init(0, index, OpenCL, block.Neurons(), optimization, iBatch)) ReturnFalse; cContextVsMF.SetActivationFunction(None); index++; if(!cGRU.Init(0, index, OpenCL, units, inside_dimension, chanels_out, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* context = cContextProjection[0]; CNeuronBaseOCL* mf = NULL; //--- if(!context.SetOutput(SecondInput, true)) ReturnFalse; //--- for(int i = 0; i < cMFProjection.Total(); i++) { mf = cMFProjection[i]; if(!mf || !mf.FeedForward(prev)) ReturnFalse; prev = mf; } //--- prev = context; for(int i = 1; i < cContextProjection.Total(); i++) { context = cContextProjection[i]; if(!context || !context.FeedForward(prev)) ReturnFalse; prev = context; } //--- if(!SumVecMatrix(context.getOutput(), mf.getOutput(), cContextVsMF.getOutput(), context.Neurons(), 1, 0, 0, 0, 1)) ReturnFalse; if(!cGRU.FeedForward(cContextVsMF.AsObject())) ReturnFalse; if(!CNeuronMSRes::feedForward(cGRU.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::calcInputGradients(CNeuronBaseOCL *NeuronOCL, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!NeuronOCL || !SecondGradient) ReturnFalse; if(!cContextProjection[0] || !cContextProjection[0].SetGradient(SecondGradient, true)) ReturnFalse; cContextProjection[0].SetActivationFunction(SecondActivation); //--- if(!CNeuronMSRes::calcInputGradients(cGRU.AsObject())) ReturnFalse; if(!cContextVsMF.CalcHiddenGradients(cGRU.AsObject())) ReturnFalse; CNeuronBaseOCL* context = cContextProjection[-1]; CNeuronBaseOCL* mf = cMFProjection[-1]; if(!context || !mf || !SumVecMatrixGrad(context.getGradient(), mf.getGradient(), cContextVsMF.getGradient(), context.Neurons(), 1, 0, 0, 0, 1)) ReturnFalse; //--- for(int i = cContextProjection.Total() - 2; i >= 0; i--) { context = cContextProjection[i]; if(!context || !context.CalcHiddenGradients(cContextProjection[i + 1])) ReturnFalse; } //--- for(int i = cMFProjection.Total() - 2; i >= 0; i--) { mf = cMFProjection[i]; if(!mf || !mf.CalcHiddenGradients(cMFProjection[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(mf)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *second) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* context = cContextProjection[0]; CNeuronBaseOCL* mf = NULL; //--- for(int i = 0; i < cMFProjection.Total(); i++) { mf = cMFProjection[i]; if(!mf || !mf.UpdateInputWeights(prev)) ReturnFalse; prev = mf; } //--- prev = context; for(int i = 1; i < cContextProjection.Total(); i++) { context = cContextProjection[i]; if(!context || !context.UpdateInputWeights(prev)) ReturnFalse; prev = context; } //--- if(!cGRU.UpdateInputWeights(cContextVsMF.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cGRU.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(!cMFProjection.Save(file_handle)) ReturnFalse; if(!cContextProjection.Save(file_handle)) ReturnFalse; if(!cGRU.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(!cMFProjection.Load(file_handle)) ReturnFalse; if(!cContextProjection.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cGRU.AsObject())) ReturnFalse; //--- if(!cMFProjection[-1] || !cContextVsMF.Init(0, cMFProjection.Total() + cContextProjection.Total(), OpenCL, cMFProjection[-1].Neurons(), optimization, iBatch)) ReturnFalse; cContextVsMF.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronResFlowHTR* Source = source; dWeightsUpdate(cMFProjection, Source, tau); dWeightsUpdate(cContextProjection, Source, tau); dWeightsUpdate(cGRU, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowHTR::Clear(void) { if(!CNeuronMSRes::Clear()) ReturnFalse; //--- if(!cMFProjection.ClearStates()) ReturnFalse; if(!cContextProjection.ClearStates()) ReturnFalse; if(!cGRU.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlowHTR::SetOpenCL(COpenCLMy *obj) { CNeuronMSRes::SetOpenCL(obj); //--- cMFProjection.SetOpenCL(OpenCL); cContextProjection.SetOpenCL(OpenCL); cGRU.SetOpenCL(OpenCL); cContextVsMF.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlowHTR::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); //--- cMFProjection.TrainMode(bTrain); cContextProjection.TrainMode(bTrain); cGRU.TrainMode(bTrain); cContextVsMF.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::LinearUpsample(CNeuronBaseOCL *data, CNeuronBaseOCL *upsample, uint upsampl_mult, uint units, uint variables) { if(!OpenCL || !data || !upsample || units <= 0 || upsampl_mult <= 0 || variables <= 0) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { units, variables, upsampl_mult }; uint kernel = def_k_LinearUpsample; setBuffer(kernel, def_k_lu_data, data.getOutputIndex()) setBuffer(kernel, def_k_lu_upsample, upsample.getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!upsample.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronBaseOCL::LinearUpsampleGrad(CNeuronBaseOCL *data, CNeuronBaseOCL *upsample, uint upsampl_mult, uint units, uint variables) { if(!OpenCL || !data || !upsample || units <= 0 || upsampl_mult <= 0 || variables <= 0) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { units, variables, upsampl_mult }; uint local_work_size[] = { 1, 1, (uint)MathMin(upsampl_mult, OpenCL.GetMaxLocalSize(2)) }; uint kernel = def_k_LinearUpsampleGrad; setBuffer(kernel, def_k_lug_data_gr, data.getGradientIndex()) setBuffer(kernel, def_k_lug_upsample_gr, upsample.getGradientIndex()) setArgument(kernel, def_k_lug_dimension_htr, (int)upsampl_mult) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!data.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResFlow : public CNeuronSpikeSuperKernelBlock { protected: CNeuronPSSE cPrepare; CNeuronEVAFlow cLTR; CLayer cContext; CLayer cMFE; CNeuronBaseOCL cUpsampleLTR; CNeuronBaseOCL cMFEvsUpLTR; CNeuronResFlowHTR cHTR; CNeuronBaseOCL cUpdatedFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronResFlow(void) {}; ~CNeuronResFlow(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint hidden_dimension, uint ltr_forecast, uint htr_points_to_segment, uint ltr_layers, uint ltr_candidates, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronResFlow; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint hidden_dimension, uint ltr_forecast, uint htr_points_to_segment, uint ltr_layers, uint ltr_candidates, ENUM_OPTIMIZATION optimization_type, uint batch) { { uint windows[] = {3, 5}; uint steps[] = {1, 1}; if(!CNeuronSpikeSuperKernelBlock::Init(numOutputs, myIndex, open_cl, hidden_dimension, dimension, windows, steps, ltr_forecast * htr_points_to_segment, 1, optimization_type, batch)) ReturnFalse; } //--- uint index = 0; if(!cPrepare.Init(0, index, OpenCL, 2 * htr_points_to_segment, dimension, hidden_dimension, 1, optimization, iBatch)) ReturnFalse; //--- Context cContext.Clear(); cContext.SetOpenCL(OpenCL); CNeuronBaseOCL* neuron = NULL; CNeuronSpikeSuperKernelBlock* block = NULL; CNeuronSpikeActivation* activ = NULL; CNeuronBatchNormOCL* norm = NULL; index++; { uint windows[] = {3, 5, 7}; uint steps[] = {2, 2, 2}; block = new CNeuronSpikeSuperKernelBlock(); if(!block || !block.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, windows, steps, (htr_points_to_segment + 3) / 4, 1, optimization, iBatch) || !cContext.Add(block)) DeleteObjAndFalse(block); } index++; activ = new CNeuronSpikeActivation(); if(!activ || !activ.Init(hidden_dimension, index, OpenCL, block.Neurons(), optimization, iBatch) || !cContext.Add(activ)) DeleteObjAndFalse(activ); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, hidden_dimension, optimization, iBatch) || !cContext.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, neuron.Neurons(), iBatch, optimization) || !cContext.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); //--- index++; if(!cLTR.Init(0, index, OpenCL, ltr_forecast, cContext[-1].Neurons(), hidden_dimension, ltr_layers, ltr_candidates, optimization, iBatch)) ReturnFalse; //--- cMFE.Clear(); cMFE.SetOpenCL(OpenCL); index++; { uint windows[] = {3, 5}; uint steps[] = {1, 1}; block = new CNeuronSpikeSuperKernelBlock(); if(!block || !block.Init(0, index, OpenCL, hidden_dimension, htr_points_to_segment * hidden_dimension, windows, steps, ltr_forecast, 1, optimization, iBatch) || !cMFE.Add(block)) DeleteObjAndFalse(block); } //--- index++; if(!cUpsampleLTR.Init(0, index, OpenCL, cLTR.Neurons()*htr_points_to_segment, optimization, iBatch)) ReturnFalse; cUpsampleLTR.SetActivationFunction(None); index++; if(!cMFEvsUpLTR.Init(0, index, OpenCL, cUpsampleLTR.Neurons() + cMFE[-1].Neurons(), optimization, iBatch)) ReturnFalse; cMFEvsUpLTR.SetActivationFunction(None); index++; if(!cHTR.Init(0, index, OpenCL, ltr_forecast * htr_points_to_segment, 2 * hidden_dimension, hidden_dimension, hidden_dimension, hidden_dimension, optimization, iBatch)) ReturnFalse; index++; if(!cUpdatedFlow.Init(0, index, OpenCL, cHTR.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cPrepare.AsObject(); CNeuronBaseOCL*curr = NULL; for(int i = 0; i < cContext.Total(); i++) { curr = cContext[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!cLTR.FeedForward(prev)) ReturnFalse; //--- uint hidden_dimension = cLTR.GetWindow(); uint ltr_forecat = cLTR.GetUnits(); uint htr_points_to_segment = cUpsampleLTR.Neurons() / cLTR.Neurons(); if(!LinearUpsample(cLTR.AsObject(), cUpsampleLTR.AsObject(), htr_points_to_segment, ltr_forecat, hidden_dimension)) ReturnFalse; //--- prev = cUpsampleLTR.AsObject(); for(int i = 0; i < cMFE.Total(); i++) { curr = cMFE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!Concat(prev.getOutput(), cUpsampleLTR.getOutput(), cMFEvsUpLTR.getOutput(), hidden_dimension, hidden_dimension, ltr_forecat * htr_points_to_segment)) ReturnFalse; if(!cHTR.FeedForward(cMFEvsUpLTR.AsObject(), cContext[-1].getOutput())) ReturnFalse; //--- if(!SumAndNormilize(cUpsampleLTR.getOutput(), cHTR.getOutput(), cUpdatedFlow.getOutput(), hidden_dimension, false, 0, 0, 0, 1) || !SumVecMatrix(cPrepare.getOutput(), cUpdatedFlow.getOutput(), cUpdatedFlow.getOutput(), hidden_dimension, 1, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronSpikeSuperKernelBlock::feedForward(cUpdatedFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeSuperKernelBlock::calcInputGradients(cUpdatedFlow.AsObject())) ReturnFalse; if(!DeActivation(cHTR.getOutput(), cHTR.getGradient(), cUpdatedFlow.getGradient(), cHTR.Activation())) ReturnFalse; if(!cContext[-1] || !cMFEvsUpLTR.CalcHiddenGradients(cHTR.AsObject(), cContext[-1].getOutput(), cContext[-1].getGradient(), (ENUM_ACTIVATION)cContext[-1].Activation())) ReturnFalse; //--- uint hidden_dimension = cLTR.GetWindow(); uint ltr_forecat = cLTR.GetUnits(); uint htr_points_to_segment = cUpsampleLTR.Neurons() / cLTR.Neurons(); //--- if(!cMFE[-1] || !DeConcat(cMFE[-1].getGradient(), cUpsampleLTR.getGradient(), cMFEvsUpLTR.getGradient(), hidden_dimension, hidden_dimension, ltr_forecat * htr_points_to_segment)) ReturnFalse; for(int i = cMFE.Total() - 2; i >= 0; i--) { if(!cMFE[i] || !cMFE[i].CalcHiddenGradients(cMFE[i + 1])) ReturnFalse; } if(!cUpsampleLTR.CalcHiddenGradients(cMFE[0])) ReturnFalse; //--- if(!SumAndNormilize(cUpsampleLTR.getOutput(), cHTR.getOutput(), cUpsampleLTR.getGradient(), hidden_dimension, false, 0, 0, 0, 1) || !SumAndNormilize(cUpsampleLTR.getGradient(), cUpdatedFlow.getGradient(), cUpsampleLTR.getGradient(), hidden_dimension, false, 0, 0, 0, 1)) ReturnFalse; float error = 0; if(!cUpsampleLTR.calcOutputGradients(cUpsampleLTR.getGradient(), error)) ReturnFalse; if(!LinearUpsampleGrad(cLTR.AsObject(), cUpsampleLTR.AsObject(), htr_points_to_segment, ltr_forecat, hidden_dimension)) ReturnFalse; Deactivation(cLTR); CNeuronBaseOCL* next = cLTR.AsObject(); for(int i = cContext.Total() - 1; i >= 0; i--) { if(!cContext[i] || !cContext[i].CalcHiddenGradients(next)) ReturnFalse; next = cContext[i]; } //--- if(!cPrepare.CalcHiddenGradients(next)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cPrepare.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cPrepare.AsObject(); CNeuronBaseOCL*curr = NULL; for(int i = 0; i < cContext.Total(); i++) { curr = cContext[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cLTR.UpdateInputWeights(prev)) ReturnFalse; //--- prev = cUpsampleLTR.AsObject(); for(int i = 0; i < cMFE.Total(); i++) { curr = cMFE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cHTR.UpdateInputWeights(cMFEvsUpLTR.AsObject(), cContext[-1].getOutput())) ReturnFalse; //--- if(!CNeuronSpikeSuperKernelBlock::updateInputWeights(cUpdatedFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::Save(const int file_handle) { if(!CNeuronSpikeSuperKernelBlock::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cContext.Save(file_handle)) ReturnFalse; if(!cLTR.Save(file_handle)) ReturnFalse; if(!cMFE.Save(file_handle)) ReturnFalse; if(!cHTR.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::Load(const int file_handle) { if(!CNeuronSpikeSuperKernelBlock::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cPrepare.AsObject())) ReturnFalse; if(!cContext.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLTR.AsObject())) ReturnFalse; if(!cMFE.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cHTR.AsObject())) ReturnFalse; //--- uint hidden_dimension = cLTR.GetWindow(); uint ltr_forecat = cLTR.GetUnits(); uint htr_points_to_segment = cHTR.Neurons() / cLTR.Neurons(); //--- uint index = 2 + cContext.Total() + cMFE.Total(); if(!cUpsampleLTR.Init(0, index, OpenCL, cHTR.Neurons(), optimization, iBatch)) ReturnFalse; cUpsampleLTR.SetActivationFunction(None); index++; if(!cMFEvsUpLTR.Init(0, index, OpenCL, cUpsampleLTR.Neurons() + cMFE[-1].Neurons(), optimization, iBatch)) ReturnFalse; cMFEvsUpLTR.SetActivationFunction(None); index += 2; if(!cUpdatedFlow.Init(0, index, OpenCL, cHTR.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeSuperKernelBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronResFlow* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cContext, Source, tau); dWeightsUpdate(cLTR, Source, tau); dWeightsUpdate(cMFE, Source, tau); dWeightsUpdate(cHTR, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlow::Clear(void) { if(!CNeuronSpikeSuperKernelBlock::Clear()) ReturnFalse; //--- if(!cPrepare.Clear()) ReturnFalse; if(!cContext.ClearStates()) ReturnFalse; if(!cLTR.Clear()) ReturnFalse; if(!cMFE.ClearStates()) ReturnFalse; if(!cHTR.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlow::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeSuperKernelBlock::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cContext.SetOpenCL(OpenCL); cLTR.SetOpenCL(OpenCL); cMFE.SetOpenCL(OpenCL); cHTR.SetOpenCL(OpenCL); cUpsampleLTR.SetOpenCL(OpenCL); cMFEvsUpLTR.SetOpenCL(OpenCL); cUpdatedFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlow::TrainMode(bool flag) { CNeuronSpikeSuperKernelBlock::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cContext.TrainMode(bTrain); cLTR.TrainMode(bTrain); cMFE.TrainMode(bTrain); cHTR.TrainMode(bTrain); cUpsampleLTR.TrainMode(bTrain); cMFEvsUpLTR.TrainMode(bTrain); cUpdatedFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronConfGateTailAwareMoE : public CNeuronBaseOCL { protected: CLayer cConfidenceGating; CLayer cScore; CParams cTailAwareMoE; CLayer cCGScore; CNeuronAddToStack cCGStack; CNeuronAddToStack cScoreStack; CNeuronBaseOCL cDensity; //--- virtual bool MixExpertsPredict(void); virtual bool MixExpertsPredictGrad(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronConfGateTailAwareMoE(void) {}; ~CNeuronConfGateTailAwareMoE(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint forecast, uint experts, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) override const { return defNeuronConfGateTailAwareMoE; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override { }; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint forecast, uint experts, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * forecast, optimization_type, batch)) ReturnFalse; activation = None; //--- cConfidenceGating.Clear(); cScore.Clear(); cCGScore.Clear(); cConfidenceGating.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cCGScore.SetOpenCL(OpenCL); //--- uint index = 0; if(!cTailAwareMoE.Init(0, index, OpenCL, dimension * forecast * 4 * experts, optimization, iBatch)) ReturnFalse; cTailAwareMoE.SetActivationFunction(None); //--- uint windows[] = {7, 5, 3}; uint steps[] = {2, 2, 2}; CNeuronSpikeSuperKernelBlock* conv = NULL; CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronSoftMaxOCL* softmax = NULL; CNeuronSpikeActivation* spike = NULL; //--- Score index++; conv = new CNeuronSpikeSuperKernelBlock(); if(!conv || !conv.Init(experts, index, OpenCL, dimension, (dimension + 1) / 2, windows, steps, units, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cScore.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(SoftPlus); index++; softmax = new CNeuronSoftMaxOCL(); if(!softmax || !softmax.Init(0, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cScore.Add(softmax)) DeleteObjAndFalse(softmax); softmax.SetHeads(1); index++; if(!cScoreStack.Init(0, index, OpenCL, 50, neuron.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- Confidence-Gating index++; conv = new CNeuronSpikeSuperKernelBlock(); if(!conv || !conv.Init(experts, index, OpenCL, dimension, (dimension + 1) / 2, windows, steps, units, 1, optimization, iBatch) || !cConfidenceGating.Add(conv)) DeleteObjAndFalse(conv); index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cConfidenceGating.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(SoftPlus); index++; norm = new CNeuronBatchNormOCL(); if(!norm || !norm.Init(0, index, OpenCL, neuron.Neurons(), iBatch, optimization) || !cConfidenceGating.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(SoftPlus); index++; spike = new CNeuronSpikeActivation(); if(!spike || !spike.Init(0, index, OpenCL, neuron.Neurons(), optimization, iBatch) || !cConfidenceGating.Add(spike)) DeleteObjAndFalse(spike); index++; if(!cCGStack.Init(0, index, OpenCL, 50, neuron.Neurons(), 1, optimization, iBatch)) ReturnFalse; //--- Confidence-Gating Score index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, experts, optimization, iBatch) || !cCGScore.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); //--- index++; if(!cDensity.Init(0, index, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cDensity.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::MixExpertsPredict(void) { if(!OpenCL || !Output || !cDensity.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[] = { Neurons() }; uint kernel = def_k_MixExpertsPredict; setBuffer(kernel, def_k_moepred_experts, cDensity.getOutputIndex()) setBuffer(kernel, def_k_moepred_outputs, getOutputIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::MixExpertsPredictGrad(void) { if(!OpenCL || !Gradient || !cDensity.getOutput()) ReturnFalse; //--- uint global_work_offset[1] = {0}; uint global_work_size[] = { Neurons() }; uint kernel = def_k_MixExpertsPredictGrad; setBuffer(kernel, def_k_moepredgr_experts, cDensity.getOutputIndex()) setBuffer(kernel, def_k_moepredgr_experts_gr, cDensity.getGradientIndex()) setBuffer(kernel, def_k_moepredgr_outputs_gr, getGradientIndex()) kernelExecute(kernel, global_work_offset, global_work_size) #ifdef _DEBUG if(!cDensity.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Score for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; #ifdef _DEBUG if(!curr.getOutput().BufferRead()) ReturnFalse; #endif } //--- Confidence-Gating prev = NeuronOCL; for(int i = 0; i < cConfidenceGating.Total(); i++) { curr = cConfidenceGating[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; #ifdef _DEBUG if(!curr.getOutput().BufferRead()) ReturnFalse; #endif } //--- Confidence-Gating Score prev = cCGScore[0]; if(!prev || !ElementMult(cConfidenceGating[-1].getOutput(), cScore[-1].getOutput(), prev.getOutput())) ReturnFalse; for(int i = 1; i < cCGScore.Total(); i++) { curr = cCGScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; #ifdef _DEBUG if(!curr.getOutput().BufferRead()) ReturnFalse; #endif } //--- Density if(bTrain) if(!cTailAwareMoE.FeedForward()) ReturnFalse; if(!MatMul(prev.getOutput(), cTailAwareMoE.getOutput(), cDensity.getOutput(), 1, prev.Neurons(), cDensity.Neurons(), 1, false)) ReturnFalse; if(!MixExpertsPredict()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!MixExpertsPredictGrad()) ReturnFalse; //--- Confidence-Gating Score CNeuronBaseOCL* next = NULL; CNeuronBaseOCL* curr = cCGScore[-1]; if(!curr || !MatMulGrad(curr.getOutput(), curr.getGradient(), cTailAwareMoE.getOutput(), cTailAwareMoE.getGradient(), cDensity.getGradient(), 1, curr.Neurons(), cDensity.Neurons(), 1, false)) ReturnFalse; next = curr; for(int i = cCGScore.Total() - 2; i >= 0; i--) { curr = cCGScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; #ifdef _DEBUG if(!curr.getGradient().BufferRead()) ReturnFalse; #endif } //--- next = cConfidenceGating[-1]; curr = cScore[-1]; if(!next || !curr || !ElementMultGrad(next.getOutput(), next.getGradient(), curr.getOutput(), curr.getGradient(), cCGScore[0].getGradient(), next.Activation(), curr.Activation())) ReturnFalse; //--- Confidence-Gating if(!cCGStack.FeedForward(next) || !DiversityLoss(cCGStack.AsObject(), cCGStack.GetStackSize(), next.Neurons(), false)) ReturnFalse; if(next.Activation() != None) if(!DeActivation(next.getOutput(), cCGStack.getGradient(), cCGStack.getGradient(), next.Activation())) ReturnFalse; if(!SumAndNormilize(next.getGradient(), cCGStack.getGradient(), next.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; for(int i = cConfidenceGating.Total() - 2; i >= 0; i--) { curr = cConfidenceGating[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; #ifdef _DEBUG if(!curr.getGradient().BufferRead()) ReturnFalse; #endif } //--- Score next = cScore[-1]; if(!cScoreStack.FeedForward(next) || !DiversityLoss(cScoreStack.AsObject(), cScoreStack.GetStackSize(), next.Neurons(), false)) ReturnFalse; if(next.Activation() != None) if(!DeActivation(next.getOutput(), cScoreStack.getGradient(), cScoreStack.getGradient(), next.Activation())) ReturnFalse; if(!SumAndNormilize(next.getGradient(), cScoreStack.getGradient(), next.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; for(int i = cScore.Total() - 2; i >= 0; i--) { curr = cScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; #ifdef _DEBUG if(!curr.getGradient().BufferRead()) ReturnFalse; #endif } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; CBufferFloat* temp = NeuronOCL.getGradient(); if(!NeuronOCL.SetGradient(NeuronOCL.getPrevOutput(), false) || !NeuronOCL.CalcHiddenGradients(cConfidenceGating[0]) || !SumAndNormilize(temp, NeuronOCL.getGradient(), temp, 1, false, 0, 0, 0, 1) || !NeuronOCL.SetGradient(temp, false)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Score for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating prev = NeuronOCL; for(int i = 0; i < cConfidenceGating.Total(); i++) { curr = cConfidenceGating[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Confidence-Gating Score prev = cCGScore[0]; for(int i = 1; i < cCGScore.Total(); i++) { curr = cCGScore[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Density if(!cTailAwareMoE.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronConfGateTailAwareMoE* Source = source; dWeightsUpdate(cScore, Source, tau); dWeightsUpdate(cConfidenceGating, Source, tau); dWeightsUpdate(cCGScore, Source, tau); dWeightsUpdate(cTailAwareMoE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cConfidenceGating.Save(file_handle)) ReturnFalse; if(!cScore.Save(file_handle)) ReturnFalse; if(!cCGScore.Save(file_handle)) ReturnFalse; if(!cTailAwareMoE.Save(file_handle)) ReturnFalse; if(!cScoreStack.Save(file_handle)) ReturnFalse; if(!cCGStack.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cConfidenceGating.Load(file_handle)) ReturnFalse; if(!cScore.Load(file_handle)) ReturnFalse; if(!cCGScore.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTailAwareMoE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScoreStack.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cCGStack.AsObject())) ReturnFalse; //--- uint index = 3 + cScore.Total() + cConfidenceGating.Total() + cCGScore.Total(); if(!cDensity.Init(0, index, OpenCL, 4 * Neurons(), optimization, iBatch)) ReturnFalse; cDensity.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronConfGateTailAwareMoE::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cConfidenceGating.ClearStates()) ReturnFalse; if(!cScore.ClearStates()) ReturnFalse; if(!cCGScore.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConfGateTailAwareMoE::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cConfidenceGating.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); cCGScore.SetOpenCL(OpenCL); cTailAwareMoE.SetOpenCL(OpenCL); cScoreStack.SetOpenCL(OpenCL); cCGStack.SetOpenCL(OpenCL); cDensity.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronConfGateTailAwareMoE::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cConfidenceGating.TrainMode(bTrain); cScore.TrainMode(bTrain); cCGScore.TrainMode(bTrain); cTailAwareMoE.TrainMode(bTrain); cScoreStack.TrainMode(bTrain); cCGStack.TrainMode(bTrain); cDensity.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronResFlowLattice : public CNeuronResFlow { protected: CNeuronConfGateTailAwareMoE cTailAwareMoE; CNeuronBaseOCL cConcat; CNeuronSpikeSuperKernelBlock cAdaptFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronResFlowLattice(void) {}; ~CNeuronResFlowLattice(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint hidden_dimension, uint ltr_forecast, uint htr_points_to_segment, uint ltr_layers, uint ltr_candidates, ENUM_OPTIMIZATION optimization_type, uint batch) override; //--- virtual int Type(void) override const { return defNeuronResFlowLattice; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint hidden_dimension, uint ltr_forecast, uint htr_points_to_segment, uint ltr_layers, uint ltr_candidates, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronResFlow::Init(numOutputs, myIndex, open_cl, dimension, hidden_dimension, ltr_forecast, htr_points_to_segment, ltr_layers, ltr_candidates, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cTailAwareMoE.Init(0, index, OpenCL, hidden_dimension, ltr_forecast, GetUnits(), ltr_candidates, optimization, iBatch)) ReturnFalse; index++; if(!cConcat.Init(0, index, OpenCL, 4 * hidden_dimension * GetUnits(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); index++; uint windows[] = {3, 5}; uint steps[] = {1, 1}; if(!cAdaptFlow.Init(0, index, OpenCL, 4 * hidden_dimension, hidden_dimension, windows, steps, GetUnits(), 1, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cPrepare.AsObject(); CNeuronBaseOCL*curr = NULL; for(int i = 0; i < cContext.Total(); i++) { curr = cContext[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!cLTR.FeedForward(prev)) ReturnFalse; //--- uint hidden_dimension = cLTR.GetWindow(); uint ltr_forecat = cLTR.GetUnits(); uint htr_points_to_segment = cUpsampleLTR.Neurons() / cLTR.Neurons(); if(!LinearUpsample(cLTR.AsObject(), cUpsampleLTR.AsObject(), htr_points_to_segment, ltr_forecat, hidden_dimension)) ReturnFalse; //--- prev = cUpsampleLTR.AsObject(); for(int i = 0; i < cMFE.Total(); i++) { curr = cMFE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!Concat(prev.getOutput(), cUpsampleLTR.getOutput(), cMFEvsUpLTR.getOutput(), hidden_dimension, hidden_dimension, GetUnits())) ReturnFalse; if(!cHTR.FeedForward(cMFEvsUpLTR.AsObject(), cContext[-1].getOutput())) ReturnFalse; //--- if(!cTailAwareMoE.FeedForward(cLTR.AsObject())) ReturnFalse; if(!Concat(cMFEvsUpLTR.getOutput(), cHTR.getOutput(), cTailAwareMoE.getOutput(), cConcat.getOutput(), 2 * hidden_dimension, hidden_dimension, hidden_dimension, GetUnits())) ReturnFalse; if(!cAdaptFlow.FeedForward(cConcat.AsObject())) ReturnFalse; //--- if(!SumVecMatrix(cPrepare.getOutput(), cAdaptFlow.getOutput(), cUpdatedFlow.getOutput(), hidden_dimension, 1, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronSpikeSuperKernelBlock::feedForward(cUpdatedFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronSpikeSuperKernelBlock::calcInputGradients(cUpdatedFlow.AsObject())) ReturnFalse; if(!DeActivation(cAdaptFlow.getOutput(), cAdaptFlow.getGradient(), cUpdatedFlow.getGradient(), cAdaptFlow.Activation())) ReturnFalse; if(!cConcat.CalcHiddenGradients(cAdaptFlow.AsObject())) ReturnFalse; //---Tail-Aware if(!DeActivation(cTailAwareMoE.getOutput(), cTailAwareMoE.getGradient(), cUpdatedFlow.getGradient(), cTailAwareMoE.Activation())) ReturnFalse; if(!cLTR.CalcHiddenGradients(cTailAwareMoE.AsObject())) ReturnFalse; //--- HTR if(!DeActivation(cHTR.getOutput(), cHTR.getGradient(), cUpdatedFlow.getGradient(), cHTR.Activation())) ReturnFalse; if(!cContext[-1] || !cMFEvsUpLTR.CalcHiddenGradients(cHTR.AsObject(), cContext[-1].getOutput(), cContext[-1].getGradient(), (ENUM_ACTIVATION)cContext[-1].Activation())) ReturnFalse; //--- MFE uint hidden_dimension = cLTR.GetWindow(); uint ltr_forecat = cLTR.GetUnits(); uint htr_points_to_segment = cUpsampleLTR.Neurons() / cLTR.Neurons(); //--- if(!cMFE[-1] || !DeConcat(cMFE[-1].getGradient(), cUpsampleLTR.getGradient(), cMFEvsUpLTR.getGradient(), hidden_dimension, hidden_dimension, ltr_forecat * htr_points_to_segment)) ReturnFalse; for(int i = cMFE.Total() - 2; i >= 0; i--) { if(!cMFE[i] || !cMFE[i].CalcHiddenGradients(cMFE[i + 1])) ReturnFalse; } if(!cUpsampleLTR.CalcHiddenGradients(cMFE[0])) ReturnFalse; //--- LTR if(!DeActivation(cUpsampleLTR.getOutput(), cUpsampleLTR.getGradient(), cUpdatedFlow.getGradient(), cUpsampleLTR.Activation())) ReturnFalse; if(!LinearUpsampleGrad(cLTR.AsObject(), cUpsampleLTR.AsObject(), htr_points_to_segment, ltr_forecat, hidden_dimension)) ReturnFalse; Deactivation(cLTR); CNeuronBaseOCL* next = cLTR.AsObject(); for(int i = cContext.Total() - 1; i >= 0; i--) { if(!cContext[i] || !cContext[i].CalcHiddenGradients(next)) ReturnFalse; next = cContext[i]; } //--- if(!cPrepare.CalcHiddenGradients(next)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cPrepare.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cPrepare.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cPrepare.AsObject(); CNeuronBaseOCL*curr = NULL; for(int i = 0; i < cContext.Total(); i++) { curr = cContext[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cLTR.UpdateInputWeights(prev)) ReturnFalse; //--- prev = cUpsampleLTR.AsObject(); for(int i = 0; i < cMFE.Total(); i++) { curr = cMFE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cHTR.UpdateInputWeights(cMFEvsUpLTR.AsObject(), cContext[-1].getOutput())) ReturnFalse; //--- if(!cTailAwareMoE.UpdateInputWeights(cLTR.AsObject())) ReturnFalse; if(!cAdaptFlow.UpdateInputWeights(cConcat.AsObject())) ReturnFalse; //--- if(!CNeuronSpikeSuperKernelBlock::updateInputWeights(cUpdatedFlow.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::Save(const int file_handle) { if(!CNeuronResFlow::Save(file_handle)) ReturnFalse; if(!cTailAwareMoE.Save(file_handle)) ReturnFalse; if(!cAdaptFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::Load(const int file_handle) { if(!CNeuronResFlow::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cTailAwareMoE.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cAdaptFlow.AsObject())) ReturnFalse; //--- if(!cConcat.Init(0, 1, OpenCL, 4 * cLTR.GetWindow() * GetUnits(), optimization, iBatch)) ReturnFalse; cConcat.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::Clear(void) { if(!CNeuronResFlow::Clear()) ReturnFalse; if(!cTailAwareMoE.Clear()) ReturnFalse; if(!cAdaptFlow.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronResFlowLattice::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronResFlow::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronResFlowLattice* Source = source; dWeightsUpdate(cTailAwareMoE, Source, tau); dWeightsUpdate(cAdaptFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlowLattice::SetOpenCL(COpenCLMy *obj) { CNeuronResFlow::SetOpenCL(obj); cTailAwareMoE.SetOpenCL(OpenCL); cAdaptFlow.SetOpenCL(OpenCL); cConcat.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronResFlowLattice::TrainMode(bool flag) { CNeuronResFlow::TrainMode(flag); cTailAwareMoE.TrainMode(bTrain); cAdaptFlow.TrainMode(bTrain); cConcat.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CFieldAwareParams : public CNeuronBaseOCL { protected: uint iFields; CParams cCandidates; CParams cFieldEmbeddings; CLayer cScore; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { return FeedForward(); } ///\ingroup neuron_base_opt virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return UpdateInputWeights(); } virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override { return true; } public: CFieldAwareParams(void) {}; ~CFieldAwareParams(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint params_to_field, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool FeedForward(void); virtual bool UpdateInputWeights(void); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defFieldAwareParams; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetFiedls(void) const { return iFields; } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint params_to_field, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, params_to_field * fields, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cCandidates.Init(0, index, OpenCL, params_to_field * candidates, optimization, iBatch)) ReturnFalse; cCandidates.SetActivationFunction(None); index++; if(!cFieldEmbeddings.Init(0, index, OpenCL, fields * embed_size, optimization, iBatch)) ReturnFalse; cFieldEmbeddings.SetActivationFunction(SIGMOID); if(fields <= embed_size) if(!cFieldEmbeddings.Identity(fields, embed_size)) ReturnFalse; index++; cScore.Clear(); cScore.SetOpenCL(OpenCL); CNeuronConvOCL* conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, embed_size, embed_size, 4 * candidates, fields, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 4 * candidates, 4 * candidates, candidates, fields, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; CNeuronSparseSoftMax* softmax = new CNeuronSparseSoftMax(); if(!softmax || !softmax.Init(0, index, OpenCL, fields, candidates, topK, optimization, iBatch) || !cScore.Add(softmax)) DeleteObjAndFalse(softmax); iFields = fields; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::FeedForward(void) { if(!bTrain) return true; //--- if(!cCandidates.FeedForward()) ReturnFalse; if(!cFieldEmbeddings.FeedForward()) ReturnFalse; CNeuronBaseOCL* prev = cFieldEmbeddings.AsObject(); CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; uint topK = softmax.DimensionOut(); uint params_to_field = Neurons() / iFields; uint candidates = cCandidates.Neurons() / params_to_field; if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cCandidates.getOutput(), Output, iFields, topK, candidates, params_to_field)) ReturnFalse; dActivation(this); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::UpdateInputWeights(void) { //--- Backprop gradients CNeuronBaseOCL* next = cScore[-1]; CNeuronBaseOCL* curr = NULL; if(!next || next.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = next; uint topK = softmax.DimensionOut(); uint params_to_field = Neurons() / iFields; uint candidates = cCandidates.Neurons() / params_to_field; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cCandidates.getOutput(), cCandidates.getGradient(), Gradient, iFields, topK, candidates, params_to_field)) ReturnFalse; Deactivation(cCandidates); for(int i = cScore.Total() - 2; i >= 0; i--) { curr = cScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!cFieldEmbeddings.CalcHiddenGradients(next)) ReturnFalse; //--- Update weights if(!cCandidates.UpdateInputWeights()) ReturnFalse; if(!cFieldEmbeddings.UpdateInputWeights()) ReturnFalse; curr = cFieldEmbeddings.AsObject(); for(int i = 0; i < cScore.Total(); i++) { next = cScore[i]; if(!next || !next.UpdateInputWeights(curr)) ReturnFalse; curr = next; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cCandidates.Save(file_handle)) ReturnFalse; if(!cFieldEmbeddings.Save(file_handle)) ReturnFalse; if(!cScore.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cCandidates.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cFieldEmbeddings.AsObject())) ReturnFalse; if(!cScore.Load(file_handle)) ReturnFalse; //--- if(!cScore[-1] || cScore[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cScore[-1]; iFields = softmax.Heads(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CFieldAwareParams::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CFieldAwareParams* Source = source; dWeightsUpdate(cCandidates, Source, tau); dWeightsUpdate(cFieldEmbeddings, Source, tau); dWeightsUpdate(cScore, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CFieldAwareParams::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cCandidates.SetOpenCL(OpenCL); cFieldEmbeddings.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CFieldAwareParams::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cCandidates.TrainMode(bTrain); cFieldEmbeddings.TrainMode(bTrain); cScore.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFieldAwareConv : public CNeuronBaseOCL { protected: CFieldAwareParams cParams; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronFieldAwareConv(void) {}; ~CNeuronFieldAwareConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronFieldAwareConv; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cParams.Neurons() / Neurons(); } virtual uint GetFiedls(void) const { return cParams.GetFiedls(); } virtual uint GetFilters(void) const { return Neurons() / GetFiedls(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint window_out, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, window_out * fields, optimization_type, batch)) ReturnFalse; if(!cParams.Init(0, 0, OpenCL, window_in * window_out, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cParams.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) if(!cParams.FeedForward()) ReturnFalse; //--- uint fields = cParams.GetFiedls(); uint window_out = Neurons() / fields; uint window_in = cParams.Neurons() / (fields * window_out); if(!NeuronOCL || NeuronOCL.Neurons() < int(window_in * fields)) ReturnFalse; if(!MatMul(NeuronOCL.getOutput(), cParams.getOutput(), Output, 1, window_in, window_out, fields, true)) ReturnFalse; dActivation(this); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint fields = cParams.GetFiedls(); uint window_out = Neurons() / fields; uint window_in = cParams.Neurons() / (fields * window_out); if(NeuronOCL.Neurons() < int(window_in * fields)) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getGradient(), cParams.getOutput(), cParams.getGradient(), Gradient, 1, window_in, window_out, fields, true)) ReturnFalse; Deactivation(NeuronOCL); Deactivation(cParams); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldAwareConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFieldAwareConv* Source = source; dWeightsUpdate(cParams, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFieldAwareConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFieldAwareConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHFAT : public CNeuronMSRes { protected: uint iHeads; //--- CNeuronFieldAwareConv cQ_Embedding; CNeuronFieldAwareConv cKV_Embedding; CParams cScale; int ibScoreIndex; CNeuronBaseOCL cMHAttentionOut; CNeuronSpikeConvBlock cW0; CNeuronBaseOCL cAttentionOut; //--- virtual bool Attention(void); virtual bool AttentionGradients(void); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override ; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMHFAT(void) {}; ~CNeuronMHFAT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMHFAT; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window, 2 * window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFiedls() * cKV_Embedding.GetFiedls() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Attention(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFiedls(); uint total_k = cKV_Embedding.GetFiedls(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, total_k, iHeads}; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFAT; setBuffer(kernel, def_k_mhfat_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_out, cMHAttentionOut.getOutputIndex()) setArgument(kernel, def_k_mhfat_dimension, dimension) setArgument(kernel, def_k_mhfat_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cMHAttentionOut.getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::AttentionGradients(void) { if(!OpenCL || ibScoreIndex < 0) ReturnFalse; //--- uint total_q = cQ_Embedding.GetFiedls(); uint total_k = cKV_Embedding.GetFiedls(); uint dimension = cQ_Embedding.Neurons() / (total_q * iHeads); //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { total_q, (uint)MathMin(MathMax(total_k, MathMax(total_q, dimension)), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFATGrad; setBuffer(kernel, def_k_mhfat_gr_q, cQ_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_q_gr, cQ_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_kv, cKV_Embedding.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_kv_gr, cKV_Embedding.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scale, cScale.getOutputIndex()) setBuffer(kernel, def_k_mhfat_gr_scale_gr, cScale.getGradientIndex()) setBuffer(kernel, def_k_mhfat_gr_scores, ibScoreIndex) setBuffer(kernel, def_k_mhfat_gr_gradients, cMHAttentionOut.getGradientIndex()) setArgument(kernel, def_k_mhfat_gr_dimension, dimension) setArgument(kernel, def_k_mhfat_gr_total_k, total_k) setArgument(kernel, def_k_mhfat_gr_mask_future, 0) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!cKV_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cQ_Embedding.getGradient().BufferRead()) ReturnFalse; if(!cScale.getGradient().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::calcInputGradients(CNeuronBaseOCL *prevLayer) { if(!prevLayer) ReturnFalse; //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!prevLayer.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Save(const int file_handle) { if(!CNeuronMSRes::Save(file_handle)) ReturnFalse; //--- if(FileWriteInteger(file_handle, int(iHeads), INT_VALUE) < INT_VALUE) ReturnFalse; if(!cQ_Embedding.Save(file_handle)) ReturnFalse; if(!cKV_Embedding.Save(file_handle)) ReturnFalse; if(!cScale.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::Load(const int file_handle) { if(!CNeuronMSRes::Load(file_handle)) ReturnFalse; //--- if(FileIsEnding(file_handle)) ReturnFalse; iHeads = (uint)FileReadInteger(file_handle); if(!LoadInsideLayer(file_handle, cQ_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKV_Embedding.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScale.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cW0.AsObject())) ReturnFalse; //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; uint index = 3; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index += 2; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHFAT::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMSRes::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHFAT* Source = source; dWeightsUpdate(cQ_Embedding, Source, tau); dWeightsUpdate(cKV_Embedding, Source, tau); dWeightsUpdate(cScale, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFAT::SetOpenCL(COpenCLMy *obj) { if(!!OpenCL && ibScoreIndex >= 0) { OpenCL.BufferFree(ibScoreIndex); ibScoreIndex = INVALID_HANDLE; } //--- CNeuronMSRes::SetOpenCL(obj); //--- cQ_Embedding.SetOpenCL(OpenCL); cKV_Embedding.SetOpenCL(OpenCL); cScale.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cMHAttentionOut.SetOpenCL(OpenCL); cAttentionOut.SetOpenCL(OpenCL); //--- ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHFAT::TrainMode(bool flag) { CNeuronMSRes::TrainMode(flag); //--- cQ_Embedding.TrainMode(bTrain); cKV_Embedding.TrainMode(bTrain); cScale.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHCrossFAT : public CNeuronMHFAT { protected: CNeuronBaseOCL cContext; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; //--- virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; public: CNeuronMHCrossFAT(void) {}; ~CNeuronMHCrossFAT(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint window_cross, uint fields_cross, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defNeuronMHCrossFAT; } //--- methods for working with files virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint window_key, uint heads, uint fields, uint window_cross, uint fields_cross, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronMSRes::Init(numOutputs, myIndex, open_cl, fields, window, 1, optimization_type, batch)) ReturnFalse; //--- iHeads = heads; uint index = 0; if(!cQ_Embedding.Init(0, index, OpenCL, window, window_key * iHeads, fields, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cQ_Embedding.SetActivationFunction(SoftPlus); index++; if(!cKV_Embedding.Init(0, index, OpenCL, window_cross, 2 * window_key * iHeads, fields_cross, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; cKV_Embedding.SetActivationFunction(SoftPlus); index++; if(!cScale.Init(0, index, OpenCL, cQ_Embedding.GetFiedls() * cKV_Embedding.GetFiedls() * iHeads, optimization, iBatch)) ReturnFalse; cScale.SetActivationFunction(TANH); ibScoreIndex = OpenCL.AddBuffer(sizeof(float) * cScale.Neurons(), CL_MEM_READ_WRITE); if(ibScoreIndex == INVALID_HANDLE) ReturnFalse; index++; if(!cMHAttentionOut.Init(0, index, OpenCL, cQ_Embedding.Neurons(), optimization, iBatch)) ReturnFalse; cMHAttentionOut.SetActivationFunction(None); index++; if(!cW0.Init(0, index, OpenCL, window_key * iHeads, window_key * iHeads, window, fields, 1, optimization, iBatch)) ReturnFalse; index++; if(!cAttentionOut.Init(0, index, OpenCL, cW0.Neurons(), optimization, iBatch)) ReturnFalse; //--- index++; if(!cContext.Init(0, index, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFiedls(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(cContext.getOutput() != Context) if(!cContext.SetOutput(Context, true)) ReturnFalse; //--- if(!cQ_Embedding.FeedForward(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.FeedForward(cContext.AsObject())) ReturnFalse; if(bTrain) if(!cScale.FeedForward()) ReturnFalse; if(!Attention()) ReturnFalse; if(!cW0.FeedForward(cMHAttentionOut.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), cW0.getOutput(), cAttentionOut.getOutput(), cW0.GetFilters(), true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMSRes::feedForward(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondGradient) ReturnFalse; if(cContext.getGradient() != SecondGradient) if(!cContext.SetGradient(SecondGradient, true)) ReturnFalse; cContext.SetActivationFunction(SecondActivation); //--- if(!CNeuronMSRes::calcInputGradients(cAttentionOut.AsObject())) ReturnFalse; if(!DeActivation(cW0.getOutput(), cW0.getGradient(), cAttentionOut.getGradient(), cW0.Activation())) ReturnFalse; if(!cMHAttentionOut.CalcHiddenGradients(cW0.AsObject())) ReturnFalse; if(!AttentionGradients()) ReturnFalse; //--- if(!prevLayer.CalcHiddenGradients(cQ_Embedding.AsObject())) ReturnFalse; if(prevLayer.Activation() != None) if(!DeActivation(prevLayer.getOutput(), cAttentionOut.getGradient(), cAttentionOut.getGradient(), prevLayer.Activation())) ReturnFalse; if(!SumAndNormilize(prevLayer.getGradient(), cAttentionOut.getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; if(!cContext.CalcHiddenGradients(cKV_Embedding.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { //--- if(!cQ_Embedding.UpdateInputWeights(NeuronOCL)) ReturnFalse; if(!cKV_Embedding.UpdateInputWeights(cContext.AsObject())) ReturnFalse; if(!cScale.UpdateInputWeights()) ReturnFalse; if(!cW0.UpdateInputWeights(cMHAttentionOut.AsObject())) ReturnFalse; if(!CNeuronMSRes::updateInputWeights(cAttentionOut.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHCrossFAT::Load(const int file_handle) { if(!CNeuronMHFAT::Load(file_handle)) ReturnFalse; if(!cContext.Init(0, 6, OpenCL, cKV_Embedding.GetWindow()*cKV_Embedding.GetFiedls(), optimization, iBatch)) ReturnFalse; cContext.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHCrossFAT::SetOpenCL(COpenCLMy *obj) { CNeuronMHFAT::SetOpenCL(obj); cContext.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronFieldPatternEmbedding : public CNeuronBaseOCL { protected: uint iFields; uint iWindow; CParams cCandidates; CLayer cScore; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronFieldPatternEmbedding(void) {}; ~CNeuronFieldPatternEmbedding(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronFieldPatternEmbedding; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetFiedls(void) const { return iFields; } virtual CBufferFloat *getWeights(void) override { return (!!cScore[-2] ? ((CNeuronConvOCL*)cScore[-2]).GetWeightsConv() : NULL); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint fields, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, embed_size * fields, optimization_type, batch)) ReturnFalse; //--- uint index = 0; if(!cCandidates.Init(0, index, OpenCL, embed_size * candidates, optimization, iBatch)) ReturnFalse; cCandidates.SetActivationFunction(None); index++; cScore.Clear(); cScore.SetOpenCL(OpenCL); CNeuronConvOCL* conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, window, window, 4 * candidates, fields, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; conv = new CNeuronConvOCL(); if(!conv || !conv.Init(0, index, OpenCL, 4 * candidates, 4 * candidates, candidates, fields, 1, optimization, iBatch) || !cScore.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SoftPlus); index++; CNeuronSparseSoftMax* softmax = new CNeuronSparseSoftMax(); if(!softmax || !softmax.Init(0, index, OpenCL, fields, candidates, topK, optimization, iBatch) || !cScore.Add(softmax)) DeleteObjAndFalse(softmax); iFields = fields; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::feedForward(CNeuronBaseOCL *NeuronOCL) { if(bTrain) if(!cCandidates.FeedForward()) ReturnFalse; CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; uint topK = softmax.DimensionOut(); uint params_to_field = Neurons() / iFields; uint candidates = cCandidates.Neurons() / params_to_field; if(!SparseMatMul(softmax.GetIndexes(), softmax.getOutput(), cCandidates.getOutput(), Output, iFields, topK, candidates, params_to_field)) ReturnFalse; dActivation(this); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; CNeuronBaseOCL* next = cScore[-1]; CNeuronBaseOCL* curr = NULL; if(!next || next.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = next; uint topK = softmax.DimensionOut(); uint params_to_field = Neurons() / iFields; uint candidates = cCandidates.Neurons() / params_to_field; if(!SparseMatMulGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cCandidates.getOutput(), cCandidates.getGradient(), Gradient, iFields, topK, candidates, params_to_field)) ReturnFalse; Deactivation(cCandidates); for(int i = cScore.Total() - 2; i >= 0; i--) { curr = cScore[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cCandidates.FeedForward()) ReturnFalse; CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cScore.Total(); i++) { curr = cScore[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cCandidates.Save(file_handle)) ReturnFalse; if(!cScore.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cCandidates.AsObject())) ReturnFalse; if(!cScore.Load(file_handle)) ReturnFalse; //--- if(!cScore[-1] || cScore[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cScore[-1]; iFields = softmax.Heads(); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronFieldPatternEmbedding::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronFieldPatternEmbedding* Source = source; dWeightsUpdate(cCandidates, Source, tau); dWeightsUpdate(cScore, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFieldPatternEmbedding::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cCandidates.SetOpenCL(OpenCL); cScore.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronFieldPatternEmbedding::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cCandidates.TrainMode(bTrain); cScore.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronAutoToken : public CNeuronBaseOCL { protected: uint iEmbedingSize; uint iTopK; uint iUnits; uint iGroups; //--- CLayer cDNN; CLayer cW; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronAutoToken(void) {}; ~CNeuronAutoToken(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint embed_size, uint units_in, uint groups, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronAutoToken; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint embed_size, uint units_in, uint groups, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, embed_size * topK * groups, optimization_type, batch)) ReturnFalse; activation = None; //--- iEmbedingSize = embed_size; iTopK = topK; iUnits = units_in; iGroups = groups; //--- cDNN.Clear(); cW.Clear(); cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); //--- CNeuronFieldAwareConv* conv = NULL; CFieldAwareParams* params = NULL; CNeuronSparseSoftMax* softmax = NULL; uint index = 0; //--- DNN conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, window_in, 2 * embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, 2 * embed_size, embed_size, units_in, embed_size, candidates, (topK + 2) / 3, optimization, iBatch) || !cDNN.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); //--- W index++; params = new CFieldAwareParams(); if(!params || !params.Init(0, index, OpenCL, units_in, groups, (embed_size + 3) / 4, candidates, (topK + 2) / 3, optimization, iBatch) || !cW.Add(params)) DeleteObjAndFalse(params); params.SetActivationFunction(None); index++; softmax = new CNeuronSparseSoftMax(); if(!softmax || !softmax.Init(0, index, OpenCL, groups, units_in, topK, optimization, iBatch) || !cW.Add(softmax)) DeleteObjAndFalse(softmax); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).FeedForward()) ReturnFalse; } else if(!curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(prev.Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = prev; if(!SparseConcatenate(softmax.GetIndexes(), softmax.getOutput(), cDNN[-1].getOutput(), Output, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || !cW[-1] || !cDNN[-1] || cW[-1].Type() != defNeuronSparseSoftMax) ReturnFalse; CNeuronSparseSoftMax* softmax = cW[-1]; if(!SparseConcatenateGrad(softmax.GetIndexes(), softmax.getOutput(), softmax.getGradient(), cDNN[-1].getOutput(), cDNN[-1].getGradient(), Gradient, iGroups, iTopK, iUnits, iEmbedingSize)) ReturnFalse; Deactivation(cDNN[-1]); //--- CNeuronBaseOCL* next = cDNN[-1]; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = cDNN.Total() - 2; i >= 0; i--) { curr = cDNN[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- W next = cW[-1]; for(int i = cW.Total() - 2; i >= 0; i--) { curr = cW[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- DNN for(int i = 0; i < cDNN.Total(); i++) { curr = cDNN[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- W for(int i = 0; i < cW.Total(); i++) { curr = cW[i]; if(!curr) ReturnFalse; if(curr.Type() == defFieldAwareParams) { if(!((CFieldAwareParams*)curr).UpdateInputWeights()) ReturnFalse; } else if(!curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iEmbedingSize); dFileWriteUInt(file_handle, iTopK); dFileWriteUInt(file_handle, iUnits); dFileWriteUInt(file_handle, iGroups); //--- if(!cDNN.Save(file_handle)) ReturnFalse; if(!cW.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iEmbedingSize); dFileReadUInt(file_handle, iTopK); dFileReadUInt(file_handle, iUnits); dFileReadUInt(file_handle, iGroups); //--- if(!cDNN.Load(file_handle)) ReturnFalse; if(!cW.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronAutoToken* Source = source; dWeightsUpdate(cDNN, Source, tau); dWeightsUpdate(cW, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronAutoToken::Clear(void) { if(!CNeuronBaseOCL::Clear()) ReturnFalse; //--- if(!cDNN.ClearStates()) ReturnFalse; if(!cW.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAutoToken::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cDNN.SetOpenCL(OpenCL); cW.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronAutoToken::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cDNN.TrainMode(bTrain); cW.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMultiMixAttention : public CNeuronBaseOCL { protected: CFieldAwareParams cParams; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMultiMixAttention(void) {}; ~CNeuronMultiMixAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronMultiMixAttention; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; //--- virtual uint GetHeads(void) { return cParams.GetFiedls(); } virtual uint GetUnits(void) { return uint(MathSqrt(cParams.Neurons() / GetHeads())); } virtual uint GetDimension(void) { return Neurons() / GetUnits(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(dimension < heads || dimension % heads > 0) ReturnFalse; if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, units * dimension, optimization_type, batch)) ReturnFalse; activation = None; if(!cParams.Init(0, 0, OpenCL, units * units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- if(bTrain) if(!cParams.FeedForward()) ReturnFalse; //--- uint heads = cParams.GetFiedls(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMul(cParams.getOutput(), NeuronOCL.getOutput(), Output, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getOutput(), Output, Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL || NeuronOCL.Neurons() < Neurons()) ReturnFalse; //--- uint heads = cParams.GetFiedls(); uint units = uint(MathSqrt(cParams.Neurons() / heads)); uint dimension = Neurons() / units; uint head_dimension = dimension / heads; //--- if(!MatMulGrad(cParams.getOutput(), cParams.getGradient(), NeuronOCL.getOutput(), NeuronOCL.getGradient(), Gradient, units, units, head_dimension, heads, true)) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), Gradient, NeuronOCL.getGradient(), dimension, false, 0, 0, 0, 1)) ReturnFalse; Deactivation(NeuronOCL); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMultiMixAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; dWeightsUpdate(cParams, ((CNeuronMultiMixAttention*)source), tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMultiMixAttention::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSparseMoEConv : public CNeuronBaseOCL { protected: CNeuronFieldPatternEmbedding cParams; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSparseMoEConv(void) {}; ~CNeuronSparseMoEConv(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint filters, uint units, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSparseMoEConv; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual uint GetWindow(void) const { return cParams.Neurons() / Neurons(); } virtual uint GetFiedls(void) const { return cParams.GetFiedls(); } virtual uint GetFilters(void) const { return Neurons() / GetFiedls(); } virtual CBufferFloat *getWeights(void) override { return cParams.getWeights(); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint filters, uint units, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, filters * units, optimization_type, batch)) ReturnFalse; if(!cParams.Init(0, 0, OpenCL, window, units, window * filters, candidates, topK, optimization, iBatch)) ReturnFalse; cParams.SetActivationFunction(None); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cParams.FeedForward(NeuronOCL)) ReturnFalse; //--- uint units = cParams.GetFiedls(); uint filters = Neurons() / units; uint window = cParams.Neurons() / (filters * units); if(!MatMul(NeuronOCL.getOutput(), cParams.getOutput(), Output, 1, window, filters, units, true)) ReturnFalse; dActivation(this); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- uint units = cParams.GetFiedls(); uint filters = Neurons() / units; uint window = cParams.Neurons() / (filters * units); if(NeuronOCL.Neurons() < int(window * units)) ReturnFalse; if(!MatMulGrad(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), cParams.getOutput(), cParams.getGradient(), Gradient, 1, window, filters, units, true)) ReturnFalse; Deactivation(cParams); if(NeuronOCL.Activation() != None) if(!DeActivation(NeuronOCL.getOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.getPrevOutput(), NeuronOCL.Activation())) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cParams.AsObject())) ReturnFalse; if(!SumAndNormilize(NeuronOCL.getGradient(), NeuronOCL.getPrevOutput(), NeuronOCL.getGradient(), window, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cParams.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; if(!cParams.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cParams.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSparseMoEConv::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSparseMoEConv* Source = source; dWeightsUpdate(cParams, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseMoEConv::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); cParams.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSparseMoEConv::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); cParams.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMTmixAttBlock : public CNeuronBaseOCL { protected: CNeuronMultiMixAttention cAttention; CLayer cSharedExpert; CLayer cFieldMoE; CLayer cSceneMoE; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMTmixAttBlock(void) {}; ~CNeuronMTmixAttBlock(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronMTmixAttBlock; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint heads, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; if(!cAttention.Init(0, index, OpenCL, dimension, units, heads, embed_size, candidates, topK, optimization, iBatch)) ReturnFalse; //--- cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint expert_dimension = dimension / heads; uint hidden_dimension = MathMax((expert_dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, expert_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, expert_dimension, hidden_dimension, windows, steps, heads, units, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, expert_dimension, units * heads, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::feedForward(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.FeedForward(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- uint dimension = cAttention.GetDimension(); if(!SumAndNormilize(cSharedExpert[-1].getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormilize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormilize(Output, cAttention.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(cAttention.getOutput(), PrevOutput, Gradient, cAttention.Activation())) ReturnFalse; uint dimension = cAttention.GetDimension(); if(!cAttention.CalcHiddenGradients(cSharedExpert[0]) || !SumAndNormilize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormilize(cAttention.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!cAttention.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormilize(cAttention.getGradient(), PrevOutput, cAttention.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- if(!NeuronOCL.CalcHiddenGradients(cAttention.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { if(!cAttention.UpdateInputWeights(NeuronOCL)) ReturnFalse; //--- CNeuronBaseOCL* prev = cAttention.AsObject(); CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = cAttention.AsObject(); for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = cAttention.AsObject(); for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cAttention.Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!LoadInsideLayer(file_handle, cAttention.AsObject())) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMTmixAttBlock::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMTmixAttBlock* Source = source; dWeightsUpdate(cAttention, Source, tau); dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cAttention.SetOpenCL(OpenCL); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMTmixAttBlock::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cAttention.TrainMode(bTrain); cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CCrossMHFlashAttention : public CNeuronBaseOCL { protected: uint iQUnits; uint iKVUnits; uint iHeads; uint iDimension; bool bMask; CBufferFloat cLogSumExp; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { return true; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override { return true; } public: CCrossMHFlashAttention(void) {}; ~CCrossMHFlashAttention(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint heads, bool mask_future, uint q_units, uint kv_units, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual int Type(void) const { return defCrossMHFlashAttention; } //--- methods for working with files virtual bool Save(int const file_handle) override; virtual bool Load(int const file_handle) override; //--- virtual void SetOpenCL(COpenCLMy *obj) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint heads, bool mask_future, uint q_units, uint kv_units, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * heads * q_units, optimization_type, batch)) ReturnFalse; activation = None; //--- if(kv_units < 1) ReturnFalse; //--- iQUnits = q_units; iKVUnits = kv_units; iHeads = heads; iDimension = dimension; bMask = mask_future; //--- if(!cLogSumExp.BufferInit(q_units * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iKVUnits, iDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttention; setBuffer(kernel, def_k_mhflat_q, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhflat_kv, Context.GetIndex()) setBuffer(kernel, def_k_mhflat_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflat_out, getOutputIndex()) setArgument(kernel, def_k_mhflat_dimension, iDimension) setArgument(kernel, def_k_mhflat_total_kv, iKVUnits) setArgument(kernel, def_k_mhflat_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) #ifdef _DEBUG if(!Output.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !prevLayer.getOutput() || !prevLayer.getGradient() || !SecondInput || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iKVUnits), iDimension, iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashAttentionGrad; setBuffer(kernel, def_k_mhflatg_q, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhflatg_q_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhflatg_kv, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhflatg_kv_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhflatg_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhflatg_out, getOutputIndex()) setBuffer(kernel, def_k_mhflatg_out_gr, getGradientIndex()) setArgument(kernel, def_k_mhflatg_dimension, iDimension) setArgument(kernel, def_k_mhflatg_total_q, iQUnits) setArgument(kernel, def_k_mhflatg_total_kv, iKVUnits) setArgument(kernel, def_k_mhflatg_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; #ifdef _DEBUG if(!prevLayer.getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iKVUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CCrossMHFlashAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iKVUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iDimension); dFileReadUInt(file_handle, bMask); //--- //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CCrossMHFlashAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronBaseOCL::SetOpenCL(obj); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMixFeedForward : public CNeuronBaseOCL { protected: CLayer cSharedExpert; CLayer cFieldMoE; CLayer cSceneMoE; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronMixFeedForward(void) {}; ~CNeuronMixFeedForward(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronMixFeedForward; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; virtual void SetActivationFunction(ENUM_ACTIVATION value) override {}; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual CBufferFloat *getWeights(void) override { return (!!cSceneMoE[-1] ? cSceneMoE[-1].getWeights() : NULL); } }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension, uint units, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { if(!CNeuronBaseOCL::Init(numOutputs, myIndex, open_cl, dimension * units, optimization_type, batch)) ReturnFalse; activation = None; //--- uint index = 0; cSharedExpert.Clear(); cFieldMoE.Clear(); cSceneMoE.Clear(); cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); uint hidden_dimension = MathMax((dimension + 3) / 4, 8); //--- Shared Expert index++; CNeuronFieldAwareConv* conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, dimension, hidden_dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(SIGMOID); index++; conv = new CNeuronFieldAwareConv(); if(!conv || !conv.Init(0, index, OpenCL, hidden_dimension, dimension, units, embed_size, candidates, topK, optimization, iBatch) || !cSharedExpert.Add(conv)) DeleteObjAndFalse(conv); conv.SetActivationFunction(TANH); //--- Field MoE index++; CNeuronSparseMoEConv* moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cFieldMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- Scene MoE index++; uint windows[] = {3, 5, 7}; uint steps[] = {1, 1, 1}; CNeuronSpikeSuperKernelBlock* scene = new CNeuronSpikeSuperKernelBlock(); if(!scene || !scene.Init(0, index, OpenCL, dimension, hidden_dimension, windows, steps, units, 1, optimization, iBatch) || !cSceneMoE.Add(scene)) DeleteObjAndFalse(scene); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, hidden_dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(SIGMOID); index++; moe = new CNeuronSparseMoEConv(); if(!moe || !moe.Init(0, index, OpenCL, hidden_dimension, dimension, units, candidates, topK, optimization, iBatch) || !cSceneMoE.Add(moe)) DeleteObjAndFalse(moe); moe.SetActivationFunction(TANH); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::feedForward(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- CNeuronFieldAwareConv* conv = cSharedExpert[-1]; uint dimension = conv.GetFilters(); if(!SumAndNormilize(conv.getOutput(), cFieldMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormilize(Output, cSceneMoE[-1].getOutput(), Output, dimension, false, 0, 0, 0, 1) || !SumAndNormilize(Output, NeuronOCL.getOutput(), Output, dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- Scene MoE CNeuronBaseOCL* next = cSceneMoE[-1]; CNeuronBaseOCL* curr = NULL; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSceneMoE.Total() - 2; i >= 0; i--) { curr = cSceneMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Field MoE next = cFieldMoE[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cFieldMoE.Total() - 2; i >= 0; i--) { curr = cFieldMoE[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Shared Expert next = cSharedExpert[-1]; if(!next || !DeActivation(next.getOutput(), next.getGradient(), Gradient, next.Activation())) ReturnFalse; for(int i = cSharedExpert.Total() - 2; i >= 0; i--) { curr = cSharedExpert[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- Comulative to Attention if(!DeActivation(NeuronOCL.getOutput(), PrevOutput, Gradient, NeuronOCL.Activation())) ReturnFalse; CNeuronFieldAwareConv* conv = cSharedExpert[0]; uint dimension = conv.GetWindow(); if(!NeuronOCL.CalcHiddenGradients(conv) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cFieldMoE[0]) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, PrevOutput, dimension, false, 0, 0, 0, 1)) ReturnFalse; if(!NeuronOCL.CalcHiddenGradients(cSceneMoE[0]) || !SumAndNormilize(NeuronOCL.getGradient(), PrevOutput, NeuronOCL.getGradient(), dimension, true, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { //--- CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Shared Expert for(int i = 0; i < cSharedExpert.Total(); i++) { curr = cSharedExpert[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Field MoE prev = NeuronOCL; for(int i = 0; i < cFieldMoE.Total(); i++) { curr = cFieldMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Scene MoE prev = NeuronOCL; for(int i = 0; i < cSceneMoE.Total(); i++) { curr = cSceneMoE[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Save(const int file_handle) { if(!CNeuronBaseOCL::Save(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Save(file_handle)) ReturnFalse; if(!cFieldMoE.Save(file_handle)) ReturnFalse; if(!cSceneMoE.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::Load(const int file_handle) { if(!CNeuronBaseOCL::Load(file_handle)) ReturnFalse; //--- if(!cSharedExpert.Load(file_handle)) ReturnFalse; if(!cFieldMoE.Load(file_handle)) ReturnFalse; if(!cSceneMoE.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMixFeedForward::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronBaseOCL::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMixFeedForward* Source = source; dWeightsUpdate(cSharedExpert, Source, tau); dWeightsUpdate(cFieldMoE, Source, tau); dWeightsUpdate(cSceneMoE, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::SetOpenCL(COpenCLMy *obj) { CNeuronBaseOCL::SetOpenCL(obj); //--- cSharedExpert.SetOpenCL(OpenCL); cFieldMoE.SetOpenCL(OpenCL); cSceneMoE.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMixFeedForward::TrainMode(bool flag) { CNeuronBaseOCL::TrainMode(flag); //--- cSharedExpert.TrainMode(bTrain); cFieldMoE.TrainMode(bTrain); cSceneMoE.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronOneTrans : public CNeuronMixFeedForward { protected: CLayer cPrepare; CNeuronBaseOCL cSequenceLast; CNeuronSparseMoEConv cKVSequenceLast; CNeuronAddToStack cStackSequence; CNeuronAddToStack cStackKVSequence; CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronOneTrans(void) {}; ~CNeuronOneTrans(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronOneTrans; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; //--- if(!CNeuronMixFeedForward::Init(numOutputs, myIndex, open_cl, dimensions[count - 1], units_out, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; CNeuronBatchNormOCL* norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; CNeuronMultiWindowsConvOCL* mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(None); index++; CNeuronFieldPatternEmbedding* emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(SIGMOID); //--- index++; if(!cSequenceLast.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cSequenceLast.SetActivationFunction(None); index++; if(!cKVSequenceLast.Init(0, index, OpenCL, embed_size, 2 * embed_size * heads, units_s, candidates, topK, optimization, iBatch)) ReturnFalse; cKVSequenceLast.SetActivationFunction(SIGMOID); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cStackKVSequence.Init(0, index, OpenCL, stack_size, 2 * embed_size * heads, units_s, optimization, iBatch)) ReturnFalse; index++; //--- uint units_ns = windows.Size() - units_s; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, (units_ns + stack_size)*embed_size, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; //--- CNeuronAutoToken* select = NULL; CNeuronSparseMoEConv* query = NULL; CCrossMHFlashAttention* attention = NULL; CNeuronSpikeConv* W0 = NULL; CNeuronMTmixAttBlock* ff = NULL; uint units_in = units_ns + stack_size; for(uint i = 0; i < layers; i++) { select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, embed_size, units_in, (layers - i)*units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; units_in = (layers - i) * units_out; query = new CNeuronSparseMoEConv(); if(!query || !query.Init(0, index, OpenCL, embed_size * topK, embed_size * heads, units_in, candidates, topK, optimization, iBatch) || !cFlow.Add(query)) DeleteObjAndFalse(query); query.SetActivationFunction(SIGMOID); index++; attention = new CCrossMHFlashAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, heads, false, units_in, stack_size, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; W0 = new CNeuronSpikeConv(); if(!W0 || !W0.Init(0, index, OpenCL, embed_size * heads, embed_size * heads, embed_size, units_in, 1, optimization, iBatch) || !cFlow.Add(W0)) DeleteObjAndFalse(W0); index++; ff = new CNeuronMTmixAttBlock(); if(!ff || !ff.Init(0, index, OpenCL, embed_size, units_in, heads, embed_size, candidates, topK, optimization, iBatch) || !cFlow.Add(ff)) DeleteObjAndFalse(ff); index++; } //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!DeConcat(cSequenceLast.getOutput(), prev.getPrevOutput(), prev.getOutput(), cSequenceLast.Neurons(), prev.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.FeedForward(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.FeedForward(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(curr.getPrevOutput(), cStackSequence.getOutput(), prev.getOutput(), prev.Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.FeedForward(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) ReturnFalse; //--- if(!CNeuronMixFeedForward::calcInputGradients(cFlow[-1])) ReturnFalse; //--- CBufferFloat* grad = cStackKVSequence.getGradient(); if(!grad.Fill(0)) ReturnFalse; CBufferFloat* out = cStackKVSequence.getOutput(); CBufferFloat* temp = cStackKVSequence.getPrevOutput(); ENUM_ACTIVATION act = (ENUM_ACTIVATION)cStackKVSequence.Activation(); CNeuronBaseOCL* curr = NULL; for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr || !curr.CalcHiddenGradients(cFlow[i + 1], out, temp, act)) ReturnFalse; if(cFlow[i + 1].Type() == defCrossMHFlashAttention) if(!SumAndNormilize(temp, grad, grad, 1, false, 0, 0, 0, 1)) ReturnFalse; } //--- curr = cPrepare[-1]; if(!curr || !DeConcat(curr.getPrevOutput(), cStackSequence.getGradient(), cFlow[0].getGradient(), cFlow[0].Neurons() - cSequenceLast.Neurons(), cStackSequence.Neurons(), 1)) ReturnFalse; if(!cKVSequenceLast.CalcHiddenGradients(cStackKVSequence.AsObject())) ReturnFalse; if(!cSequenceLast.CalcHiddenGradients(cKVSequenceLast.AsObject())) ReturnFalse; temp = cSequenceLast.getGradient(); if(!cSequenceLast.SetGradient(cSequenceLast.getPrevOutput(), false) || !cSequenceLast.CalcHiddenGradients(cStackSequence.AsObject()) || !SumAndNormilize(temp, cSequenceLast.getGradient(), temp, 1, false, 0, 0, 0, 1) || !cSequenceLast.SetGradient(temp, false)) ReturnFalse; if(!Concat(cSequenceLast.getGradient(), curr.getPrevOutput(), curr.getGradient(), cSequenceLast.Neurons(), curr.Neurons() - cSequenceLast.Neurons(), 1)) ReturnFalse; Deactivation(curr); for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(cPrepare[i + 1])) ReturnFalse; } //--- if(!NeuronOCL.CalcHiddenGradients(curr)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!cKVSequenceLast.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackSequence.UpdateInputWeights(cSequenceLast.AsObject())) ReturnFalse; if(!cStackKVSequence.UpdateInputWeights(cKVSequenceLast.AsObject())) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr || !curr.UpdateInputWeights(prev, cStackKVSequence.getOutput())) ReturnFalse; prev = curr; } if(!CNeuronMixFeedForward::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMixFeedForward::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronOneTrans* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cKVSequenceLast, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cStackKVSequence, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Save(const int file_handle) { //--- if(!CNeuronMixFeedForward::Save(file_handle)) ReturnFalse; //--- if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; if(!cSequenceLast.Save(file_handle)) ReturnFalse; if(!cKVSequenceLast.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cStackKVSequence.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Load(const int file_handle) { //--- if(!CNeuronMixFeedForward::Load(file_handle)) ReturnFalse; //--- if(!cPrepare.Load(file_handle)) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cKVSequenceLast.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackKVSequence.AsObject())) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronOneTrans::Clear(void) { //--- if(!CNeuronMixFeedForward::Clear()) ReturnFalse; //--- if(!cPrepare.ClearStates()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; if(!cSequenceLast.Clear()) ReturnFalse; if(!cKVSequenceLast.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cStackKVSequence.Clear()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOneTrans::SetOpenCL(COpenCLMy *obj) { CNeuronMixFeedForward::SetOpenCL(obj); //--- cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); cSequenceLast.SetOpenCL(OpenCL); cKVSequenceLast.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cStackKVSequence.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronOneTrans::TrainMode(bool flag) { CNeuronMixFeedForward::TrainMode(flag); //--- cPrepare.TrainMode(bTrain); cFlow.TrainMode(bTrain); cSequenceLast.TrainMode(bTrain); cKVSequenceLast.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cStackKVSequence.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronMHTHCrossAttention : public CNeuronMTmixAttBlock { protected: uint iQUnits; uint iXUnits; uint iHeads; uint iXDimension; bool bMask; //--- CBufferFloat cLogSumExp; CLayer cPrepareQ; CLayer cW0; //--- virtual bool AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context); virtual bool AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None); //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) override; virtual bool calcInputGradients(CNeuronBaseOCL *prevLayer) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override { ReturnFalse; } virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) override; public: CNeuronMHTHCrossAttention(void) {}; ~CNeuronMHTHCrossAttention(void) {}; //--- //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronMHTHCrossAttention; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint dimension_q, uint units_q, uint heads, uint dimension_x, uint unit_x, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Init(numOutputs, myIndex, open_cl, dimension_q, units_q, heads, embed_size, candidates, topK, optimization_type, batch)) ReturnFalse; //--- iQUnits = units_q; iXUnits = unit_x; iHeads = heads; iXDimension = dimension_x; bMask = false; //--- cPrepareQ.Clear(); cW0.Clear(); cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); //--- Query uint index = 0; uint head_size = (dimension_q + heads - 1) / heads; CNeuronSpikeConvBlock* conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_q, dimension_q, heads * head_size, units_q, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, head_size, head_size, dimension_x, units_q * heads, 1, optimization, iBatch) || !cPrepareQ.Add(conv)) DeleteObjAndFalse(conv); //--- W0 index++; CNeuronBaseOCL* neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, dimension_x * units_q * heads, optimization, iBatch) || !cW0.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, dimension_x, dimension_x, head_size, units_q * heads, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, heads * head_size, heads * head_size, dimension_q, units_q, 1, optimization, iBatch) || !cW0.Add(conv)) DeleteObjAndFalse(conv); //--- if(!cLogSumExp.BufferInit(units_q * heads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionOut(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { if(!OpenCL || !NeuronOCL || !cW0[0] || !cW0[0].getOutput() || !NeuronOCL.getOutput() || !Context) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { iQUnits, (uint)MathMin(MathMax(iXUnits, iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCA; setBuffer(kernel, def_k_mhstca_query, NeuronOCL.getOutputIndex()) setBuffer(kernel, def_k_mhstca_X, Context.GetIndex()) setBuffer(kernel, def_k_mhstca_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_output, cW0[0].getOutputIndex()) setArgument(kernel, def_k_mhstca_dimension, iXDimension) setArgument(kernel, def_k_mhstca_total_X, iXUnits) setArgument(kernel, def_k_mhstca_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) dActivation(cW0[0]); #ifdef _DEBUG if(!cW0[0].getOutput().BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::AttentionInsideGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!OpenCL || !prevLayer || !cW0[0] || !cW0[0].getOutput() || !prevLayer.getOutput() || !SecondInput || !cW0[0].getGradient() || !prevLayer.getGradient() || !SecondGradient) ReturnFalse; //--- uint global_work_offset[3] = {0}; uint global_work_size[] = { MathMax(iQUnits, iXUnits), (uint)MathMin( MathMax( MathMax(iQUnits * iHeads, iXUnits), iXDimension), OpenCL.GetMaxLocalSize(1)), iHeads }; uint local_work_size[] = { 1, global_work_size[1], 1}; uint kernel = def_k_MHFlashSTCAGrad; setBuffer(kernel, def_k_mhstca_gr_query, prevLayer.getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_query_gr, prevLayer.getGradientIndex()) setBuffer(kernel, def_k_mhstca_gr_X, SecondInput.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_X_gr, SecondGradient.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_logsumexp, cLogSumExp.GetIndex()) setBuffer(kernel, def_k_mhstca_gr_output, cW0[0].getOutputIndex()) setBuffer(kernel, def_k_mhstca_gr_output_gr, cW0[0].getGradientIndex()) setArgument(kernel, def_k_mhstca_gr_dimension, iXDimension) setArgument(kernel, def_k_mhstca_gr_total_q, iQUnits) setArgument(kernel, def_k_mhstca_gr_total_X, iXUnits) setArgument(kernel, def_k_mhstca_gr_mask_future, int(bMask)) kernelExecuteLoc(kernel, global_work_offset, global_work_size, local_work_size) //--- Deactivation(prevLayer); if(SecondActivation != None) if(!DeActivation(SecondInput, SecondGradient, SecondGradient, SecondActivation)) ReturnFalse; #ifdef _DEBUG if(!cW0[0].getGradient().BufferRead()) ReturnFalse; if(!SecondGradient.BufferRead()) ReturnFalse; #endif //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::feedForward(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } //--- if(!AttentionOut(prev, Context)) ReturnFalse; prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; } if(!SumAndNormilize(NeuronOCL.getOutput(), prev.getOutput(), prev.getOutput(), prev.Neurons() / iQUnits, true, 0, 0, 0, 1)) ReturnFalse; if(!CNeuronMTmixAttBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::calcInputGradients(CNeuronBaseOCL *prevLayer, CBufferFloat *SecondInput, CBufferFloat *SecondGradient, ENUM_ACTIVATION SecondActivation = None) { if(!prevLayer || !SecondInput || !SecondGradient) ReturnFalse; //--- if(!CNeuronMTmixAttBlock::calcInputGradients(cW0[-1])) ReturnFalse; //--- CNeuronBaseOCL* curr = NULL; for(int i = cW0.Total() - 2; i >= 0; i--) { curr = cW0[i]; if(!curr || !curr.CalcHiddenGradients(cW0[i + 1])) ReturnFalse; } //--- if(!AttentionInsideGradients(cPrepareQ[-1], SecondInput, SecondGradient, SecondActivation)) ReturnFalse; //--- for(int i = cPrepareQ.Total() - 2; i >= 0; i--) { curr = cPrepareQ[i]; if(!curr || !curr.CalcHiddenGradients(cPrepareQ[i + 1])) ReturnFalse; } if(!prevLayer.CalcHiddenGradients(curr)) ReturnFalse; if(prevLayer.Activation() != None) { if(!DeActivation(prevLayer.getOutput(), cW0[-1].getPrevOutput(), cW0[-1].getGradient(), prevLayer.Activation()) || !SumAndNormilize(prevLayer.getGradient(), cW0[-1].getPrevOutput(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; } else if(!SumAndNormilize(prevLayer.getGradient(), cW0[-1].getGradient(), prevLayer.getGradient(), 1, false, 0, 0, 0, 1)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::updateInputWeights(CNeuronBaseOCL *NeuronOCL, CBufferFloat *Context) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- for(int i = 0; i < cPrepareQ.Total(); i++) { curr = cPrepareQ[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- prev = cW0[0]; for(int i = 1; i < cW0.Total(); i++) { curr = cW0[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- if(!CNeuronMTmixAttBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Save(const int file_handle) { if(!CNeuronMTmixAttBlock::Save(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Save(file_handle)) ReturnFalse; if(!cW0.Save(file_handle)) ReturnFalse; //--- dFileWriteUInt(file_handle, iQUnits); dFileWriteUInt(file_handle, iXUnits); dFileWriteUInt(file_handle, iHeads); dFileWriteUInt(file_handle, iXDimension); dFileWriteUInt(file_handle, uint(bMask)); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::Load(const int file_handle) { cLogSumExp.BufferFree(); //--- if(!CNeuronMTmixAttBlock::Load(file_handle)) ReturnFalse; //--- if(!cPrepareQ.Load(file_handle)) ReturnFalse; if(!cW0.Load(file_handle)) ReturnFalse; //--- dFileReadUInt(file_handle, iQUnits); dFileReadUInt(file_handle, iXUnits); dFileReadUInt(file_handle, iHeads); dFileReadUInt(file_handle, iXDimension); uint mask = 0; dFileReadUInt(file_handle, mask); bMask = bool(mask); //--- if(!cLogSumExp.BufferInit(iQUnits * iHeads, 0) || !cLogSumExp.BufferCreate(OpenCL)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronMHTHCrossAttention::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronMTmixAttBlock::WeightsUpdate(source, tau)) ReturnFalse; //--- CNeuronMHTHCrossAttention* Source = source; dWeightsUpdate(cPrepareQ, Source, tau); dWeightsUpdate(cW0, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::SetOpenCL(COpenCLMy *obj) { cLogSumExp.BufferFree(); CNeuronMTmixAttBlock::SetOpenCL(obj); //--- cPrepareQ.SetOpenCL(OpenCL); cW0.SetOpenCL(OpenCL); cLogSumExp.BufferCreate(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronMHTHCrossAttention::TrainMode(bool flag) { CNeuronMTmixAttBlock::TrainMode(flag); //--- cPrepareQ.TrainMode(bTrain); cW0.TrainMode(bTrain); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ class CNeuronSTCA : public CNeuronSpikeConvBlock { protected: CLayer cPrepare; CNeuronBaseOCL cLastSequence; CNeuronBaseOCL cLastNonSequence; CNeuronAddToStack cStackSequence; CFieldAwareParams cScenarios; CLayer cFlow; //--- virtual bool feedForward(CNeuronBaseOCL *NeuronOCL) override; virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) override; virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL) override; public: CNeuronSTCA(void) {}; ~CNeuronSTCA(void) {}; //--- virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch); //--- virtual bool Save(const int file_handle) override; virtual bool Load(const int file_handle) override; //--- virtual int Type(void) override const { return defNeuronSTCA; } virtual void SetOpenCL(COpenCLMy *obj) override; virtual void TrainMode(bool flag) override; //--- virtual bool WeightsUpdate(CNeuronBaseOCL *source, float tau) override; virtual bool Clear(void) override; }; //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint &dimensions[], uint units_s, uint units_out, uint heads, uint scenarios, uint stack_size, uint layers, uint embed_size, uint candidates, uint topK, ENUM_OPTIMIZATION optimization_type, uint batch) { uint count = dimensions.Size(); if(units_s <= 0 || units_out <= 0 || count < (units_s + 2)) ReturnFalse; uint dimension_out = dimensions[count - 1]; uint units_ns = count - units_s - 1; uint bottleneck = (embed_size + topK - 1) / topK; //--- if(!CNeuronSpikeConvBlock::Init(numOutputs, myIndex, open_cl, bottleneck * topK, bottleneck * topK, dimension_out, units_out, 1, optimization_type, batch)) ReturnFalse; //--- Prepare arrays cPrepare.Clear(); cFlow.Clear(); cPrepare.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); //--- CNeuronBaseOCL* neuron = NULL; CNeuronBatchNormOCL* norm = NULL; CNeuronMultiWindowsConvOCL* mwc = NULL; CNeuronFieldPatternEmbedding* emb = NULL; CNeuronAutoToken* select = NULL; CNeuronSwiGLUOCL* swiglu = NULL; CNeuronSpikeConvBlock* conv = NULL; CNeuronMHTHCrossAttention* attention = NULL; //--- Prepare inputs uint windows[]; if(ArrayCopy(windows, dimensions, 0, 0, count - 1) < int(count - 1)) ReturnFalse; uint index = 0; norm = new CNeuronBatchNormOCL(); uint total_windows = 0; for(uint i = 0; i < windows.Size(); i++) total_windows += windows[i]; if(!norm || !norm.Init(0, index, OpenCL, total_windows, iBatch, optimization) || !cPrepare.Add(norm)) DeleteObjAndFalse(norm); norm.SetActivationFunction(None); index++; mwc = new CNeuronMultiWindowsConvOCL(); if(!mwc || !mwc.Init(0, index, OpenCL, windows, embed_size, 1, 1, optimization, iBatch) || !cPrepare.Add(mwc)) DeleteObjAndFalse(mwc); mwc.SetActivationFunction(SIGMOID); index++; emb = new CNeuronFieldPatternEmbedding(); if(!emb || !emb.Init(0, index, OpenCL, embed_size, windows.Size(), embed_size, candidates, topK, optimization, iBatch) || !cPrepare.Add(emb)) DeleteObjAndFalse(emb); emb.SetActivationFunction(TANH); //--- Sequence/NonSequence index++; if(!cLastSequence.Init(0, index, OpenCL, units_s * embed_size, optimization, iBatch)) ReturnFalse; cLastSequence.SetActivationFunction(None); index++; if(!cLastNonSequence.Init(0, index, OpenCL, units_ns * embed_size, optimization, iBatch)) ReturnFalse; cLastNonSequence.SetActivationFunction(None); index++; if(!cStackSequence.Init(0, index, OpenCL, stack_size, embed_size, units_s, optimization, iBatch)) ReturnFalse; index++; if(!cScenarios.Init(0, index, OpenCL, embed_size, scenarios, bottleneck, candidates, topK, optimization_type, batch)) ReturnFalse; cScenarios.SetActivationFunction(TANH); //--- Flow index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (scenarios + units_ns), optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, scenarios + units_ns, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); for(uint l = 0; l < layers; l++) { index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut()*topK, swiglu.GetWindowOut()*topK, embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); index++; attention = new CNeuronMHTHCrossAttention(); if(!attention || !attention.Init(0, index, OpenCL, embed_size, units_out, heads, embed_size, stack_size, bottleneck, candidates, topK, optimization, iBatch) || !cFlow.Add(attention)) DeleteObjAndFalse(attention); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, embed_size, embed_size, bottleneck, units_out, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); index++; conv = new CNeuronSpikeConvBlock(); if(!conv || !conv.Init(0, index, OpenCL, swiglu.GetWindowOut(), swiglu.GetWindowOut(), embed_size, units_out, 1, optimization, iBatch) || !cFlow.Add(conv)) DeleteObjAndFalse(conv); if(l < layers - 1) { index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * (l + 2) * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); } } index++; neuron = new CNeuronBaseOCL(); if(!neuron || !neuron.Init(0, index, OpenCL, embed_size * layers * units_out, optimization, iBatch) || !cFlow.Add(neuron)) DeleteObjAndFalse(neuron); neuron.SetActivationFunction(None); index++; select = new CNeuronAutoToken(); if(!select || !select.Init(0, index, OpenCL, embed_size, bottleneck, neuron.Neurons() / embed_size, units_out, candidates, topK, optimization, iBatch) || !cFlow.Add(select)) DeleteObjAndFalse(select); index++; swiglu = new CNeuronSwiGLUOCL(); if(!swiglu || !swiglu.Init(0, index, OpenCL, bottleneck, bottleneck, bottleneck, units_out * topK, 1, optimization, iBatch) || !cFlow.Add(swiglu)) DeleteObjAndFalse(swiglu); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::feedForward(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.FeedForward(prev)) ReturnFalse; prev = curr; #ifdef _DEBUG if(!curr.getOutput().BufferRead()) ReturnFalse; #endif } //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFiedls(); uint units_out = GetUnits(); //--- Sequence/NonSequence if(!DeConcat(cLastSequence.getOutput(), cLastNonSequence.getOutput(), prev.getOutput(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; if(!cStackSequence.FeedForward(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.FeedForward()) ReturnFalse; prev = cFlow[0]; if(!prev || !Concat(cLastNonSequence.getOutput(), cScenarios.getOutput(), prev.getOutput(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; bool attention_flag = false; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() == defNeuronBaseOCL) { if(!stack_querys || !prev || !Concat(prev.getOutput(), stack_querys.getOutput(), curr.getOutput(), embedding_size, curr.Neurons() / units_out - embedding_size, units_out)) ReturnFalse; stack_querys = curr; } else if(!curr.FeedForward(prev, cStackSequence.getOutput())) ReturnFalse; if(!stack_querys && curr.Type() == defNeuronMHTHCrossAttention) stack_querys = prev; prev = curr; #ifdef _DEBUG if(!curr.getOutput().BufferRead()) ReturnFalse; #endif } //--- if(!CNeuronSpikeConvBlock::feedForward(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::calcInputGradients(CNeuronBaseOCL *NeuronOCL) { if(!NeuronOCL) return false; //--- if(!CNeuronSpikeConvBlock::feedForward(cFlow[-1])) ReturnFalse; //--- //--- uint embedding_size = cStackSequence.GetDimension(); uint units_s = cLastSequence.Neurons() / embedding_size; uint units_ns = cLastNonSequence.Neurons() / embedding_size; uint scenarios = cScenarios.GetFiedls(); uint units_out = GetUnits(); CNeuronBaseOCL* next = cFlow[-1]; CNeuronBaseOCL* curr = NULL; CNeuronBaseOCL* stack_querys = NULL; //--- Flow for(int i = cFlow.Total() - 2; i >= 0; i--) { curr = cFlow[i]; if(!curr) ReturnFalse; if(next.Type() == defNeuronBaseOCL) { if(!DeActivation(curr.getOutput(), curr.getGradient(), next.getGradient(), curr.Activation())) ReturnFalse; } else if(!curr.CalcHiddenGradients(next, cStackSequence.getOutput(), (!stack_querys ? cStackSequence.getGradient() : cStackSequence.getPrevOutput()), (ENUM_ACTIVATION)cStackSequence.Activation())) ReturnFalse; if(curr.Type() == defNeuronMHTHCrossAttention && !!stack_querys) { if(!SumAndNormilize(cStackSequence.getGradient(), cStackSequence.getPrevOutput(), cStackSequence.getGradient(), embedding_size, false, 0, 0, 0, 1)) ReturnFalse; } if(i > 0 && curr.Type() == defNeuronBaseOCL) { if(!!stack_querys) { if(!DeConcat(stack_querys.getPrevOutput(), curr.getPrevOutput(), stack_querys.getGradient(), embedding_size, curr.Neurons() / units_out - embedding_size, units_out) || !SumAndNormilize(curr.getGradient(), curr.getPrevOutput(), curr.getGradient(), embedding_size, false, 0, 0, 0, 1)) ReturnFalse; } stack_querys = curr; } next = curr; } //--- Sequence/NonSequence if(!DeConcat(cLastNonSequence.getGradient(), cScenarios.getGradient(), cFlow[0].getGradient(), embedding_size * units_ns, embedding_size * scenarios, 1)) ReturnFalse; Deactivation(cLastNonSequence); Deactivation(cScenarios); if(!cLastSequence.CalcHiddenGradients(cStackSequence.AsObject())) ReturnFalse; next = cPrepare[-1]; if(!next || !Concat(cLastSequence.getGradient(), cLastNonSequence.getGradient(), next.getGradient(), embedding_size * units_s, embedding_size * units_ns, 1)) ReturnFalse; //--- Inputs for(int i = cPrepare.Total() - 2; i >= 0; i--) { curr = cPrepare[i]; if(!curr || !curr.CalcHiddenGradients(next)) ReturnFalse; next = curr; } //--- if(!NeuronOCL.CalcHiddenGradients(next)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::updateInputWeights(CNeuronBaseOCL *NeuronOCL) { CNeuronBaseOCL* prev = NeuronOCL; CNeuronBaseOCL* curr = NULL; //--- Inputs for(int i = 0; i < cPrepare.Total(); i++) { curr = cPrepare[i]; if(!curr || !curr.UpdateInputWeights(prev)) ReturnFalse; prev = curr; } //--- Sequence/NonSequence if(!cStackSequence.UpdateInputWeights(cLastSequence.AsObject())) ReturnFalse; //--- Flow if(!cScenarios.UpdateInputWeights()) ReturnFalse; prev = cFlow[0]; for(int i = 1; i < cFlow.Total(); i++) { curr = cFlow[i]; if(!curr) ReturnFalse; if(curr.Type() != defNeuronBaseOCL) if(!curr.UpdateInputWeights(prev, cStackSequence.getOutput())) ReturnFalse; prev = curr; } //--- if(!CNeuronSpikeConvBlock::updateInputWeights(prev)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Save(const int file_handle) { if(!CNeuronSpikeConvBlock::Save(file_handle)) ReturnFalse; if(!cPrepare.Save(file_handle)) ReturnFalse; if(!cLastSequence.Save(file_handle)) ReturnFalse; if(!cLastNonSequence.Save(file_handle)) ReturnFalse; if(!cStackSequence.Save(file_handle)) ReturnFalse; if(!cScenarios.Save(file_handle)) ReturnFalse; if(!cFlow.Save(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Load(const int file_handle) { if(!CNeuronSpikeConvBlock::Load(file_handle)) ReturnFalse; if(!cPrepare.Load(file_handle)) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cLastNonSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cStackSequence.AsObject())) ReturnFalse; if(!LoadInsideLayer(file_handle, cScenarios.AsObject())) ReturnFalse; if(!cFlow.Load(file_handle)) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::WeightsUpdate(CNeuronBaseOCL *source, float tau) { if(!CNeuronSpikeConvBlock::WeightsUpdate(source, tau)) ReturnFalse; CNeuronSTCA* Source = source; dWeightsUpdate(cPrepare, Source, tau); dWeightsUpdate(cLastSequence, Source, tau); dWeightsUpdate(cLastNonSequence, Source, tau); dWeightsUpdate(cStackSequence, Source, tau); dWeightsUpdate(cScenarios, Source, tau); dWeightsUpdate(cFlow, Source, tau); //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ bool CNeuronSTCA::Clear(void) { if(!CNeuronSpikeConvBlock::Clear()) ReturnFalse; if(!cPrepare.ClearStates()) ReturnFalse; if(!cLastSequence.Clear()) ReturnFalse; if(!cLastNonSequence.Clear()) ReturnFalse; if(!cStackSequence.Clear()) ReturnFalse; if(!cScenarios.Clear()) ReturnFalse; if(!cFlow.ClearStates()) ReturnFalse; //--- return true; } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTCA::SetOpenCL(COpenCLMy *obj) { CNeuronSpikeConvBlock::SetOpenCL(obj); cPrepare.SetOpenCL(OpenCL); cLastSequence.SetOpenCL(OpenCL); cLastNonSequence.SetOpenCL(OpenCL); cStackSequence.SetOpenCL(OpenCL); cScenarios.SetOpenCL(OpenCL); cFlow.SetOpenCL(OpenCL); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ void CNeuronSTCA::TrainMode(bool flag) { CNeuronSpikeConvBlock::TrainMode(flag); cPrepare.TrainMode(bTrain); cLastSequence.TrainMode(bTrain); cLastNonSequence.TrainMode(bTrain); cStackSequence.TrainMode(bTrain); cScenarios.TrainMode(bTrain); cFlow.TrainMode(bTrain); } //+------------------------------------------------------------------+