#ifndef NEURONET_DEFINITIONS_MQH #define NEURONET_DEFINITIONS_MQH /// \file /// \brief NeuroNet.mqh /// Library for creating Neural network for use in MQL5 experts /// \author [DNG](https://www.mql5.com/en/users/dng) /// \copyright Copyright 2019, DNG //+------------------------------------------------------------------+ ///\mainpage NeuronNet /// Library for creating Neural network for use in MQL5 experts. /// - \ref const /// - \ref enums /// - \ref ObjectTypes /// - \ref group1 /// - [Class Hierarchy](hierarchy.html) /// - [Files](files.html) //+------------------------------------------------------------------+ #property copyright "Copyright 2019, DNG" #property link "https://www.mql5.com" #property version "1.00" //+------------------------------------------------------------------+ //| Include standart libraries //+------------------------------------------------------------------+ #include #include #include #include #include //+------------------------------------------------------------------+ // Defines //+------------------------------------------------------------------+ ///\defgroup const Global constants ///@{ #define lr 1.0e-2f ///>19))^(rnd_t^(rnd_t>>8)) //--- uint rnd_x = MathRand(), rnd_y = MathRand(), rnd_z = MathRand(), rnd_w = MathRand(), rnd_t = 0; ///@} #define CreateKernel(id, name) if(!opencl.KernelCreate(id, name)) { \ PrintFormat("Error of create kernel %s: %d line %d", #name, GetLastError(), __LINE__); \ ReturnFalse; } #define setBuffer(kernel, id, buffer) if(!OpenCL.SetArgumentBuffer(kernel, id, buffer)) { \ printf("Error of set parameter kernel %s: %d; line %d", OpenCL.GetKernelName(kernel), GetLastError(), __LINE__); \ ReturnFalse; } #define setArgument(kernel, id, value) if(!OpenCL.SetArgument(kernel, id, value)) { \ printf("Error of set parameter kernel %s: %d; line %d", OpenCL.GetKernelName(kernel), GetLastError(), __LINE__); \ ReturnFalse; } #define kernelExecute(kernel,offset,global) if(!OpenCL.Execute(kernel, global.Size(), offset, global)) { \ string error; \ CLGetInfoString(OpenCL.GetContext(), CL_ERROR_DESCRIPTION, error); \ printf("Error of execution kernel %s %s:\n %s", __FUNCTION__, OpenCL.GetKernelName(kernel), error); \ string s_global=StringFormat("global = {%d",global[0]); \ for(uint i=1;i T1 pow(T2 a, T1 b) { return (T1)MathPow((T1)a, (T1)b); } //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ union uaLongChar { long data[3]; uchar cdata[24]; }; //+------------------------------------------------------------------+ ///\defgroup ObjectTypes Defines Object types identified ///Used to identify classes in a library ///@{ //+------------------------------------------------------------------+ ///\defgroup arr Arrays ///Used to identify array classes ///\{ #define defArrayConnects 0x7782 ///the link. ///@{ #define def_k_FeedForward 0 ///< Index of #FeedForward kernel #define def_k_ff_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in layer and n - number of outputs (neurons in next layer) #define def_k_ff_matrix_i 1 ///< Inputs tesor #define def_k_ff_matrix_o 2 ///< Output tensor #define def_k_ff_inputs 3 ///< Number of inputs #define def_k_ff_activation 4 ///< Activation type (#ENUM_ACTIVATION) ///@} ///\defgroup neuron_base_gr Gradients Calculation kernels /// Describes the process of gradients calculation for the Neuron Base. ///\details Detailed description on the link. ///@{ #define def_k_CalcOutputGradient 1 ///< Index of Output gradients calculation kernel (#CalcOutputGradient) #define def_k_cog_matrix_t 0 ///< Target tensor #define def_k_cog_matrix_o 1 ///< Output tensor #define def_k_cog_matrix_ig 2 ///< Tensor of gradients at previous layer #define def_k_cog_activation 3 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cog_error 4 ///< Error //--- #define def_k_CalcHiddenGradient 2 ///< Index of Hidden gradients calculation kernel (#CalcHiddenGradient) #define def_k_chg_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_chg_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_chg_matrix_o 2 ///< Output tensor #define def_k_chg_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_chg_outputs 4 ///< Number of outputs #define def_k_chg_activation 5 ///< Activation type (#ENUM_ACTIVATION) ///@} ///\defgroup neuron_base_opt Updating Weights Calculation kernel /// Describes the process of optimization weights for the Neuron Base. ///\details Detailed description on the link. /// For Adam optimization look the link. ///@{ #define def_k_UpdateWeightsMomentum 3 ///< Index SGD optomization Update weights kernel (#UpdateWeightsMomentum) #define def_k_uwm_matrix_w 0 ///< SGD Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwm_matrix_g 1 ///< SGD Tensor of gradients at current layer #define def_k_uwm_matrix_i 2 ///< SGD Inputs tesor #define def_k_uwm_matrix_dw 3 ///< SGD Matrix of delta weights in last correction #define def_k_uwm_inputs 4 ///< SGD Number of inputs #define def_k_uwm_learning_rates 5 ///< SGD Learning rates #define def_k_uwm_momentum 6 ///< SGD Momentum multiplier //--- #define def_k_UpdateWeightsAdam 4 ///< Index Adam optomization Update weights kernel (#UpdateWeightsAdam) #define def_k_uwa_matrix_w 0 ///< Adam Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwa_matrix_g 1 ///< Adam Tensor of gradients at current layer #define def_k_uwa_matrix_i 2 ///< Adam Inputs tesor #define def_k_uwa_matrix_m 3 ///< Adam Matrix of first momentum #define def_k_uwa_matrix_v 4 ///< Adam Matrix of seconfd momentum #define def_k_uwa_inputs 5 ///< Adam Number of inputs #define def_k_uwa_l 6 ///< Adam Learning rates #define def_k_uwa_b1 7 ///< Adam First momentum multiplier #define def_k_uwa_b2 8 ///< Adam Second momentum multiplier //--- #define def_k_UpdateWeightsLS 28 ///< Index Least Squares optomization Update weights kernel (#UpdateWeightsLS) #define def_k_uwls_matrix_w 0 ///< Least Squares Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_uwls_matrix_g 1 ///< Least Squares Tensor of gradients at current layer #define def_k_uwls_matrix_i 2 ///< Least Squares Inputs tesor #define def_k_uwls_matrix_xg 3 ///< Least Squares Matrix of summ x*g #define def_k_uwls_matrix_xx 4 ///< Least Squares Matrix of summ x*x #define def_k_uwls_inputs 5 ///< Least Squares Number of inputs #define def_k_uwls_l 6 ///< Least Squares Learning rates #define def_k_uwls_update 7 ///< Least Squares Update flag ///@} ///@} //--- ///\defgroup neuron_proof Pooling layer's neuron /// Describes the process for the Neuron of pooling layer. ///@{ ///\defgroup neuron_proof_ff Pooling layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of pooling layer. ///@{ #define def_k_FeedForwardProof 5 ///< Index of the kernel of the Pooling neuron for Feed forward process (#FeedForwardProof) #define def_k_ffp_matrix_i 0 ///< Inputs tesor #define def_k_ffp_matrix_o 1 ///< Output tensor #define def_k_ffp_inputs 2 ///< Number of inputs #define def_k_ffp_window 3 ///< Size of input window #define def_k_ffp_step 4 ///< Step size ///@} //--- ///\defgroup neuron_proof_gr Pooling layer's neuron Gradients Calculation kernels /// Describes the gradient calculation process for the Neuron of pooling layer. ///@{ #define def_k_CalcInputGradientProof 6 ///< Index of the kernel of the Pooling neuron to transfer gradient to previous layer (#CalcInputGradientProof) #define def_k_cigp_matrix_i 0 ///< Inputs tesor #define def_k_cigp_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_cigp_matrix_o 2 ///< Output tensor #define def_k_cigp_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_cigp_outputs 4 ///< Number of outputs #define def_k_cigp_window 5 ///< Size of input window #define def_k_cigp_step 6 ///< Step size ///@} ///@} //--- ///\defgroup neuron_conv Convolution layer's neuron /// Describes the process for the Neuron of convolution layer. ///@{ ///\defgroup neuron_conv_ff Convolution layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of convolution layer. ///@{ #define def_k_FeedForwardConv 7 ///< Index of the kernel of the convolution neuron for Feed forward process (#FeedForwardConv) #define def_k_ffc_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_ffc_matrix_i 1 ///< Inputs tesor #define def_k_ffc_matrix_o 2 ///< Output tensor #define def_k_ffc_inputs 3 ///< Number of inputs #define def_k_ffc_step 4 ///< Step size #define def_k_ffc_window_in 5 ///< Size of input window #define def_k_ffс_window_out 6 ///< Size of output window #define def_k_ffc_activation 7 ///< Activation type (#ENUM_ACTIVATION) ///@} //--- ///\defgroup neuron_conv_gr Convolution layer's neuron Gradients Calculation kernels /// Describes the gradient calculation process for the Neuron of convolution layer. ///@{ #define def_k_CalcHiddenGradientConv 8 ///< Index of the kernel of the convolution neuron to transfer gradient to previous layer (#CalcHiddenGradientConv) #define def_k_chgc_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_chgc_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_chgc_matrix_o 2 ///< Output tensor #define def_k_chgc_matrix_ig 3 ///< Tensor of gradients at previous layer #define def_k_chgc_outputs 4 ///< Number of outputs #define def_k_chgc_step 5 ///< Step size #define def_k_chgc_window_in 6 ///< Size of input window #define def_k_chgc_window_out 7 ///< Size of output window #define def_k_chgc_activation 8 ///< Activation type (#ENUM_ACTIVATION) #define def_k_chgc_shift_out 9 ///< Activation type (#ENUM_ACTIVATION) ///@} //--- ///\defgroup neuron_conv_opt Convolution layer's neuron Update weights kernels /// Describes the optimization process for the Neuron of convolution layer. ///@{ #define def_k_UpdateWeightsConvMomentum 9 ///< Index of the kernel of the convolution neuron to update weights SGD (#UpdateWeightsConvMomentum) #define def_k_uwcm_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwcm_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwcm_matrix_i 2 ///< Inputs tesor #define def_k_uwcm_matrix_dw 3 ///< Matrix of delta weights in last correction #define def_k_uwcm_inputs 4 ///< Number of inputs #define def_k_uwcm_learning_rates 5 ///< Learning rates #define def_k_uwcm_momentum 6 ///< Momentum multiplier #define def_k_uwcm_window_in 7 ///< Size of input window #define def_k_uwcm_window_out 8 ///< Size of output window #define def_k_uwcm_step 9 ///< Step size //--- #define def_k_UpdateWeightsConvAdam 10 ///< Index of the kernel of the convolution neuron to update weights Adam (#UpdateWeightsConvAdam) #define def_k_uwca_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwca_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwca_matrix_i 2 ///< Inputs tesor #define def_k_uwca_matrix_m 3 ///< Matrix of first momentum #define def_k_uwca_matrix_v 4 ///< Matrix of seconfd momentum #define def_k_uwca_inputs 5 ///< Number of inputs #define def_k_uwca_l 6 ///< Learning rates #define def_k_uwca_b1 7 ///< First momentum multiplier #define def_k_uwca_b2 8 ///< Second momentum multiplier #define def_k_uwca_window_in 9 ///< Size of input window #define def_k_uwca_window_out 10 ///< Size of output window #define def_k_uwca_step 11 ///< Step size //--- #define def_k_UpdateWeightsConvLS 29 ///< Index of the kernel of the convolution neuron to update weights Least Squares(#UpdateWeightsConvLS) #define def_k_uwcls_matrix_w 0 ///< Weights matrix (m+1)*n, where m - input window and n - output window #define def_k_uwcls_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_uwcls_matrix_i 2 ///< Inputs tesor #define def_k_uwcls_matrix_xg 3 ///< Matrix of first momentum #define def_k_uwcls_matrix_xx 4 ///< Matrix of seconfd momentum #define def_k_uwcls_inputs 5 ///< Number of inputs #define def_k_uwcls_l 6 ///< Learning rates #define def_k_uwcls_update 7 ///< Update flag #define def_k_uwcls_window_in 8 ///< Size of input window #define def_k_uwcls_window_out 9 ///< Size of output window #define def_k_uwcls_step 10 ///< Step size ///@} ///@} //--- ///\defgroup neuron_atten Attention layer's neuron /// Describes the process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ ///\defgroup neuron_atten_ff Attention layer's neuron Feed Forward /// Describes the feed forward process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ #define def_k_AttentionScore 11 ///< Index of the kernel of the attention neuron to calculate score matrix (#AttentionScore) #define def_k_as_querys 0 ///< Matrix of Querys #define def_k_as_keys 1 ///< Matriz of Keys #define def_k_as_score 2 ///< Matrix of Scores #define def_k_as_dimension 3 ///< Dimension of Key #define def_k_as_mask 4 ///< 1 - calc only previous units, 0 - calc all //--- #define def_k_AttentionOut 12 ///< Index of the Attention Neuron Output calculation kernel (#AttentionOut) #define def_k_aout_scores 0 ///< Matrix of Scores #define def_k_aout_values 1 ///< Matrix of Values #define def_k_aout_inputs 2 ///< Inputs tesor #define def_k_aout_out 3 ///< Output tesor //--- #define def_k_MatrixSum 13 ///< Index of the kernel for calculation Sum of 2 matrix with multiplyer (#SumMatrix) #define def_k_sum_matrix1 0 ///< First matrix #define def_k_sum_matrix2 1 ///< Second matrix #define def_k_sum_matrix_out 2 ///< Output matrix #define def_k_sum_dimension 3 ///< Dimension of matrix #define def_k_sum_multiplyer 4 ///< Multiplyer for output #define def_k_sum_shift_in1 5 #define def_k_sum_shift_in2 6 #define def_k_sum_shift_out 7 //--- #define def_k_Matrix5Sum 19 ///< Index of the kernel for calculation Sum of 2 matrix with multiplyer (#SumMatrix) #define def_k_sum5_matrix1 0 ///< First matrix #define def_k_sum5_matrix2 1 ///< Second matrix #define def_k_sum5_matrix3 2 ///< Third matrix #define def_k_sum5_matrix4 3 ///< Fourth matrix #define def_k_sum5_matrix5 4 ///< Fifth matrix #define def_k_sum5_matrix_out 5 ///< Output matrix #define def_k_sum5_dimension 6 ///< Dimension of matrix #define def_k_sum5_multiplyer 7 ///< Multiplyer for output //--- #define def_k_MHAttentionScore 20 ///< Index of the kernel of the multi-heads attention neuron to calculate score matrix (#MHAttentionScore) #define def_k_mhas_qkv 0 ///< Matrix of Queries, Keys, Values #define def_k_mhas_score 1 ///< Matrix of Scores #define def_k_mhas_dimension 2 ///< Dimension of Key #define def_k_mhas_mask 3 ///< 1 - calc only previous units, 0 - calc all //--- #define def_k_MHAttentionOut 21 ///< Index of the kernel of the multi-heads attention neuron to calculate multi-heads out matrix (#MHAttentionOut) #define def_k_mhao_score 0 ///< Matrix of Scores #define def_k_mhao_qkv 1 ///< Matrix of Queries, Keys, Values #define def_k_mhao_out 2 ///< Matrix of Outputs #define def_k_mhao_dimension 3 ///< Dimension of Key //--- #define def_k_ConcatenateMatrix 17 ///< Index of the Multi Head Attention Neuron Concatenate Output kernel (#ConcatenateBuffers) #define def_k_conc_input1 0 ///< Matrix of Buffer 1 #define def_k_conc_window1 1 ///< Window of Buffer 1 #define def_k_conc_input2 2 ///< Matrix of Buffer 2 #define def_k_conc_window2 3 ///< Window of Buffer 2 #define def_k_conc_input3 4 ///< Matrix of Buffer 3 #define def_k_conc_window3 5 ///< Window of Buffer 3 #define def_k_conc_input4 6 ///< Matrix of Buffer 4 #define def_k_conc_window4 7 ///< Window of Buffer 4 #define def_k_conc_out 8 ///< Output tesor ///@} //--- ///\defgroup neuron_atten_gr Attention layer's neuron Gradients Calculation /// Describes the gradients calculation process for the Neuron of attention layer. ///\details Detailed description on the link. ///@{ #define def_k_AttentionGradients 14 ///< Index of the kernel for gradients calculation process (#AttentionInsideGradients) #define def_k_ag_querys 0 ///< Matrix of Querys #define def_k_ag_querys_g 1 ///< Matrix of Querys' Gradients #define def_k_ag_keys 2 ///< Matrix of Keys #define def_k_ag_keys_g 3 ///< Matrix of Keys' Gradients #define def_k_ag_values 4 ///< Matrix of Values #define def_k_ag_values_g 5 ///< Matrix of Values' Gradients #define def_k_ag_scores 6 ///< Matrix of Scores #define def_k_ag_gradient 7 ///< Matrix of Gradients from previous iteration //--- #define def_k_DeconcatenateMatrix 18 ///< Index of the Multi Head Attention Neuron Deconcatenate Output kernel (#DeconcatenateBuffers) #define def_k_dconc_output1 0 ///< Matrix of Buffer 1 #define def_k_dconc_window1 1 ///< Window of Buffer 1 #define def_k_dconc_output2 2 ///< Matrix of Buffer 2 #define def_k_dconc_window2 3 ///< Window of Buffer 2 #define def_k_dconc_output3 4 ///< Matrix of Buffer 3 #define def_k_dconc_window3 5 ///< Window of Buffer 3 #define def_k_dconc_output4 6 ///< Matrix of Buffer 4 #define def_k_dconc_window4 7 ///< Window of Buffer 4 #define def_k_dconc_inputs 8 ///< Input tesor //--- #define def_k_MHAttentionGradients 22 ///< Index of the kernel for gradients calculation process (#AttentionInsideGradients) #define def_k_mhag_qkv 0 ///< Matrix of Queries, Keys, Values #define def_k_mhag_qkv_g 1 ///< Matrix of Gradients to Queries, Keys, Values #define def_k_mhag_score 2 ///< Matrix of Scores #define def_k_mhag_gradient 3 ///< Matrix of Gradients from previous iteration //--- #define def_k_Dropout 23 ///< Index of the kernel for Dropout process (#Dropout) #define def_k_dout_input 0 ///< Inputs Tensor #define def_k_dout_map 1 ///< Map Tensor #define def_k_dout_out 2 ///< Out Tensor #define def_k_dout_dimension 3 ///< Dimension of Inputs ///@} ///@} //--- ///\defgroup neuron_norm Kernels of matrix normalization process /// Describes the process of matrix normalization. ///\details Detailed description on the link. ///@{ #define def_k_Normalize 15 ///< Index of the kernel for matrix normalization (#Normalize) #define def_k_norm_buffer 0 ///< In/Out Matrix #define def_k_norm_dimension 1 ///< Dimension of matrix //--- #define def_k_NormalizeWeights 16 ///< Index of the kernel for weights matrix normalization (#NormalizeWeights) //--- #define def_k_BatchFeedForward 24 ///< Index of the kernel for Batch Normalization Feed Forward process (#CNeuronBathcNormOCL) #define def_k_bff_inputs 0 ///< Inputs data tenzor #define def_k_bff_options 1 ///< Tenzor of variables #define def_k_bff_output 2 ///< Tenzor of output data #define def_k_bff_batch 3 ///< Batch size #define def_k_bff_optimization 4 ///< Optimization type #define def_k_bff_activation 5 ///< Activation type //--- #define def_k_CalcHiddenGradientBatch 25 ///< Index of the Kernel of the Batch neuron to transfer gradient to previous layer (#CNeuronBathcNormOCL) #define def_k_bchg_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_bchg_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_bchg_matrix_i 2 ///<[in] Tensor of previous layer output #define def_k_bchg_matrix_ig 3 ///<[out] Tensor of gradients at previous layer #define def_k_bchg_activation 4 ///< Activation type (#ENUM_ACTIVATION) #define def_k_bchg_batch 5 ///< Batch size #define def_k_bchg_optimization 6 ///< Optimization type //--- #define def_k_UpdateBatchOptionsMomentum 26 ///< Index of the kernel for Describe the process of SGD optimization options for the Batch normalization Neuron (#CNeuronBatchNormOCL). #define def_k_buom_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_buom_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_buom_learning_rates 2 ///< Learning rates #define def_k_buom_momentum 3 ///< Momentum multiplier //--- #define def_k_UpdateBatchOptionsAdam 27 ///< Index of the kernel for Describe the process of Adam optimization options for the Batch normalization Neuron (#CNeuronBatchNormOCL). #define def_k_buoa_options 0 ///<[in] Options matrix m*(7 or 9), where m - Number of neurons in previous layer #define def_k_buoa_matrix_g 1 ///<[in] Tensor of gradients at current layer #define def_k_buoa_l 2 ///< Learning rates #define def_k_buoa_b1 3 ///< First momentum multiplier #define def_k_buoa_b2 4 ///< Second momentum multiplier ///@} ///\defgroup VAE neuron Kernels of Variant Aoutoencodre /// Describes the process of Variant Aoutoencodre. ///@{ #define def_k_VAEFeedForward 30 #define def_k_vaeff_inputs 0 #define def_k_vaeff_random 1 #define def_k_vaeff_outputd 2 //--- #define def_k_VAECalcHiddenGradient 31 #define def_k_vaehg_input 0 #define def_k_vaehg_inp_grad 1 #define def_k_vaehg_random 2 #define def_k_vaehg_gradient 3 #define def_k_vaehg_kld_mult 4 ///@} ///\defgroup LSTM neuron Kernels of RNN unit /// Describes the process of RNN. ///@{ #define def_k_LSTM_FeedForward 32 #define def_k_lstmff_inputs 0 #define def_k_lstmff_inputs_size 1 #define def_k_lstmff_weights 2 #define def_k_lstmff_concatenated 3 #define def_k_lstmff_memory 4 #define def_k_lstmff_outputs 5 //--- #define def_k_LSTM_ConcatenatedGradient 33 #define def_k_lstmcg_gradient 0 #define def_k_lstmcg_concatenated_gradient 1 #define def_k_lstmcg_memory 2 #define def_k_lstmcg_concatenated 3 //--- #define def_k_LSTM_HiddenGradient 34 #define def_k_lstmhg_concatenated_gradient 0 #define def_k_lstmhg_inputs_gradient 1 #define def_k_lstmhg_weights_gradient 2 #define def_k_lstmhg_hidden_state 3 #define def_k_lstmhg_inputs 4 #define def_k_lstmhg_weeights 5 #define def_k_lstmhg_output 6 #define def_k_lstmhg_hidden_size 7 #define def_k_lstmhg_inputs_size 8 //--- #define def_k_LSTM_UpdateWeightsAdam 35 #define def_k_lstmuw_weights 0 #define def_k_lstmuw_weights_gradient 1 #define def_k_lstmuw_matrix_m 2 #define def_k_lstmuw_matrix_v 3 #define def_k_lstmuw_l 4 #define def_k_lstmuw_b1 5 #define def_k_lstmuw_b2 6 ///@} ///\defgroup SoftMax activation Kernels ///@{ #define def_k_SoftMax_FeedForward 36 #define def_k_softmaxff_inputs 0 #define def_k_softmaxff_outputs 1 //--- #define def_k_SoftMax_HiddenGradient 37 #define def_k_softmaxhg_outputs 0 #define def_k_softmaxhg_output_gr 1 #define def_k_softmaxhg_input_gr 2 //--- #define def_k_SoftMax_OutputGradient 38 #define def_k_softmaxog_outputs 0 #define def_k_softmaxog_targets 1 #define def_k_softmaxog_output_gr 2 ///@} ///\defgroup SoftMax activation Kernels ///@{ #define def_k_FQF_Cosine 39 #define def_k_fqf_cosine_softmax 0 #define def_k_fqf_cosine_outputs 1 //--- #define def_k_FQF_Output 40 #define def_k_fqfout_quantiles 0 #define def_k_fqfout_delta_taus 1 #define def_k_fqfout_output 2 #define def_k_fqfout_total 3 //--- #define def_k_FQF_OutputGradient 41 #define def_k_fqfoutgr_quantiles 0 #define def_k_fqfoutgr_taus 1 #define def_k_fqfoutgr_output_gr 2 #define def_k_fqfoutgr_quantiles_gr 3 #define def_k_fqfoutgr_taus_gr 4 //--- #define def_k_FQF_QuantileGradient 42 #define def_k_fqfqgr_state_enbeding 0 #define def_k_fqfqgr_taus_embedding 1 #define def_k_fqfqgr_quantiles_gr 2 #define def_k_fqfqgr_state_gr 3 #define def_k_fqfqgr_taus_gr 4 //--- #define def_k_FQF_CosineGradient 43 #define def_k_fqfcosgr_softmax 0 #define def_k_fqfcosgr_output_gr 1 #define def_k_fqfcosgr_softmax_gr 2 //--- //--- #define def_k_MHSparseAttentionScore 44 ///< Index of the kernel of the multi-heads sparse attention neuron to calculate score matrix (#MHSparseAttentionScore) #define def_k_mhas_sparse 3 ///< less than 1.0 сoefficient of sparse //--- #define def_k_MHSparseAttentionOut 45 ///< Index of the kernel of the multi-heads sparse attention neuron to calculate multi-heads out matrix (#MHSparseAttentionOut) //--- #define def_k_FFMultiModels 46 ///< Index of the kernel of the multi-models neuron to calculate feed forward #define def_k_HGMultiModels 47 ///< Index of the kernel of the multi-models neuron to calculate hiden gradient #define def_k_chg_model 6 ///< Number of model to calculate #define def_k_UWMultiModels 48 ///< Index of the kernel of the multi-models neuron to update weights #define def_k_uwa_model 9 ///< Number of model to update ///@} ///@} #define def_k_ConcatFeedForward 49 ///< Index of #FeedForward kernel #define def_k_cff_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in layer and n - number of outputs (neurons in next layer) #define def_k_cff_matrix_i1 1 ///< Inputs tesor #define def_k_cff_matrix_i2 2 ///< Inputs tesor #define def_k_cff_matrix_o 3 ///< Output tensor #define def_k_cff_inputs1 4 ///< Number of inputs #define def_k_cff_inputs2 5 ///< Number of inputs #define def_k_cff_activation 6 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_ConcatCalcHiddenGradient 50 ///< Index of Hidden gradients calculation kernel (#CalcHiddenGradient) #define def_k_cchg_matrix_w 0 ///< Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cchg_matrix_g 1 ///< Tensor of gradients at current layer #define def_k_cchg_matrix_o1 2 ///< Output tensor #define def_k_cchg_matrix_o2 3 ///< Output tensor #define def_k_cchg_matrix_ig1 4 ///< Tensor of gradients at previous layer #define def_k_cchg_matrix_ig2 5 ///< Tensor of gradients at previous layer #define def_k_cchg_outputs 6 ///< Number of outputs #define def_k_cchg_inputs1 7 ///< Number of inputs1 #define def_k_cchg_inputs2 8 ///< Number of inputs2 #define def_k_cchg_activation1 9 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cchg_activation2 10 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_ConcatUpdWeightsMomentum 51 ///< Index SGD optomization Update weights kernel (#UpdateWeightsMomentum) #define def_k_cuwm_matrix_w 0 ///< SGD Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cuwm_matrix_g 1 ///< SGD Tensor of gradients at current layer #define def_k_cuwm_matrix_i1 2 ///< SGD Inputs tesor #define def_k_cuwm_matrix_i2 3 ///< SGD Inputs tesor #define def_k_cuwm_matrix_dw 4 ///< SGD Matrix of delta weights in last correction #define def_k_cuwm_inputs1 5 ///< SGD Number of inputs #define def_k_cuwm_inputs2 6 ///< SGD Number of inputs #define def_k_cuwm_learning_rates 7 ///< SGD Learning rates #define def_k_cuwm_momentum 8 ///< SGD Momentum multiplier //--- #define def_k_ConcatUpdWeightsAdam 52 ///< Index Adam optomization Update weights kernel (#UpdateWeightsAdam) #define def_k_cuwa_matrix_w 0 ///< Adam Weights matrix (m+1)*n, where m - number of neurons in previous layer and n - number of neurons in current layer #define def_k_cuwa_matrix_g 1 ///< Adam Tensor of gradients at current layer #define def_k_cuwa_matrix_i1 2 ///< Adam Inputs tesor #define def_k_cuwa_matrix_i2 3 ///< Adam Inputs tesor #define def_k_cuwa_matrix_m 4 ///< Adam Matrix of first momentum #define def_k_cuwa_matrix_v 5 ///< Adam Matrix of seconfd momentum #define def_k_cuwa_inputs1 6 ///< Adam Number of inputs #define def_k_cuwa_inputs2 7 ///< Adam Number of inputs #define def_k_cuwa_l 8 ///< Adam Learning rates #define def_k_cuwa_b1 9 ///< Adam First momentum multiplier #define def_k_cuwa_b2 10 ///< Adam Second momentum multiplier //--- #define def_k_SoftUpdate 53 #define def_k_su_target 0 #define def_k_su_source 1 #define def_k_su_tau 2 //--- #define def_k_SoftUpdateAdam 54 #define def_k_sua_target 0 #define def_k_sua_source 1 #define def_k_sua_matrix_m 2 #define def_k_sua_matrix_v 3 #define def_k_sua_tau 4 #define def_k_sua_b1 5 #define def_k_sua_b2 6 //--- #define def_k_SAC_AlphaLogProbs 55 #define def_k_sac_alp_outputs 0 #define def_k_sac_alp_quantiles 1 #define def_k_sac_alp_probs 2 #define def_k_sac_alp_alphas 3 #define def_k_sac_alp_log_probs 4 #define def_k_sac_alp_random 5 #define def_k_sac_alp_count_quants 6 #define def_k_sac_alp_activation 7 //--- #define def_k_SAC_AlphaGradients 56 #define def_k_sac_alg_outputs 0 #define def_k_sac_alg_gradient 1 #define def_k_sac_alg_log_probs 2 #define def_k_sac_alg_alphas_grad 3 #define def_k_sac_alg_activation 4 //--- #define def_k_SAC_OutputGradient 57 #define def_k_sacoutgr_quantiles 0 #define def_k_sacoutgr_taus 1 #define def_k_sacoutgr_output_gr 2 #define def_k_sacoutgr_quantiles_gr 3 #define def_k_sacoutgr_taus_gr 4 #define def_k_sacoutgr_outputs 5 #define def_k_sacoutgr_count_quants 6 #define def_k_sacoutgr_activation 7 //--- #define def_k_SAC_CalcLogProbs 58 #define def_k_sacclp_outputs 0 #define def_k_sacclp_quantiles 1 #define def_k_sacclp_probs 2 #define def_k_sacclp_alphas 3 #define def_k_sacclp_log_probs 4 #define def_k_sacclp_count_quants 5 #define def_k_sacclp_activation 6 //--- #define def_k_Embedding 59 #define def_k_emb_inputs 0 #define def_k_emb_outputs 1 #define def_k_emb_weights 2 #define def_k_emb_windows 3 #define def_k_emb_std 4 #define def_k_emb_stack_size 5 //--- #define def_k_EmbeddingHiddenGradient 60 #define def_k_ehg_inputs_gradient 0 #define def_k_ehg_outputs_gradient 1 #define def_k_ehg_weights 2 #define def_k_ehg_windows 3 #define def_k_ehg_std 4 #define def_k_ehg_window_out 5 //--- #define def_k_EmbeddingUpdateWeightsAdam 61 #define def_k_euw_weights 0 #define def_k_euw_gradient 1 #define def_k_euw_inputs 2 #define def_k_euw_matrix_m 3 #define def_k_euw_matrix_v 4 #define def_k_euw_windows 5 #define def_k_euw_std 6 #define def_k_euw_window_out 7 #define def_k_euw_learning_rate 8 #define def_k_euw_b1 9 #define def_k_euw_b2 10 //--- #define def_k_Transpose 62 #define def_k_tr_matrix_in 0 #define def_k_tr_matrix_out 1 //--- #define def_k_MH2AttentionOut 63 #define def_k_mh2ao_q 0 ///< Matrix of Queries #define def_k_mh2ao_kv 1 ///< Matrix of Keys, Values #define def_k_mh2ao_score 2 ///< Matrix of Scores #define def_k_mh2ao_out 3 ///< Matrix of Outputs #define def_k_mh2ao_dimension 4 ///< Dimension of Key #define def_k_mh2ao_heads_kv 5 ///< Heads of Key #define def_k_mh2ao_mask 6 ///< mask 1 - calc only previous units, 0 - calc all //--- #define def_k_MH2AttentionInsideGradients 64 #define def_k_mh2aig_q 0 ///< Matrix of Queries #define def_k_mh2aig_qg 1 ///< Matrix of Queries gradient #define def_k_mh2aig_kv 2 ///< Matrix of Keys, Values #define def_k_mh2aig_kvg 3 ///< Matrix of Keys, Values gradient #define def_k_mh2aig_score 4 ///< Matrix of Scores #define def_k_mh2aig_outg 5 ///< Matrix of Outputs gradient #define def_k_mh2aig_kunits 6 ///< Size of Key #define def_k_mh2aig_heads_kv 7 ///< Heads of Key //--- #define def_k_CGConv_HiddenGradient 65 #define def_k_cgc_matrix_g 0 ///<[in] Tensor of gradients at current layer #define def_k_cgc_matrix_f 1 ///<[in] Previous layer Output tensor #define def_k_cgc_matrix_s 2 ///<[in] Previous layer Output tensor #define def_k_cgc_matrix_fg 3 ///<[out] Tensor of gradients at previous layer #define def_k_cgc_matrix_sg 4 ///<[out] Tensor of gradients at previous layer #define def_k_cgc_activationf 5 ///< Activation type (#ENUM_ACTIVATION) #define def_k_cgc_activations 6 ///< Activation type (#ENUM_ACTIVATION) //--- #define def_k_XCiTFeedForward 66 #define def_k_XCiTff_qkv 0 #define def_k_XCiTff_score 1 #define def_k_XCiTff_out 2 //--- #define def_k_XCiTInsideGradients 67 #define def_k_XCiTig_qkv 0 #define def_k_XCiTig_qkv_g 1 #define def_k_XCiTig_scores 2 #define def_k_XCiTig_gradient 3 //--- #define def_k_DOTFeedForward 68 #define def_k_dot_qkv 0 #define def_k_dot_score 1 #define def_k_dot_rpb 2 #define def_k_dot_out 3 //--- #define def_k_DOTInsideGradients 69 #define def_k_dotg_qkv 0 #define def_k_dotg_qkv_g 1 #define def_k_dotg_scores 2 #define def_k_dotg_rpb 3 #define def_k_dotg_rpb_g 4 #define def_k_dotg_gradient 5 //--- #define def_k_RPBUpdateAdam 70 #define def_k_rpbw_rpb 0 #define def_k_rpbw_gradient 1 #define def_k_rpbw_matrix_m 2 #define def_k_rpbw_matrix_v 3 #define def_k_rpbw_b1 4 #define def_k_rpbw_b2 5 //--- #define def_k_GTEFeedForward 71 #define def_k_gteff_qkv 0 #define def_k_gteff_score 1 #define def_k_gteff_out 2 #define def_k_gteff_dimension 3 //--- #define def_k_GTEInsideGradients 72 #define def_k_gteig_qkv 0 #define def_k_gteig_qkv_g 1 #define def_k_gteig_scores 2 #define def_k_gteig_gradient 3 //--- #define def_k_FeedForwardNODEInpK 73 #define def_k_ffdopriInp_matrix_i 0 #define def_k_ffdopriInp_matrix_k1 1 #define def_k_ffdopriInp_matrix_k2 2 #define def_k_ffdopriInp_matrix_k3 3 #define def_k_ffdopriInp_matrix_k4 4 #define def_k_ffdopriInp_matrix_k5 5 #define def_k_ffdopriInp_matrix_k6 6 #define def_k_ffdopriInp_matrix_beta 7 #define def_k_ffdopriInp_matrix_o 8 //--- #define def_k_FeedForwardNODEF 74 #define def_k_ffdoprif_matrix_w 0 #define def_k_ffdoprif_matrix_i 1 #define def_k_ffdoprif_matrix_o 2 #define def_k_ffdoprif_dimension 3 #define def_k_ffdoprif_step 4 #define def_k_ffdoprif_activation 5 //--- #define def_k_HiddenGradientNODEInpK 75 //--- #define def_k_HiddenGradientNODEF 76 #define def_k_hddoprif_matrix_w 0 #define def_k_hddoprif_matrix_g 1 #define def_k_hddoprif_matrix_i 2 #define def_k_hddoprif_matrix_ig 3 #define def_k_hddoprif_dimension_out 4 #define def_k_hddoprif_activation 5 //--- #define def_k_NODEF_UpdateWeightsAdam 77 #define def_k_uwdoprif_matrix_w 0 #define def_k_uwdoprif_matrix_gk1 1 #define def_k_uwdoprif_matrix_gk2 2 #define def_k_uwdoprif_matrix_gk3 3 #define def_k_uwdoprif_matrix_gk4 4 #define def_k_uwdoprif_matrix_gk5 5 #define def_k_uwdoprif_matrix_gk6 6 #define def_k_uwdoprif_matrix_ik1 7 #define def_k_uwdoprif_matrix_ik2 8 #define def_k_uwdoprif_matrix_ik3 9 #define def_k_uwdoprif_matrix_ik4 10 #define def_k_uwdoprif_matrix_ik5 11 #define def_k_uwdoprif_matrix_ik6 12 #define def_k_uwdoprif_matrix_m 13 #define def_k_uwdoprif_matrix_v 14 #define def_k_uwdoprif_alpha 15 #define def_k_uwdoprif_lenth 16 #define def_k_uwdoprif_l 17 #define def_k_uwdoprif_b1 18 #define def_k_uwdoprif_b2 19 //--- #define def_k_TimeDerivative 78 #define def_k_tdqkv 0 #define def_k_tddqkv 1 #define def_k_tddimension 2 //--- #define def_k_HGTimeDerivative 79 //--- #define def_k_FeedForwardContAtt 80 #define def_k_caqkv 0 #define def_k_cadqkv 1 #define def_k_cascore 2 #define def_k_caout 3 #define def_k_cadimension 4 #define def_k_caheads 5 //--- #define def_k_HiddenGradientContAtt 81 #define def_k_hgcaqkv 0 #define def_k_hgcaqkv_g 1 #define def_k_hgcadqkv 2 #define def_k_hgcadqkv_g 3 #define def_k_hgcascore 4 #define def_k_hgcaout_g 5 #define def_k_hgcadimension 6 //--- #define def_k_RevInFeedForward 82 #define def_k_revffinputs 0 #define def_k_revffoptions 1 #define def_k_revffoutput 2 #define def_k_revffoptions_size 3 #define def_k_revffoptimization 4 //--- #define def_k_RevInHiddenGraddient 83 #define def_k_revhginputs 0 #define def_k_revhginputs_gr 1 #define def_k_revhgoptions 2 #define def_k_revhgoutput_gr 3 #define def_k_revhgoptions_size 4 #define def_k_revhgoptimization 5 #define def_k_revhgactivation 6 //--- #define def_k_DeActivation 84 #define def_k_deact_inputs 0 #define def_k_deact_inputs_gr 1 #define def_k_deact_output_gr 2 #define def_k_deact_activation 3 //--- #define def_k_PatchCreate 85 #define def_k_ptc_inputs 0 #define def_k_ptc_weights 1 #define def_k_ptc_outputs 2 #define def_k_ptc_inputs_total 3 #define def_k_ptc_window_in 4 #define def_k_ptc_step 5 #define def_k_ptc_activation 6 //--- #define def_k_PatchHiddenGradient 86 #define def_k_pthg_inputs 0 #define def_k_pthg_inputs_gr 1 #define def_k_pthg_weights 2 #define def_k_pthg_outputs_gr 3 #define def_k_pthg_window_in 4 #define def_k_pthg_step 5 #define def_k_pthg_window_out 6 #define def_k_pthg_outputs_total 7 #define def_k_pthg_activation 8 //--- #define def_k_PatchUpdateWeightsAdam 87 #define def_k_ptuwa_weights 0 #define def_k_ptuwa_outputs_gr 1 #define def_k_ptuwa_inputs 2 #define def_k_ptuwa_weights_m 3 #define def_k_ptuwa_weights_v 4 #define def_k_ptuwa_inputs_total 5 #define def_k_ptuwa_l 6 #define def_k_ptuwa_b1 7 #define def_k_ptuwa_b2 8 #define def_k_ptuwa_step 9 //--- #define def_k_MatMult 88 #define def_k_mm_matr1 0 #define def_k_mm_matr2 1 #define def_k_mm_result 2 #define def_k_mm_dimension 3 #define def_k_mm_multvarsecond 4 //--- #define def_k_FFT 89 #define def_k_fft_inputs_re 0 #define def_k_fft_inputs_im 1 #define def_k_fft_outputs_re 2 #define def_k_fft_outputs_im 3 #define def_k_fft_input_window 4 #define def_k_fft_input_complex 5 #define def_k_fft_output_window 6 #define def_k_fft_reverse 7 //--- #define def_k_ComplexLayer 90 #define def_k_cl_inputs_re 0 #define def_k_cl_inputs_im 1 #define def_k_cl_outputs_re 2 #define def_k_cl_outputs_im 3 //--- #define def_k_ComplexLayerGradient 91 //--- #define def_k_GradientMSA 92 #define def_k_gmsa_target 0 #define def_k_gmsa_forecast 1 #define def_k_gmsa_gradient 2 //--- #define def_k_CumulativeGradient 93 #define def_k_cg_gradient1 0 #define def_k_cg_gradient2 1 #define def_k_cg_gradient_out 2 #define def_k_cg_alpha 3 //--- #define def_k_FeedForwardComplexConv 94 #define def_k_CalcHiddenGradientComplexConv 95 #define def_k_UpdateWeightsComplexConvMomentum 96 #define def_k_UpdateWeightsComplexConvAdam 97 #define def_k_ComplexMHAttentionGradients 98 #define def_k_ComplexMHAttentionScore 99 #define def_k_ComplexMHAttentionOut 100 #define def_k_ComplexSoftMax_FeedForward 101 #define def_k_ComplexSoftMax_HiddenGradient 102 #define def_k_ComplexSoftMax_OutputGradient 103 //--- #define def_k_ComplexNormalize 104 #define def_k_cn_inputs 0 #define def_k_cn_outputs 1 #define def_k_cn_means 2 #define def_k_cn_vars 3 #define def_k_cn_dimension 4 //--- #define def_k_ComplexUnNormalize 105 //--- #define def_k_ComplexNormalizeGradient 106 #define def_k_cng_inputs_gr 0 #define def_k_cng_outputs_gr 1 #define def_k_cng_vars 2 #define def_k_cng_dimension 3 //--- #define def_k_ComplexUnNormalizeGradient 107 //--- #define def_k_MainFreqWeight 108 #define def_k_mfw_freq 0 #define def_k_mfw_weight 1 #define def_k_mfw_dimension 2 //--- #define def_k_WeightedSum 109 #define def_k_ws_inputs1 0 #define def_k_ws_inputs2 1 #define def_k_ws_outputs 2 #define def_k_ws_weight 3 #define def_k_ws_dimension 4 //--- #define def_k_WeightedSumGradient 110 //--- #define def_k_FeedForwardS3 111 #define def_k_s3_inputs 0 #define def_k_s3_probability 1 #define def_k_s3_weights 2 #define def_k_s3_outputs 3 #define def_k_s3_positions 4 #define def_k_s3_window 5 #define def_k_s3_total 6 //--- #define def_k_InsideGradientS3 112 #define def_k_s3g_inputs 0 #define def_k_s3g_inputs_gr 1 #define def_k_s3g_probability 2 #define def_k_s3g_probability_gr 3 #define def_k_s3g_weights 4 #define def_k_s3g_outputs_gr 5 #define def_k_s3g_positions 6 #define def_k_s3g_window 7 #define def_k_s3g_total 8 //--- #define def_k_WeightGradientS3 113 #define def_k_s3w_inputs 0 #define def_k_s3w_positions 1 #define def_k_s3w_outputs_gr 2 #define def_k_s3w_weights_gr 3 #define def_k_s3w_window 4 #define def_k_s3w_total 5 //--- #define def_k_MH2PyrAttentionOut 114 #define def_k_pam_q 0 #define def_k_pam_kv 1 #define def_k_pam_score 2 #define def_k_pam_out 3 #define def_k_pam_dimension 4 #define def_k_pam_heads_kv 5 #define def_k_pam_window 6 //--- #define def_k_PLR 115 #define def_k_plr_inputs 0 #define def_k_plr_outputs 1 #define def_k_plt_isttp 2 #define def_k_plr_transpose 3 #define def_k_plr_step 4 //--- #define def_k_PLRGrad 116 #define def_k_plrg_inputs_gr 0 #define def_k_plrg_outputs 1 #define def_k_plrg_outputs_gr 2 #define def_k_plrg_transpose 3 //--- #define def_k_UpdateWeightsAdamMini 117 #define def_k_wuam_matrix_w 0 #define def_k_wuam_matrix_g 1 #define def_k_wuam_matrix_i 2 #define def_k_wuam_matrix_m 3 #define def_k_wuam_matrix_v 4 #define def_k_wuam_l 5 #define def_k_wuam_b1 6 #define def_k_wuam_b2 7 //--- #define def_k_UpdateWeightsConvAdamMini 118 #define def_k_wucam_matrix_w 0 #define def_k_wucam_matrix_g 1 #define def_k_wucam_matrix_i 2 #define def_k_wucam_matrix_m 3 #define def_k_wucam_matrix_v 4 #define def_k_wucam_inputs 5 #define def_k_wucam_l 6 #define def_k_wucam_b1 7 #define def_k_wucam_b2 8 #define def_k_wucam_step 9 //--- #define def_k_CutTrendAndOther 119 #define def_k_ct_inputs 0 #define def_k_ct_plr 1 #define def_k_ct_trend 2 #define def_k_ct_other 3 //--- #define def_k_CutTrendAndOtherGradient 120 #define def_k_ctg_inputs_gr 0 #define def_k_ctg_plr 1 #define def_k_ctg_plr_gr 2 #define def_k_ctg_trend_gr 3 #define def_k_ctg_other_gr 4 //--- #define def_k_CutOneFromAnother 121 #define def_k_ctofa_inputs 0 #define def_k_ctofa_cut 1 #define def_k_ctofa_other 2 //--- #define def_k_CutOneFromAnotherGradient 122 //--- #define def_k_UniTrajPrepare 123 #define def_k_utp_history 0 #define def_k_utp_h_mask 1 #define def_k_utp_future 2 #define def_k_utp_f_mask 3 #define def_k_utp_output 4 #define def_k_utp_h_total 5 #define def_k_utp_f_total 6 //--- #define def_k_UniTrajPrepareGrad 124 #define def_k_utpg_history_gr 0 #define def_k_utpg_future_gr 1 #define def_k_utpg_output 2 #define def_k_utpg_output_gr 3 #define def_k_utpg_h_total 4 #define def_k_utpg_f_total 5 //--- #define def_k_UniTrajBTS 125 #define def_k_utbts_concat_inp 0 #define def_k_utbts_d_forw 1 #define def_k_utbts_d_bakw 2 #define def_k_utbts_total 3 //--- #define def_k_GateElementMul 126 #define def_k_gem_inputs1 0 #define def_k_gem_inputs2 1 #define def_k_gem_gate 2 #define def_k_gem_out 3 //--- #define def_k_GateElementMulGrad 127 #define def_k_gemg_inputs1 0 #define def_k_gemg_inputs1_gr 1 #define def_k_gemg_inputs2 2 #define def_k_gemg_inputs2_gr 3 #define def_k_gemg_gate 4 #define def_k_gemg_gate_gr 5 #define def_k_gemg_out 6 #define def_k_gemg_activ1 7 #define def_k_gemg_activ2 8 #define def_k_gemg_activ_gr 9 //--- #define def_k_HiVTPrepare 128 #define def_k_hivtp_data 0 #define def_k_hivtp_output 1 //--- #define def_k_TransposeRCD 129 //--- #define def_k_MatMultGrad 130 #define def_k_mmg_matr1 0 #define def_k_mmg_matr1_gr 1 #define def_k_mmg_matr2 2 #define def_k_mmg_matr2_gr 3 #define def_k_mmg_result_gr 4 #define def_k_mmg_dimension 5 #define def_k_mmg_multvarsecond 6 //--- #define def_k_OrthoganalLoss 131 #define def_k_ol_data 0 #define def_k_ol_grad 1 #define def_k_ol_add 2 //--- #define def_k_CalcDistance 132 #define def_k_cd_data 0 #define def_k_cd_distance 1 #define def_k_cd_dimension 2 //--- #define def_k_FeedForwardLocalMax 133 #define def_k_fflm_matrix_i 0 #define def_k_fflm_distance 1 #define def_k_fflm_matrix_o 2 #define def_k_fflm_radius 3 //--- #define def_k_CalcInputGradientLocalMax 134 #define def_k_cglm_matrix_i 0 #define def_k_cglm_matrix_ig 1 #define def_k_cglm_distance 2 #define def_k_cglm_matrix_o 3 #define def_k_cglm_matrix_g 4 #define def_k_cglm_radius 5 //--- #define def_k_MHMaskAttentionOut 135 #define def_k_mask_at_q 0 #define def_k_mask_at_kv 1 #define def_k_mask_at_score 2 #define def_k_mask_at_mask 3 #define def_k_mask_at_out 4 #define def_k_mask_at_dimension 5 #define def_k_mask_at_heads_kv 6 #define def_k_mask_at_mask_level 7 //--- #define def_k_MHMaskAttentionInsideGradients 136 #define def_k_mask_atg_q 0 #define def_k_mask_atg_q_g 1 #define def_k_mask_atg_kv 2 #define def_k_mask_atg_kv_g 3 #define def_k_mask_atg_mask 4 #define def_k_mask_atg_mask_g 5 #define def_k_mask_atg_scores 6 #define def_k_mask_atg_gradient 7 #define def_k_mask_atg_kunits 8 #define def_k_mask_atg_heads_kv 9 #define def_k_mask_atg_mask_level 10 //--- #define def_k_CalcPositionBias 137 #define def_k_cpb_data1 0 #define def_k_cpb_data2 1 #define def_k_cpb_result 2 #define def_k_cpb_dimension 3 //--- #define def_k_MHPosBiasAttentionOut 138 #define def_k_pbao_q 0 #define def_k_pbao_k 1 #define def_k_pbao_v 2 #define def_k_pbao_score 3 #define def_k_pbao_pos_bias 4 #define def_k_pbao_out 5 #define def_k_pbao_dimension 6 #define def_k_pbao_heads_kv 7 #define def_k_pbao_use_pos_bias 8 //--- #define def_k_MHPosBiasAttentionInsideGradients 139 #define def_k_pbaog_q 0 #define def_k_pbaog_q_g 1 #define def_k_pbaog_k 2 #define def_k_pbaog_k_g 3 #define def_k_pbaog_v 4 #define def_k_pbaog_v_g 5 #define def_k_pbaog_scores 6 #define def_k_pbaog_gradient 7 #define def_k_pbaog_kunits 8 #define def_k_pbaog_heads_kv 9 //--- #define def_k_DiversityLoss 140 #define def_k_dl_data 0 #define def_k_dl_grad 1 #define def_k_dl_dimension 2 #define def_k_dl_activation 3 #define def_k_dl_add 4 //--- #define def_k_MHRelativeAttentionOut 141 #define def_k_rat_q 0 #define def_k_rat_k 1 #define def_k_rat_v 2 #define def_k_rat_bk 3 #define def_k_rat_bv 4 #define def_k_rat_gc 5 #define def_k_rat_gp 6 #define def_k_rat_score 7 #define def_k_rat_out 8 #define def_k_rat_dimension 9 //--- #define def_k_MHRelativeAttentionInsideGradients 142 #define def_k_ratg_q 0 #define def_k_ratg_q_g 1 #define def_k_ratg_k 2 #define def_k_ratg_k_g 3 #define def_k_ratg_v 4 #define def_k_ratg_v_g 5 #define def_k_ratg_bk 6 #define def_k_ratg_bk_g 7 #define def_k_ratg_bv 8 #define def_k_ratg_bv_g 9 #define def_k_ratg_gc 10 #define def_k_ratg_gc_g 11 #define def_k_ratg_gp 12 #define def_k_ratg_gp_g 13 #define def_k_ratg_scores 14 #define def_k_ratg_gradient 15 #define def_k_ratg_kunits 16 //--- #define def_k_CalcAlignmentGradient 143 #define def_k_aliggr_matrix_o1 0 #define def_k_aliggr_matrix_o2 1 #define def_k_aliggr_matrix_g1 2 #define def_k_aliggr_matrix_g2 3 #define def_k_aliggr_activation 4 #define def_k_aliggr_add 5 //--- #define def_k_FeatureSmoothing 144 #define def_k_fs_feature 0 #define def_k_fs_outputs 1 #define def_k_fs_smoothing 2 //--- #define def_k_FeatureSmoothingGradient 145 //--- #define def_k_BatchFeedForwardAddNoise 146 #define def_k_normwithnoise_inputs 0 #define def_k_normwithnoise_options 1 #define def_k_normwithnoise_noise 2 #define def_k_normwithnoise_output 3 #define def_k_normwithnoise_batch 4 #define def_k_normwithnoise_optimization 5 #define def_k_normwithnoise_activation 6 #define def_k_normwithnoise_alpha 7 //--- #define def_k_HyperProjection 147 #define def_k_lp_inputs 0 #define def_k_lp_outputs 1 //--- #define def_k_HyperProjectionGrad 148 #define def_k_lpg_inputs 0 #define def_k_lpg_inputs_gr 1 #define def_k_lpg_outputs_gr 2 //--- #define def_k_LogMap 149 #define def_k_logmap_features 0 #define def_k_logmap_centroids 1 #define def_k_logmap_curvatures 2 #define def_k_logmap_outputs 3 #define def_k_logmap_product 4 #define def_k_logmap_distance 5 #define def_k_logmap_norma 6 //--- #define def_k_LogMapGrad 150 #define def_k_logmapgr_features 0 #define def_k_logmapgr_features_gr 1 #define def_k_logmapgr_centroids 2 #define def_k_logmapgr_centroids_gr 3 #define def_k_logmapgr_curvatures 4 #define def_k_logmapgr_curvatures_gr 5 #define def_k_logmapgr_outputs 6 #define def_k_logmapgr_outputs_gr 7 #define def_k_logmapgr_product 8 #define def_k_logmapgr_distance 9 #define def_k_logmapgr_norma 10 //--- #define def_k_CalcEpsilonWeights 151 #define def_k_epsw_matrix_w 0 #define def_k_epsw_matrix_g 1 #define def_k_epsw_matrix_i 2 #define def_k_epsw_matrix_epsw 3 #define def_k_epsw_rho 4 //--- #define def_k_CalcEpsilonWeightsConv 152 #define def_k_epswconv_matrix_w 0 #define def_k_epswconv_matrix_g 1 #define def_k_epswconv_matrix_i 2 #define def_k_epswconv_matrix_epsw 3 #define def_k_epswconv_inputs 4 #define def_k_epswconv_rho 5 #define def_k_epswconv_step 6 //--- #define def_k_PLRMultiAgents 153 //--- #define def_k_PLRMultiAgentsGrad 154 #define def_k_plrg_agents 4 //--- #define def_k_FeedForwardMHConv 155 //--- #define def_k_CalcHiddenGradientMHConv 156 #define def_k_chgc_heads 10 //--- #define def_k_UpdateWeightsMHConvAdam 157 #define def_k_uwca_heads 12 //--- #define def_k_MoreLessEqual 158 #define def_k_mle_inputs 0 #define def_k_mle_outputs 1 //--- #define def_k_MultiScaleRelativeAttentionOut 159 //--- #define def_k_SSM2D_FeedForward 160 #define def_k_ssm2d_ah 0 #define def_k_ssm2d_b_time 1 #define def_k_ssm2d_b_var 2 #define def_k_ssm2d_px_time 3 #define def_k_ssm2d_px_var 4 #define def_k_ssm2d_c_time 5 #define def_k_ssm2d_c_var 6 #define def_k_ssm2d_delta_time 7 #define def_k_ssm2d_delta_var 8 #define def_k_ssm2d_hidden 9 #define def_k_ssm2d_y 10 //--- #define def_k_SSM2D_CalcHiddenGradient 161 #define def_k_ssm2dhg_ah 0 #define def_k_ssm2dhg_grad_ah 1 #define def_k_ssm2dhg_b_time 2 #define def_k_ssm2dhg_grad_b_time 3 #define def_k_ssm2dhg_b_var 4 #define def_k_ssm2dhg_grad_b_var 5 #define def_k_ssm2dhg_px_time 6 #define def_k_ssm2dhg_grad_px_time 7 #define def_k_ssm2dhg_px_var 8 #define def_k_ssm2dhg_grad_px_var 9 #define def_k_ssm2dhg_c_time 10 #define def_k_ssm2dhg_grad_c_time 11 #define def_k_ssm2dhg_c_var 12 #define def_k_ssm2dhg_grad_c_var 13 #define def_k_ssm2dhg_delta_time 14 #define def_k_ssm2dhg_grad_delta_time 15 #define def_k_ssm2dhg_delta_var 16 #define def_k_ssm2dhg_grad_delta_var 17 #define def_k_ssm2dhg_hidden 18 #define def_k_ssm2dhg_grad_y 19 //--- #define def_k_PScan 162 #define def_k_psc_A 0 #define def_k_psc_X 1 #define def_k_psc_H 2 #define def_k_psc_X_out 3 //--- #define def_k_PScan_CalcHiddenGradient 163 #define def_k_pscgr_A 0 #define def_k_pscgr_X 1 #define def_k_pscgr_H 2 #define def_k_pscgr_grad_X_out 3 #define def_k_pscgr_grad_A 4 #define def_k_pscgr_grad_X 5 #define def_k_pscgr_grad_H 6 //--- #define def_k_DiagMatMult 164 #define def_k_diagmm_diag 0 #define def_k_diagmm_matr 1 #define def_k_diagmm_result 2 #define def_k_diagmm_activation 3 //--- #define def_k_DiagMatMultGrad 165 #define def_k_diagmmgr_diag 0 #define def_k_diagmmgr_grad_diag 1 #define def_k_diagmmgr_matr 2 #define def_k_diagmmgr_grad_matr 3 #define def_k_diagmmgr_grad_result 4 //--- #define def_k_TopKgates 166 #define def_k_topK_inputs 0 #define def_k_topK_noises 1 #define def_k_topK_gates 2 #define def_k_topK_k 3 //--- #define def_k_TopKgatesGrad 167 #define def_k_topKgr_inputs 0 #define def_k_topKgr_grad_inputs 1 #define def_k_topKgr_noises 2 #define def_k_topKgr_gates 3 #define def_k_topKgr_grad_gates 4 //--- #define def_k_MaskByDistance 168 #define def_k_maskDist_buf_real 0 #define def_k_maskDist_buf_imag 1 #define def_k_maskDist_mask 2 #define def_k_maskDist_dimension 3 //--- #define def_k_MaskAttention 169 #define def_k_maskatt_q 0 #define def_k_maskatt_kv 1 #define def_k_maskatt_scores 2 #define def_k_maskatt_masks 3 #define def_k_maskatt_out 4 #define def_k_maskatt_dimension 5 #define def_k_maskatt_heads_kv 6 //--- #define def_k_MaskAttentionGradients 170 #define def_k_maskattgr_q 0 #define def_k_maskattgr_q_g 1 #define def_k_maskattgr_kv 2 #define def_k_maskattgr_kv_g 3 #define def_k_maskattgr_scores 4 #define def_k_maskattgr_gradient 5 #define def_k_maskattgr_kunits 6 #define def_k_maskattgr_heads_kv 7 //--- #define def_k_FeedForwardMultWinConv 171 #define def_k_ffmwc_matrix_w 0 #define def_k_ffmwc_matrix_i 1 #define def_k_ffmwc_matrix_o 2 #define def_k_ffmwc_windows_in 3 #define def_k_ffmwc_inputs 4 #define def_k_ffmwc_windows_total 5 #define def_k_ffmwc_window_out 6 #define def_k_ffmwc_activation 7 //--- #define def_k_CalcHiddenGradientMultWinConv 172 #define def_k_chgmwc_matrix_w 0 #define def_k_chgmwc_matrix_i 1 #define def_k_chgmwc_matrix_ig 2 #define def_k_chgmwc_matrix_og 3 #define def_k_chgmwc_windows_in 4 #define def_k_chgmwc_outputs 5 #define def_k_chgmwc_windows_total 6 #define def_k_chgmwc_window_out 7 #define def_k_chgmwc_activation 8 //--- #define def_k_UpdateWeightsMultWinConvAdam 173 #define def_k_uwmwc_matrix_w 0 #define def_k_uwmwc_matrix_og 1 #define def_k_uwmwc_matrix_i 2 #define def_k_uwmwc_matrix_m 3 #define def_k_uwmwc_matrix_v 4 #define def_k_uwmwc_windows_in 5 #define def_k_uwmwc_windows_total 6 #define def_k_uwmwc_window_out 7 #define def_k_uwmwc_inputs 8 #define def_k_uwmwc_outputs 9 #define def_k_uwmwc_l 10 #define def_k_uwmwc_b1 11 #define def_k_uwmwc_b2 12 //--- #define def_k_MaskAttentionComplex 174 #define def_k_maskattcom_q 0 #define def_k_maskattcom_kv 1 #define def_k_maskattcom_scores 2 #define def_k_maskattcom_masks 3 #define def_k_maskattcom_out 4 #define def_k_maskattcom_dimension 5 #define def_k_maskattcom_heads_kv 6 //--- #define def_k_MaskAttentionGradientsComplex 175 #define def_k_maskattcomgr_q 0 #define def_k_maskattcomgr_q_g 1 #define def_k_maskattcomgr_kv 2 #define def_k_maskattcomgr_kv_g 3 #define def_k_maskattcomgr_scores 4 #define def_k_maskattcomgr_mask 5 #define def_k_maskattcomgr_mask_g 6 #define def_k_maskattcomgr_gradient 7 #define def_k_maskattcomgr_kunits 8 #define def_k_maskattcomgr_heads_kv 9 //--- #define def_k_CSLSTM_FeedForward 176 #define def_k_cslstmff_concatenated 0 #define def_k_cslstmff_memory 1 #define def_k_cslstmff_output 2 //--- #define def_k_CSLSTM_CalcHiddenGradient 177 #define def_k_cslstmhg_concatenated 0 #define def_k_cslstmhg_concatenated_grad 1 #define def_k_cslstmhg_memory 2 #define def_k_cslstmhg_output_grad 3 //--- #define def_k_ProbAttentionQeuryImp 178 #define def_k_probat_querys 0 #define def_k_probat_keys_values 1 #define def_k_probat_index_keys 2 #define def_k_probat_querys_imp 3 #define def_k_probat_dimension 4 //--- #define def_k_TopKImportanceToIndex 179 #define def_k_imptoind_importance 0 #define def_k_imptoind_indexes 1 #define def_k_imptoind_tok_k 2 //--- #define def_k_QIndexAttention 180 #define def_k_indatt_q 0 #define def_k_indatt_kv 1 #define def_k_indatt_scores 2 #define def_k_indatt_indexes 3 #define def_k_indatt_out 4 #define def_k_indatt_dimension 5 #define def_k_indatt_heads_kv 6 //--- #define def_k_QIndexAttentionGradients 181 #define def_k_indattgr_q 0 #define def_k_indattgr_q_g 1 #define def_k_indattgr_kv 2 #define def_k_indattgr_kv_g 3 #define def_k_indattgr_indexes 4 #define def_k_indattgr_scores 5 #define def_k_indattgr_gradient 6 #define def_k_indattgr_kunits 7 #define def_k_indattgr_heads_kv 8 //--- #define def_k_TSPositonEncoder 182 #define def_k_tspe_data 0 #define def_k_tspe_time 1 #define def_k_tspe_output 2 #define def_k_tspe_period 3 //--- #define def_k_FeedForwardMultWinConvWPad 183 #define def_k_ffmwconvwpad_matrix_w 0 #define def_k_ffmwconvwpad_matrix_i 1 #define def_k_ffmwconvwpad_matrix_o 2 #define def_k_ffmwconvwpad_windows_in 3 #define def_k_ffmwconvwpad_inputs 4 #define def_k_ffmwconvwpad_step 5 #define def_k_ffmwconvwpad_window_out 6 #define def_k_ffmwconvwpad_activation 7 //--- #define def_k_CalcHiddenGradientMultWinConvWPad 184 #define def_k_hgmwconvwpad_matrix_w 0 #define def_k_hgmwconvwpad_matrix_i 1 #define def_k_hgmwconvwpad_matrix_ig 2 #define def_k_hgmwconvwpad_matrix_og 3 #define def_k_hgmwconvwpad_windows_in 4 #define def_k_hgmwconvwpad_outputs 5 #define def_k_hgmwconvwpad_step 6 #define def_k_hgmwconvwpad_window_out 7 #define def_k_hgmwconvwpad_filters 8 #define def_k_hgmwconvwpad_activation 9 //--- #define def_k_UpdateWeightsMultWinConvAdamWPad 185 #define def_k_uwmwconvwpad_matrix_w 0 #define def_k_uwmwconvwpad_matrix_og 1 #define def_k_uwmwconvwpad_matrix_i 2 #define def_k_uwmwconvwpad_matrix_m 3 #define def_k_uwmwconvwpad_matrix_v 4 #define def_k_uwmwconvwpad_windows_in 5 #define def_k_uwmwconvwpad_windows_total 6 #define def_k_uwmwconvwpad_window_out 7 #define def_k_uwmwconvwpad_inputs 8 #define def_k_uwmwconvwpad_step 9 #define def_k_uwmwconvwpad_outputs 10 #define def_k_uwmwconvwpad_l 11 #define def_k_uwmwconvwpad_b1 12 #define def_k_uwmwconvwpad_b2 13 //--- #define def_k_ConcatDiff 186 #define def_k_concdiff_data 0 #define def_k_concdiff_output 1 #define def_k_concdiff_step 2 //--- #define def_k_FeedForwardMaskMultWinConv 187 #define def_k_ffmmwc_matrix_w 0 #define def_k_ffmmwc_matrix_i 1 #define def_k_ffmmwc_masks 2 #define def_k_ffmmwc_matrix_o 3 #define def_k_ffmmwc_inputs 4 #define def_k_ffmmwc_window_in 5 #define def_k_ffmmwc_windows_total 6 #define def_k_ffmmwc_activation 7 //--- #define def_k_CalcHiddenGradientMaskMultWinConv 188 #define def_k_chgmmwc_matrix_w 0 #define def_k_chgmmwc_matrix_i 1 #define def_k_chgmmwc_matrix_ig 2 #define def_k_chgmmwc_matrix_og 3 #define def_k_chgmmwc_masks 4 #define def_k_chgmmwc_masks_g 5 #define def_k_chgmmwc_outputs 6 #define def_k_chgmmwc_window_in 7 #define def_k_chgmmwc_window_out 8 #define def_k_chgmmwc_activation 9 //--- #define def_k_UpdateWeightsMaskMultWinConvAdam 189 #define def_k_uwmmwc_matrix_w 0 #define def_k_uwmmwc_matrix_og 1 #define def_k_uwmmwc_matrix_i 2 #define def_k_uwmmwc_masks 3 #define def_k_uwmmwc_matrix_m 4 #define def_k_uwmmwc_matrix_v 5 #define def_k_uwmmwc_windows_total 6 #define def_k_uwmmwc_inputs 7 #define def_k_uwmmwc_outputs 8 #define def_k_uwmmwc_l 9 #define def_k_uwmmwc_b1 10 #define def_k_uwmmwc_b2 11 //--- #define def_k_MainFreq 190 #define def_k_mf_freq_r 0 #define def_k_mf_freq_im 1 #define def_k_mf_main_freq 2 #define def_k_mf_dimension 3 //--- #define def_k_FeedForwardAdaptConv 191 #define def_k_ffac_matrix_w 0 #define def_k_ffac_matrix_i 1 #define def_k_ffac_matrix_o 2 #define def_k_ffac_main_freq 3 #define def_k_ffac_inputs 4 #define def_k_ffac_window_in 5 #define def_k_ffac_activation 6 //--- #define def_k_CalcHiddenGradientAdaptConv 192 #define def_k_chgac_matrix_w 0 #define def_k_chgac_matrix_i 1 #define def_k_chgac_matrix_ig 2 #define def_k_chgac_matrix_og 3 #define def_k_chgac_main_freq 4 #define def_k_chgac_outputs 5 #define def_k_chgac_window_in 6 #define def_k_chgac_window_out 7 #define def_k_chgac_activation 8 //--- #define def_k_UpdateWeightsAdaptConvAdam 193 #define def_k_uwac_matrix_w 0 #define def_k_uwac_matrix_og 1 #define def_k_uwac_matrix_i 2 #define def_k_uwac_matrix_m 3 #define def_k_uwac_matrix_v 4 #define def_k_uwac_main_freq 5 #define def_k_uwac_inputs 6 #define def_k_uwac_outputs 7 #define def_k_uwac_l 8 #define def_k_uwac_b1 9 #define def_k_uwac_b2 10 //--- #define def_k_RoPE 194 #define def_k_rope_inputs 0 #define def_k_rope_position_emb 1 #define def_k_rope_outputs 2 //--- #define def_k_CalcHiddenGradRoPE 195 //--- #define def_k_MatrixDif 196 #define def_k_dif_matrix1 0 #define def_k_dif_matrix2 1 #define def_k_dif_matrix_out 2 #define def_k_dif_multiplyer 3 #define def_k_dif_shift_in1 4 #define def_k_dif_shift_in2 5 #define def_k_dif_shift_out 6 //--- #define def_k_MatrixDifGrad 197 //--- #define def_k_IdentMatrixDif 198 #define def_k_iddif_matrix_in 0 #define def_k_iddif_matrix_out 1 #define def_k_iddif_multiplyer 2 #define def_k_iddif_shift_in 3 #define def_k_iddif_shift_out 4 //--- #define def_k_IdentMatrixDifGrad 199 //--- #define def_k_SumVecMatrix 200 #define def_k_svecmat_vector_in 0 #define def_k_svecmat_matrix_in 1 #define def_k_svecmat_matrix_out 2 #define def_k_svecmat_multiplyer 3 #define def_k_svecmat_shift_in1 4 #define def_k_svecmat_shift_in2 5 #define def_k_svecmat_shift_out 6 //--- #define def_k_SumVecMatrixGrad 201 //--- #define def_k_InterpolationAttention 202 #define def_k_ia_matrix_in 0 #define def_k_ia_W 1 #define def_k_ia_A 2 #define def_k_ia_GL 3 #define def_k_ia_Adj 4 #define def_k_ia_H 5 #define def_k_ia_Atten 6 #define def_k_ia_matrix_out 7 #define def_k_ia_dimension 8 //--- #define def_k_InterpolationAttentionGrad 203 #define def_k_iag_matrix_in 0 #define def_k_iag_matrix_in_gr 1 #define def_k_iag_W 2 #define def_k_iag_W_gr 3 #define def_k_iag_A 4 #define def_k_iag_A_gr 5 #define def_k_iag_GL 6 #define def_k_iag_GL_gr 7 #define def_k_iag_Adj 8 #define def_k_iag_H 9 #define def_k_iag_H_gr 10 #define def_k_iag_Atten 11 #define def_k_iag_matrix_out_gr 12 #define def_k_iag_dimension 13 //--- #define def_k_IdentMatrixSum 204 #define def_k_idsum_matrix_in 0 #define def_k_idsum_matrix_out 1 #define def_k_idsum_multiplyer 2 #define def_k_idsum_shift_in 3 #define def_k_idsum_shift_out 4 //--- #define def_k_Activation 205 #define def_k_act_inputs 0 #define def_k_act_outputs 1 #define def_k_act_activation 2 //--- #define def_k_PeriodNorm 206 #define def_k_pn_inputs 0 #define def_k_pn_mean_stdevs 1 #define def_k_pn_outputs 2 #define def_k_pn_total_inputs 3 //--- #define def_k_PeriodNormGrad 207 #define def_k_png_inputs 0 #define def_k_png_inputs_gr 1 #define def_k_png_mean_stdevs 2 #define def_k_png_mean_stdevs_gr 3 #define def_k_png_outputs 4 #define def_k_png_outputs_gr 5 #define def_k_png_total_inputs 6 //--- #define def_k_AdaptSpatialNorm 208 #define def_k_asn_inputs 0 #define def_k_asn_attention 1 #define def_k_asn_mean_stdevs 2 #define def_k_asn_outputs 3 //--- #define def_k_AdaptSpatialNormGrad 209 #define def_k_asng_inputs 0 #define def_k_asng_inputs_gr 1 #define def_k_asng_attention 2 #define def_k_asng_attention_gr 3 #define def_k_asng_mean_stdevs 4 #define def_k_asng_mean_stdevs_gr 5 #define def_k_asng_outputs_gr 6 #define def_k_asng_total_inputs 7 //--- #define def_k_AttentNorm 210 #define def_k_atn_inputs 0 #define def_k_atn_attention 1 #define def_k_atn_means 2 #define def_k_atn_stdevs 3 #define def_k_atn_outputs 4 #define def_k_atn_total_inputs 5 #define def_k_atn_segment_size 6 //--- #define def_k_AttentNormGrad 211 #define def_k_atng_inputs 0 #define def_k_atng_inputs_gr 1 #define def_k_atng_attention 2 #define def_k_atng_attention_gr 3 #define def_k_atng_means 4 #define def_k_atng_stdevs 5 #define def_k_atng_means_gr 6 #define def_k_atng_outputs_gr 7 #define def_k_atng_total_inputs 8 #define def_k_atng_segment_size 9 //--- #define def_k_ChebStep 212 #define def_k_cheb_support 0 #define def_k_cheb_outputs 1 #define def_k_cheb_step 2 //--- #define def_k_ChebStepGrad 213 #define def_k_chebgr_support 0 #define def_k_chebgr_support_g 1 #define def_k_chebgr_outputs 2 #define def_k_chebgr_outputs_g 3 #define def_k_chebgr_step 4 //--- #define def_k_SignificantNeighborsSampling 214 #define def_k_sns_data 0 #define def_k_sns_candidates 1 #define def_k_sns_random_cands 2 #define def_k_sns_neighbors 3 #define def_k_sns_dimension 4 //--- #define def_k_SparseMHScores 215 #define def_k_smhs_data 0 #define def_k_smhs_indexes 1 #define def_k_smhs_scores 2 #define def_k_smhs_sparse 3 //--- #define def_k_SparseMHScoresGrad 216 #define def_k_smhs_gr_data_gr 0 #define def_k_smhs_gr_indexes 1 #define def_k_smhs_gr_scores 2 #define def_k_smhs_gr_scores_gr 3 //--- #define def_k_SparseMatMult 217 #define def_k_smm_sparse_index 0 #define def_k_smm_sparse_data 1 #define def_k_smm_full 2 #define def_k_smm_result 3 #define def_k_smm_full_rows 4 //--- #define def_k_SparseMatMultGrad 218 #define def_k_smm_gr_sparse_index 0 #define def_k_smm_gr_sparse_data 1 #define def_k_smm_gr_sparse_gr 2 #define def_k_smm_gr_full 3 #define def_k_smm_gr_full_gr 4 #define def_k_smm_gr_result_gr 5 #define def_k_smm_gr_sparse_rows 6 #define def_k_smm_gr_sparse_cols 7 #define def_k_smm_gr_full_rows 8 #define def_k_smm_gr_full_cols 9 //--- #define def_k_RandomWalk 219 #define def_k_rw_data 0 #define def_k_rw_inv_diag 1 #define def_k_rw_norm 2 #define def_k_rw_total_cols 3 //--- #define def_k_ConcatByLabel 220 #define def_k_cbl_data 0 #define def_k_cbl_label 1 #define def_k_cbl_embedding1 2 #define def_k_cbl_embedding2 3 #define def_k_cbl_output 4 #define def_k_cbl_dimension_data 5 #define def_k_cbl_dimension_emb1 6 #define def_k_cbl_dimension_emb2 7 #define def_k_cbl_frame1 8 #define def_k_cbl_frame2 9 #define def_k_cbl_period1 10 #define def_k_cbl_period2 11 //--- #define def_k_ConcatByLabelGrad 221 #define def_k_cbl_gr_units 12 //--- #define def_k_GlobalLocalAttention 222 #define def_k_glatt_q 0 #define def_k_glatt_kv 1 #define def_k_glatt_scores 2 #define def_k_glatt_mask 3 #define def_k_glatt_label 4 #define def_k_glatt_out 5 #define def_k_glatt_dimension 6 #define def_k_glatt_total_kv 7 #define def_k_glatt_total_mask 8 //--- #define def_k_GlobalLocalAttentionGrad 223 #define def_k_glatt_gr_q 0 #define def_k_glatt_gr_q_gr 1 #define def_k_glatt_gr_kv 2 #define def_k_glatt_gr_kv_gr 3 #define def_k_glatt_gr_scores 4 #define def_k_glatt_gr_mask 5 #define def_k_glatt_gr_mask_gr 6 #define def_k_glatt_gr_label 7 #define def_k_glatt_gr_out_gr 8 #define def_k_glatt_gr_dimension 9 #define def_k_glatt_gr_total_q 10 #define def_k_glatt_gr_total_kv 11 #define def_k_glatt_gr_total_mask 12 //--- #define def_k_SparseSoftMax 224 #define def_k_ssoftmax_data 0 #define def_k_ssoftmax_outputs 1 #define def_k_ssoftmax_indexes 2 #define def_k_ssoftmax_out_dimension 3 //--- #define def_k_SparseSoftMaxGrad 225 #define def_k_ssoftmax_gr_data_gr 0 #define def_k_ssoftmax_gr_outputs 1 #define def_k_ssoftmax_gr_outputs_gr 2 #define def_k_ssoftmax_gr_indexes 3 #define def_k_ssoftmax_gr_out_dimension 4 //--- #define def_k_FloatToSpike 226 #define def_k_fts_values 0 #define def_k_fts_levels 1 #define def_k_fts_outputs 2 //--- #define def_k_FloatToSpikeGrad 227 #define def_k_ftsg_values 0 #define def_k_ftsg_values_gr 1 #define def_k_ftsg_levels_gr 2 #define def_k_ftsg_gradients 3 //--- #define def_k_SpikeMHAttention 228 #define def_k_smhat_qkv 0 #define def_k_smhat_diag_bias 1 #define def_k_smhat_scores 2 #define def_k_smhat_out 3 #define def_k_smhat_dimension 4 #define def_k_smhat_mask_future 5 //--- #define def_k_SpikeMHAttentionGrad 229 #define def_k_smhat_gr_qkv 0 #define def_k_smhat_gr_qkv_gr 1 #define def_k_smhat_gr_diag_bias 2 #define def_k_smhat_gr_diag_bias_gr 3 #define def_k_smhat_gr_scores 4 #define def_k_smhat_gr_gradients 5 #define def_k_smhat_gr_dimension 6 #define def_k_smhat_gr_mask_future 7 //--- #define def_k_STFS 230 #define def_k_stfs_inputs 0 #define def_k_stfs_mask_time 1 #define def_k_stfs_mask_spatial 2 #define def_k_stfs_outputs 3 //--- #define def_k_STFSGrad 231 //--- #define def_k_AddToStack 232 #define def_k_ats_inputs 0 #define def_k_ats_stack 1 #define def_k_ats_stack_size 2 //--- #define def_k_AggregationByTime 233 #define def_k_agt_inputs 0 #define def_k_agt_stack 1 #define def_k_agt_outputs 2 #define def_k_agt_stack_size 3 #define def_k_agt_levels 4 //--- #define def_k_AggregationByTimeGrad 234 #define def_k_agtg_inputs_gr 0 #define def_k_agtg_outputs_gr 1 #define def_k_agtg_levels 2 //--- #define def_k_GRU 235 #define def_k_gru_XH 0 #define def_k_gru_prev_state 1 #define def_k_gru_outputs 2 //--- #define def_k_GRU_Grad 236 #define def_k_gru_gr_XH 0 #define def_k_gru_gr_XH_gr 1 #define def_k_gru_gr_prev_state 2 #define def_k_gru_gr_outputs_gr 3 //--- #define def_k_ScalarToVector 237 #define def_k_stv_scalar 0 #define def_k_stv_vector_in 1 #define def_k_stv_vector_out 2 //--- #define def_k_ScalarToVectorGrad 238 #define def_k_stvg_scalar 0 #define def_k_stvg_scalar_gr 1 #define def_k_stvg_vector_in 2 #define def_k_stvg_vector_in_gr 3 #define def_k_stvg_vector_out_gr 4 #define def_k_stvg_dimension 5 //--- #define def_k_CalcFlow 239 #define def_k_cf_value 0 #define def_k_cf_prev_value 1 #define def_k_cf_flow 2 //--- #define def_k_DilatedCorrelation 240 #define def_k_dcor_feature 0 #define def_k_dcor_shifts 1 #define def_k_dcor_correlations 2 #define def_k_dcor_dimension 3 //--- #define def_k_DilatedCorrelationGrad 241 #define def_k_dcorgr_feature 0 #define def_k_dcorgr_feature_gr 1 #define def_k_dcorgr_shifts 2 #define def_k_dcorgr_corr_gr 3 #define def_k_dcorgr_total_corr 4 //--- #define def_k_DilatedDifference 242 #define def_k_ddiff_feature 0 #define def_k_ddiff_shifts 1 #define def_k_ddiff_differences 2 //--- #define def_k_DilatedDifferenceGrad 243 #define def_k_ddiffgr_feature 0 #define def_k_ddiffgr_feature_gr 1 #define def_k_ddiffgr_shifts 2 #define def_k_ddiffgr_differences_gr 3 #define def_k_ddiffgr_total_shifts 4 //--- #define def_k_PerturbedMatrix 244 #define def_k_pm_inputs 0 #define def_k_pm_perturb 1 #define def_k_pm_output 2 #define def_k_pm_perturb_mult 3 //--- #define def_k_PerturbedMatrixGrad 245 //--- #define def_k_LinearUpsample 246 #define def_k_lu_data 0 #define def_k_lu_upsample 1 //--- #define def_k_LinearUpsampleGrad 247 #define def_k_lug_data_gr 0 #define def_k_lug_upsample_gr 1 #define def_k_lug_dimension_htr 2 //--- #define def_k_MixExpertsPredict 248 #define def_k_moepred_experts 0 #define def_k_moepred_outputs 1 //--- #define def_k_MixExpertsPredictGrad 249 #define def_k_moepredgr_experts 0 #define def_k_moepredgr_experts_gr 1 #define def_k_moepredgr_outputs_gr 2 //--- #define def_k_MHFAT 250 #define def_k_mhfat_q 0 #define def_k_mhfat_kv 1 #define def_k_mhfat_scale 2 #define def_k_mhfat_scores 3 #define def_k_mhfat_out 4 #define def_k_mhfat_dimension 5 #define def_k_mhfat_mask_future 6 //--- #define def_k_MHFATGrad 251 #define def_k_mhfat_gr_q 0 #define def_k_mhfat_gr_q_gr 1 #define def_k_mhfat_gr_kv 2 #define def_k_mhfat_gr_kv_gr 3 #define def_k_mhfat_gr_scale 4 #define def_k_mhfat_gr_scale_gr 5 #define def_k_mhfat_gr_scores 6 #define def_k_mhfat_gr_gradients 7 #define def_k_mhfat_gr_dimension 8 #define def_k_mhfat_gr_total_k 9 #define def_k_mhfat_gr_mask_future 10 //--- #define def_k_SparseConcatenate 252 #define def_k_sparconc_sparse_index 0 #define def_k_sparconc_sparse_data 1 #define def_k_sparconc_full 2 #define def_k_sparconc_result 3 #define def_k_sparconc_full_rows 4 //--- #define def_k_SparseConcatenateGrad 253 #define def_k_sparconc_gr_sparse_index 0 #define def_k_sparconc_gr_sparse_data 1 #define def_k_sparconc_gr_sparse_gr 2 #define def_k_sparconc_gr_full 3 #define def_k_sparconc_gr_full_gr 4 #define def_k_sparconc_gr_result_gr 5 #define def_k_sparconc_gr_sparse_rows 6 #define def_k_sparconc_gr_sparse_cols 7 #define def_k_sparconc_gr_full_rows 8 #define def_k_sparconc_gr_full_cols 9 //--- #define def_k_MHFlashAttention 254 #define def_k_mhflat_q 0 #define def_k_mhflat_kv 1 #define def_k_mhflat_logsumexp 2 #define def_k_mhflat_out 3 #define def_k_mhflat_dimension 4 #define def_k_mhflat_total_kv 5 #define def_k_mhflat_mask_future 6 //--- #define def_k_MHFlashAttentionGrad 255 #define def_k_mhflatg_q 0 #define def_k_mhflatg_q_gr 1 #define def_k_mhflatg_kv 2 #define def_k_mhflatg_kv_gr 3 #define def_k_mhflatg_logsumexp 4 #define def_k_mhflatg_out 5 #define def_k_mhflatg_out_gr 6 #define def_k_mhflatg_dimension 7 #define def_k_mhflatg_total_q 8 #define def_k_mhflatg_total_kv 9 #define def_k_mhflatg_mask_future 10 //--- #define def_k_MHFlashSTCA 256 #define def_k_mhstca_query 0 #define def_k_mhstca_X 1 #define def_k_mhstca_logsumexp 2 #define def_k_mhstca_output 3 #define def_k_mhstca_dimension 4 #define def_k_mhstca_total_X 5 #define def_k_mhstca_mask_future 6 //--- #define def_k_MHFlashSTCAGrad 257 #define def_k_mhstca_gr_query 0 #define def_k_mhstca_gr_query_gr 1 #define def_k_mhstca_gr_X 2 #define def_k_mhstca_gr_X_gr 3 #define def_k_mhstca_gr_logsumexp 4 #define def_k_mhstca_gr_output 5 #define def_k_mhstca_gr_output_gr 6 #define def_k_mhstca_gr_dimension 7 #define def_k_mhstca_gr_total_q 8 #define def_k_mhstca_gr_total_X 9 #define def_k_mhstca_gr_mask_future 10 //--- #define def_k_ConcatVecMatrix 258 #define def_k_concvm_vector_in 0 #define def_k_concvm_matrix_in 1 #define def_k_concvm_matrix_out 2 #define def_k_concvm_dimension_v 3 #define def_k_concvm_colums_m 4 #define def_k_concvm_multvarsecond 5 //--- #define def_k_ConcatVecMatrixGrad 259 #define def_k_concvm_gr_vector_in_gr 0 #define def_k_concvm_gr_matrix_in_gr 1 #define def_k_concvm_gr_matrix_out_gr 2 #define def_k_concvm_gr_dimension_v 3 #define def_k_concvm_gr_colums_m 4 #define def_k_concvm_gr_rows 5 #define def_k_concvm_gr_variables 6 #define def_k_concvm_gr_multvarsecond 7 //--- #define def_k_INFNetBGU 260 #define def_k_infnbgu_scal_and_shift 0 #define def_k_infnbgu_sequence 1 #define def_k_infnbgu_context 2 #define def_k_infnbgu_scenarios 3 #define def_k_infnbgu_sequence_out 4 #define def_k_infnbgu_context_out 5 #define def_k_infnbgu_scenarios_out 6 #define def_k_infnbgu_sequence_units 7 #define def_k_infnbgu_sequence_vars 8 #define def_k_infnbgu_context_units 9 #define def_k_infnbgu_scenarios_units 10 //--- #define def_k_INFNetBGUGrad 261 #define def_k_infnbgugr_scal_and_shift 0 #define def_k_infnbgugr_scal_and_shift_grad 1 #define def_k_infnbgugr_sequence 2 #define def_k_infnbgugr_context 3 #define def_k_infnbgugr_scenarios 4 #define def_k_infnbgugr_sequence_grad 5 #define def_k_infnbgugr_context_grad 6 #define def_k_infnbgugr_scenarios_grad 7 #define def_k_infnbgugr_sequence_out_grad 8 #define def_k_infnbgugr_context_out_grad 9 #define def_k_infnbgugr_scenarios_out_grad 10 #define def_k_infnbgugr_sequence_units 11 #define def_k_infnbgugr_sequence_vars 12 #define def_k_infnbgugr_context_units 13 #define def_k_infnbgugr_scenarios_units 14 //--- #define def_k_MHCrossAttvsSim 262 #define def_k_casim_query 0 #define def_k_casim_key 1 #define def_k_casim_value 2 #define def_k_casim_prototype 3 #define def_k_casim_levels 4 #define def_k_casim_logsumexp 5 #define def_k_casim_output_at 6 #define def_k_casim_similarity 7 #define def_k_casim_dimension 8 #define def_k_casim_total_X 9 //--- #define def_k_MHCrossAttvsSimGrad 263 #define def_k_casimgr_query 0 #define def_k_casimgr_key 1 #define def_k_casimgr_value 2 #define def_k_casimgr_prototype 3 #define def_k_casimgr_levels 4 #define def_k_casimgr_logsumexp 5 #define def_k_casimgr_output_at 6 #define def_k_casimgr_similarity 7 #define def_k_casimgr_grad_output_at 8 #define def_k_casimgr_grad_similarity 9 #define def_k_casimgr_grad_query 10 #define def_k_casimgr_grad_key 11 #define def_k_casimgr_grad_value 12 #define def_k_casimgr_grad_prototype 13 #define def_k_casimgr_grad_levels 14 #define def_k_casimgr_dimension 15 #define def_k_casimgr_total_X 16 //--- #define def_k_MarketStateDensity 264 #define def_k_msd_sequence 0 #define def_k_msd_distribution 1 #define def_k_msd_sequence_size 2 #define def_k_msd_quantiles 3 #define def_k_msd_quant_step 4 //--- #define def_k_NLLGrad 265 #define def_k_nllgr_log_b 0 #define def_k_nllgr_grad_in 1 #define def_k_nllgr_grad_out 2 #define def_k_nllgr_grad_log_b 3 //--- #define def_k_SparseSoftmaxHausdorff 266 #define def_k_ssh_plan 0 #define def_k_ssh_decis 1 #define def_k_ssh_output 2 #define def_k_ssh_top_index 3 #define def_k_ssh_dimension 4 #define def_k_ssh_top_k 5 //--- #define def_k_SparseSoftmaxHausdorffGrad 267 #define def_k_sshg_plan 0 #define def_k_sshg_grad_plan 1 #define def_k_sshg_decis 2 #define def_k_sshg_grad_decis 3 #define def_k_sshg_grad_out 4 #define def_k_sshg_top_index 5 #define def_k_sshg_dimension 6 #define def_k_sshg_count_plan 7 #define def_k_sshg_count_decis 8 #define def_k_sshg_top_k 9 //--- #define def_k_MomADTTMMeanScores 268 #define def_k_madttm_ms_distance 0 #define def_k_madttm_ms_output 1 #define def_k_madttm_ms_decisions 2 //--- #define def_k_MomADTTMMeanScoresGrad 269 #define def_k_madttm_msg_scores_gr 0 #define def_k_madttm_msg_distance_gr 1 #define def_k_madttm_msg_decisions 2 //+------------------------------------------------------------------+ //| | //+------------------------------------------------------------------+ #resource "NeuroNet.cl" as string cl_program ///\defgroup enums ENUM ///@{ //+------------------------------------------------------------------+ /// Enum of activation formula used //+------------------------------------------------------------------+ enum ENUM_ACTIVATION { None = -1, ///< Without activation formula TANH = 0, ///< Use \f$tanh(x)\f$ for activation neuron SIGMOID = 1, ///< Use \f$\frac{1}{1+e^x}\f$ for activation neuron LReLU = 2, ///< For activation neuron use LReLU \f[\left\{ \begin{array} a x>=0, \ x \\x<0, \ 0.01*x \end{array} \right.\f] SoftPlus = 3, GELU = 4, MinusSoftPlus = 5, ELU = 6, }; //+------------------------------------------------------------------+ /// Enum of optimization method used //+------------------------------------------------------------------+ enum ENUM_OPTIMIZATION { SGD = 0, ///< Stochastic gradient descent ADAM = 1, ///< Adam LS = 2, ///< Least Squares ADAM_MINI = 3 ///< Adam-mini }; ///@} //+------------------------------------------------------------------+ ///\class CConnection ///\brief Class of connection to anothe neuron ///\details Detailed description on the link. //+------------------------------------------------------------------+ #endif // NEURONET_DEFINITIONS_MQH #ifndef NEURONET_BUILDING_FACADE #include "NeuroNet.mqh" #endif // NEURONET_BUILDING_FACADE