383 lines
No EOL
33 KiB
MQL5
383 lines
No EOL
33 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| Defines.mqh |
|
|
//| Copyright 2021, MetaQuotes Ltd. |
|
|
//| https://www.mql5.com |
|
|
//+------------------------------------------------------------------+
|
|
#property copyright "Copyright 2021, MetaQuotes Ltd."
|
|
#property link "https://www.mql5.com"
|
|
//+------------------------------------------------------------------+
|
|
//| Resources |
|
|
//+------------------------------------------------------------------+
|
|
#resource "opencl_program.cl" as string OCLprogram
|
|
//---
|
|
#define TYPE double
|
|
#define MATRIX matrix<TYPE>
|
|
#define VECTOR vector<TYPE>
|
|
#define LOCAL_SIZE 256
|
|
const string ExtType=StringFormat("#define TYPE %s\r\n"
|
|
"#define TYPE4 %s4\r\n"
|
|
"#define LOCAL_SIZE %d\r\n",
|
|
typename(TYPE),typename(TYPE),LOCAL_SIZE);
|
|
#define cl_program ExtType+OCLprogram
|
|
//---
|
|
#define defLossSmoothFactor 1000
|
|
#define defLearningRate (TYPE)3.0e-4
|
|
#define defBeta1 (TYPE)0.9
|
|
#define defBeta2 (TYPE)0.999
|
|
#define defLambdaL1 (TYPE)0
|
|
#define defLambdaL2 (TYPE)0
|
|
//+------------------------------------------------------------------+
|
|
//| Constants |
|
|
//+------------------------------------------------------------------+
|
|
#define Defines
|
|
#define defNeuronNet 0x8000
|
|
#define defArrayLayers 0x8001
|
|
#define defBuffer 0x8002
|
|
#define defActivation 0x8003
|
|
#define defLayerDescription 0x8004
|
|
#define defNeuronBase 0x8010
|
|
#define defNeuronConv 0x8011
|
|
#define defNeuronProof 0x8012
|
|
#define defNeuronLSTM 0x8013
|
|
#define defNeuronAttention 0x8014
|
|
#define defNeuronMHAttention 0x8015
|
|
#define defNeuronGPT 0x8016
|
|
#define defNeuronDropout 0x8017
|
|
#define defNeuronBatchNorm 0x8018
|
|
//---
|
|
#define defFileName StringFormat("%s_%s_%s.nns",MQLInfoString(MQL_PROGRAM_NAME),_Symbol,StringSubstr(EnumToString(_Period),7))
|
|
//+------------------------------------------------------------------+
|
|
//| OpenCL kernels |
|
|
//+------------------------------------------------------------------+
|
|
#define def_k_PerceptronFeedForward 0
|
|
#define def_k_LineActivation 1
|
|
#define def_k_SigmoidActivation 2
|
|
#define def_k_SigmoidDerivative 3
|
|
#define def_k_TANHActivation 4
|
|
#define def_k_TANHDerivative 5
|
|
#define def_k_LReLuActivation 6
|
|
#define def_k_LReLuDerivative 7
|
|
#define def_k_SoftMAXActivation 8
|
|
#define def_k_SoftMAXDerivative 9
|
|
#define def_k_SwishActivation 10
|
|
#define def_k_SwishDerivative 11
|
|
#define def_k_CalcOutputGradient 12
|
|
#define def_k_CalcHiddenGradient 13
|
|
#define def_k_CalcDeltaWeights 14
|
|
#define def_k_SGDUpdate 15
|
|
#define def_k_MomentumUpdate 16
|
|
#define def_k_AdaGradUpdate 17
|
|
#define def_k_RMSPropUpdate 18
|
|
#define def_k_AdaDeltaUpdate 19
|
|
#define def_k_AdamUpdate 20
|
|
#define def_k_ProofFeedForward 21
|
|
#define def_k_ProofHiddenGradients 22
|
|
#define def_k_ConvolutionFeedForward 23
|
|
#define def_k_ConvolutionHiddenGradients 24
|
|
#define def_k_ConvolutionDeltaWeights 25
|
|
#define def_k_LSTMFeedForward 26
|
|
#define def_k_LSTMHiddenGradients 27
|
|
#define def_k_AttentionFeedForward 28
|
|
#define def_k_AttentionScoreGradients 29
|
|
#define def_k_AttentionHiddenGradients 30
|
|
#define def_k_Sum 31
|
|
#define def_k_LayerNormalize 32
|
|
#define def_k_LayerNormalizeGradient 33
|
|
#define def_k_GPTFeedForward 34
|
|
#define def_k_GPTScoreGradients 35
|
|
#define def_k_GPTHiddenGradients 36
|
|
#define def_k_BatchNormFeedForward 37
|
|
#define def_k_BatchNormCalcHiddenGradient 38
|
|
#define def_k_BatchNormCalcDeltaWeights 39
|
|
#define def_k_MaskMult 40
|
|
#define def_k_Split 41
|
|
#define def_k_Concatenate 42
|
|
//+------------------------------------------------------------------+
|
|
//| OpenCL parameters |
|
|
//+------------------------------------------------------------------+
|
|
//--- perceptron feed-forward pass
|
|
#define def_pff_inputs 0
|
|
#define def_pff_weights 1
|
|
#define def_pff_outputs 2
|
|
#define def_pff_inputs_total 3
|
|
//--- define the error gradient of the results layer
|
|
#define def_outgr_target 0
|
|
#define def_outgr_outputs 1
|
|
#define def_outgr_gradients 2
|
|
#define def_outgr_loss_function 3
|
|
//--- define the error gradient of the hidden layer
|
|
#define def_hidgr_gradient_inputs 0
|
|
#define def_hidgr_weights 1
|
|
#define def_hidgr_gradients 2
|
|
#define def_hidgr_outputs_total 3
|
|
//--- define the error gradient at the weight matrix level
|
|
#define def_delt_inputs 0
|
|
#define def_delt_delta_weights 1
|
|
#define def_delt_gradients 2
|
|
//--- optimize parameters using stochastic gradient descent
|
|
#define def_sgd_delta_weights 0
|
|
#define def_sgd_weights 1
|
|
#define def_sgd_total 2
|
|
#define def_sgd_batch_size 3
|
|
#define def_sgd_learningRate 4
|
|
#define def_sgd_Lambda1 5
|
|
#define def_sgd_Lambda2 6
|
|
//--- optimize parameters using momentum method
|
|
#define def_moment_delta_weights 0
|
|
#define def_moment_weights 1
|
|
#define def_moment_momentum 2
|
|
#define def_moment_total 3
|
|
#define def_moment_batch_size 4
|
|
#define def_moment_learningRate 5
|
|
#define def_moment_beta 6
|
|
#define def_moment_Lambda1 7
|
|
#define def_moment_Lambda2 8
|
|
//--- optimize parameters using the AdaGrad method
|
|
#define def_adagrad_delta_weights 0
|
|
#define def_adagrad_weights 1
|
|
#define def_adagrad_momentum 2
|
|
#define def_adagrad_total 3
|
|
#define def_adagrad_batch_size 4
|
|
#define def_adagrad_learningRate 5
|
|
#define def_adagrad_Lambda1 6
|
|
#define def_adagrad_Lambda2 7
|
|
//--- optimize parameters using the RMSProp method
|
|
#define def_rms_delta_weights 0
|
|
#define def_rms_weights 1
|
|
#define def_rms_momentum 2
|
|
#define def_rms_total 3
|
|
#define def_rms_batch_size 4
|
|
#define def_rms_learningRate 5
|
|
#define def_rms_beta 6
|
|
#define def_rms_Lambda1 7
|
|
#define def_rms_Lambda2 8
|
|
//--- optimize parameters using the AdaDelta method
|
|
#define def_adadelt_delta_weights 0
|
|
#define def_adadelt_weights 1
|
|
#define def_adadelt_momentumW 2
|
|
#define def_adadelt_momentumG 3
|
|
#define def_adadelt_total 4
|
|
#define def_adadelt_batch_size 5
|
|
#define def_adadelt_beta1 6
|
|
#define def_adadelt_beta2 7
|
|
#define def_adadelt_Lambda1 8
|
|
#define def_adadelt_Lambda2 9
|
|
//--- optimize parameters using the Adam method
|
|
#define def_adam_delta_weights 0
|
|
#define def_adam_weights 1
|
|
#define def_adam_momentumM 2
|
|
#define def_adam_momentumV 3
|
|
#define def_adam_total 4
|
|
#define def_adam_batch_size 5
|
|
#define def_adam_learningRate 6
|
|
#define def_adam_beta1 7
|
|
#define def_adam_beta2 8
|
|
#define def_adam_Lambda1 9
|
|
#define def_adam_Lambda2 10
|
|
//--- feed-forward of the pooling layer
|
|
#define def_prff_inputs 0
|
|
#define def_prff_outputs 1
|
|
#define def_prff_inputs_total 2
|
|
#define def_prff_input_neurons 3
|
|
#define def_prff_window 4
|
|
#define def_prff_step 5
|
|
#define def_prff_activation 6
|
|
//--- gradient propagation through the pooling layer
|
|
#define def_prhgr_inputs 0
|
|
#define def_prhgr_gradient_inputs 1
|
|
#define def_prhgr_outputs 2
|
|
#define def_prhgr_gradients 3
|
|
#define def_prhgr_inputs_total 4
|
|
#define def_prhgr_outputs_total 5
|
|
#define def_prhgr_window 6
|
|
#define def_prhgr_step 7
|
|
#define def_prhgr_neurons 8
|
|
#define def_prhgr_activation 9
|
|
//--- feed-forward of the convolutional layer
|
|
#define def_cff_inputs 0
|
|
#define def_cff_weights 1
|
|
#define def_cff_outputs 2
|
|
#define def_cff_inputs_total 3
|
|
#define def_cff_window 4
|
|
#define def_cff_step 5
|
|
#define def_cff_window_out 6
|
|
#define def_cff_transposed_out 7
|
|
//--- gradient propagation through the convolutional layer
|
|
#define def_convhgr_gradient_inputs 0
|
|
#define def_convhgr_weights 1
|
|
#define def_convhgr_gradients 2
|
|
#define def_convhgr_window 3
|
|
#define def_convhgr_step 4
|
|
#define def_convhgr_window_out 5
|
|
#define def_convhgr_neurons 6
|
|
#define def_convhgr_transposed_out 7
|
|
//--- gradient propagation to the weight matrix of the convolutional layer
|
|
#define def_convdelt_inputs 0
|
|
#define def_convdelt_delta_weights 1
|
|
#define def_convdelt_gradients 2
|
|
#define def_convdelt_inputs_total 3
|
|
#define def_convdelt_step 4
|
|
#define def_convdelt_neurons 5
|
|
#define def_convdelt_transposed_out 6
|
|
//--- LSTM block feed-forward pass
|
|
#define def_lstmff_forgetgate 0
|
|
#define def_lstmff_inputgate 1
|
|
#define def_lstmff_outputgate 2
|
|
#define def_lstmff_newcontent 3
|
|
#define def_lstmff_memory 4
|
|
#define def_lstmff_hiddenstate 5
|
|
#define def_lstmff_outputs_total 6
|
|
//--- gradient propagation through the LSTM block
|
|
#define def_lstmhgr_outputs 0
|
|
#define def_lstmhgr_gradients 1
|
|
#define def_lstmhgr_inputgate 2
|
|
#define def_lstmhgr_outputgate 3
|
|
#define def_lstmhgr_newcontent 4
|
|
#define def_lstmhgr_memory 5
|
|
#define def_lstmhgr_fg_gradients 6
|
|
#define def_lstmhgr_ig_gradients 7
|
|
#define def_lstmhgr_og_gradients 8
|
|
#define def_lstmhgr_nc_gradients 9
|
|
#define def_lstmhgr_outputs_total 10
|
|
//--- Attention block feed-forward pass
|
|
#define def_attff_querys 0
|
|
#define def_attff_keys 1
|
|
#define def_attff_scores 2
|
|
#define def_attff_values 3
|
|
#define def_attff_outputs 4
|
|
#define def_attff_window 5
|
|
#define def_attff_key_size 6
|
|
#define def_attff_mask 7
|
|
//--- define gradient at the matrix of dependence coefficients in the attention block
|
|
#define def_attscr_scores 0
|
|
#define def_attscr_scores_grad 1
|
|
#define def_attscr_values 2
|
|
#define def_attscr_values_grad 3
|
|
#define def_attscr_outputs_grad 4
|
|
#define def_attscr_scores_temp 5
|
|
#define def_attscr_window 6
|
|
//--- gradient propagation through the Attention block
|
|
#define def_atthgr_querys 0
|
|
#define def_atthgr_querys_grad 1
|
|
#define def_atthgr_keys 2
|
|
#define def_atthgr_keys_grad 3
|
|
#define def_atthgr_scores_grad 4
|
|
#define def_atthgr_key_size 5
|
|
//--- GPT feed-forward
|
|
#define def_gptff_querys 0
|
|
#define def_gptff_keys 1
|
|
#define def_gptff_scores 2
|
|
#define def_gptff_values 3
|
|
#define def_gptff_outputs 4
|
|
#define def_gptff_key_size 5
|
|
#define def_gptff_units 6
|
|
#define def_gptff_current 7
|
|
//--- define gradient at the matrix of dependence coefficients in GPT
|
|
#define def_gptscr_scores 0
|
|
#define def_gptscr_scores_grad 1
|
|
#define def_gptscr_values 2
|
|
#define def_gptscr_values_grad 3
|
|
#define def_gptscr_outputs_grad 4
|
|
#define def_gptscr_scores_temp 5
|
|
#define def_gptscr_window 6
|
|
#define def_gptscr_units 7
|
|
#define def_gptscr_current 8
|
|
//--- gradient propagation through GPT
|
|
#define def_gpthgr_querys 0
|
|
#define def_gpthgr_querys_grad 1
|
|
#define def_gpthgr_keys 2
|
|
#define def_gpthgr_scores_grad 3
|
|
#define def_gpthgr_key_size 4
|
|
#define def_gpthgr_units 5
|
|
#define def_gpthgr_current 6
|
|
//--- batch normalization feed-forward
|
|
#define def_bnff_inputs 0
|
|
#define def_bnff_options 1
|
|
#define def_bnff_weights 2
|
|
#define def_bnff_outputs 3
|
|
#define def_bnff_batch 4
|
|
#define def_bnff_total 5
|
|
//--- gradient propagation through the batch normalization layer
|
|
#define def_bnhgr_options 0
|
|
#define def_bnhgr_gradient 1
|
|
#define def_bnhgr_inputs 2
|
|
#define def_bnhgr_gradient_inputs 3
|
|
#define def_bnhgr_weights 4
|
|
#define def_bnhgr_batch 5
|
|
#define def_bnhgr_total 6
|
|
//--- gradient propagation to optimizable parameters in batch normalization
|
|
#define def_bndelt_options 0
|
|
#define def_bndelt_delta_weights 1
|
|
#define def_bndelt_gradient 2
|
|
//--- data masking
|
|
#define def_mask_inputs 0
|
|
#define def_mask_mask 1
|
|
#define def_mask_outputs 2
|
|
#define def_mask_total 3
|
|
//--- sum of vectors
|
|
#define def_sum_inputs1 0
|
|
#define def_sum_inputs2 1
|
|
#define def_sum_outputs 2
|
|
//--- vector normalization
|
|
#define def_layernorm_inputs 0
|
|
#define def_layernorm_outputs 1
|
|
#define def_layernorm_std 2
|
|
#define def_layernorm_vector_size 3
|
|
#define def_layernorm_std_shift 4
|
|
//--- vector normalization gradient
|
|
#define def_layernormgr_outputs 0
|
|
#define def_layernormgr_out_grad 1
|
|
#define def_layernormgr_inp_grad 2
|
|
#define def_layernormgr_std 3
|
|
#define def_layernormgr_vector_size 4
|
|
#define def_layernormgr_std_shift 5
|
|
//--- activation functions
|
|
#define def_activ_inputs 0
|
|
#define def_activ_outputs 1
|
|
#define def_activ_param_a 2
|
|
#define def_activ_param_b 3
|
|
//--- adjust the gradient to the derivative of the activation function
|
|
#define def_deactgr_outputs 0
|
|
#define def_deactgr_gradients 1
|
|
#define def_deactgr_deact_gradient 2
|
|
#define def_deactgr_act_param_a 3
|
|
#define def_deactgr_act_param_b 4
|
|
//--- SoftMax
|
|
#define def_softmax_input 0
|
|
#define def_softmax_output 1
|
|
#define def_softmax_total 2
|
|
//--- Split
|
|
#define def_split_source 0
|
|
#define def_split_target1 1
|
|
#define def_split_target2 2
|
|
#define def_split_total_source 3
|
|
#define def_split_total_target1 4
|
|
#define def_split_total_target2 5
|
|
//--- Concatenate
|
|
#define def_concat_source1 0
|
|
#define def_concat_source2 1
|
|
#define def_concat_target 2
|
|
#define def_concat_total_source1 3
|
|
#define def_concat_total_sourse2 4
|
|
#define def_concat_total_target 5
|
|
//+------------------------------------------------------------------+
|
|
//| Enumerations |
|
|
//+------------------------------------------------------------------+
|
|
//--- activation functions of the pooling layer
|
|
enum ENUM_PROOF
|
|
{
|
|
AF_MAX_POOLING,
|
|
AF_AVERAGE_POOLING
|
|
};
|
|
//--- optimization methods
|
|
enum ENUM_OPTIMIZATION
|
|
{
|
|
None = -1,
|
|
SGD,
|
|
MOMENTUM,
|
|
AdaGrad,
|
|
RMSProp,
|
|
AdaDelta,
|
|
Adam
|
|
};
|
|
//+------------------------------------------------------------------+ |