forked from animatedread/Warrior_EA
b91c7b1f7a
The //| box blocks were excluded from0b06f8eand5efdb48and were what remained: 160 of them ran to 10+ lines, the longest to 88. Compressed to their leading topic sentences - 5 lines for a function header, 8 for a file header - keeping the box format and the standard MQL5 name/author lines verbatim. Verified at the BYTE level this time, across every in-scope file: the list of non-comment lines is byte-identical to HEAD and braces balance. The first check compared a locale-decoded 'git show' against a UTF-8 read and flagged 25 files that had not changed at all - every BOM and every non-ASCII line mismatched. 47,696 -> 40,665 lines in scope; comment share 38% -> 26%. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1158 lines
58 KiB
MQL5
1158 lines
58 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include <Arrays\ArrayDouble.mqh>
|
|
#include <Arrays\ArrayInt.mqh>
|
|
#include <Arrays\ArrayObj.mqh>
|
|
#include <OpenCL\OpenCL.mqh>
|
|
#include "..\System\AtomicFile.mqh"
|
|
//--- 3rd-tier CPU fallback (used when neither OpenCL nor DirectML/D3D12 GPU accel are available,
|
|
//--- e.g. a VM with no GPU passthrough, or this machine's OpenCL/DirectML init failed).
|
|
#define CPU_THREADS_PER_NETWORK 2
|
|
//+------------------------------------------------------------------+
|
|
//| The percentage CDirectMLMy needs in order to land on |
|
|
//| CPU_THREADS_PER_NETWORK workers, given the detected core count. |
|
|
//| (WarriorCPU.dll takes a percentage of cores, not a thread count.) |
|
|
//+------------------------------------------------------------------+
|
|
int EffectiveCpuLoadPercent()
|
|
{
|
|
int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES);
|
|
//--- Unknown core count: assume a small machine rather than a large one. Guessing high here would
|
|
//--- reintroduce exactly the oversubscription this function exists to prevent.
|
|
if(cores <= 0)
|
|
cores = 4;
|
|
//--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the
|
|
//--- DLL, so this is the one value that needs no arithmetic.
|
|
if(cores <= CPU_THREADS_PER_NETWORK)
|
|
return 100;
|
|
//--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread
|
|
//--- short of the target is a worse error than landing one over.
|
|
int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores);
|
|
return (int)MathMax(1, MathMin(100, pct));
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
//--- Adam (Kingma & Ba, 2014) hyperparameters. 3.0e-4 also happens to sit inside the noisy/non-
|
|
//--- stationary-trading-data range (0.0003-0.0005) this project had separately tuned lr to before
|
|
//--- this input existed, so no behavior conflict.
|
|
input double AdamLearningRate = 0.0003; // Adam learning rate
|
|
input double AdamBeta1 = 0.9; // Adam beta1
|
|
input double AdamBeta2 = 0.999; // Adam beta2
|
|
//--- SGD+momentum hyperparameters.
|
|
input double SgdLearningRate = 0.0003; // SGD learning rate
|
|
input double SgdMomentum = 0.9; // SGD momentum
|
|
//--- Live learning rate: starts at the Adam input, then decayed and restored by the training loop,
|
|
//--- so it is a mutable global rather than a constant.
|
|
//--- g_-PREFIXED, and it has to be: as a bare `eta` this collided with a local of the same name in
|
|
//--- the standard library's Math\Stat\Math.mqh (the incomplete-gamma branch), which the compiler
|
|
//--- reports as "declaration of 'eta' hides global variable". Same fault as the b1/b2/lr/momentum
|
|
//--- macros retired in ea2552e - a single-token global name in a header that library code is
|
|
//--- compiled beside. The ETA_DECAY_FACTOR/ETA_MIN/m_etaCeiling vocabulary around it is unchanged,
|
|
//--- so the symbol still reads as the learning rate everywhere it appears.
|
|
double g_eta = AdamLearningRate;
|
|
#define defConnect 0x7781
|
|
#define defArrayConnects 0x7782
|
|
#define defNeuronBase 0x7783
|
|
#define defNeuron 0x7784
|
|
#define defNeuronConv 0x7785
|
|
#define defNeuronPool 0x7786
|
|
#define defLayer 0x7787
|
|
#define defArrayLayer 0x7788
|
|
#define defNet 0x7789
|
|
#define defNeuronLSTM 0x7791
|
|
//--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like
|
|
//--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps
|
|
//--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh.
|
|
#define defNeuronBatchNorm 0x7792
|
|
//---
|
|
#define defBufferDouble 0x7882
|
|
#define defNeuronBaseOCL 0x7883
|
|
#define defNeuronLSTMOCL 0x7884
|
|
#define defNeuronConvOCL 0x7885
|
|
#define defNeuronPoolOCL 0x7886
|
|
#define defNeuronBatchNormOCL 0x7887
|
|
//---
|
|
#define def_k_FeedForward 0
|
|
#define def_k_ff_matrix_w 0
|
|
#define def_k_ff_matrix_i 1
|
|
#define def_k_ff_matrix_o 2
|
|
#define def_k_ff_inputs 3
|
|
#define def_k_ff_activation 4
|
|
//---
|
|
#define def_k_CaclOutputGradient 1
|
|
#define def_k_cog_matrix_t 0
|
|
#define def_k_cog_matrix_o 1
|
|
#define def_k_cog_matrix_ig 2
|
|
#define def_k_cog_activation 3
|
|
//---
|
|
#define def_k_CaclHiddenGradient 2
|
|
#define def_k_chg_matrix_w 0
|
|
#define def_k_chg_matrix_g 1
|
|
#define def_k_chg_matrix_o 2
|
|
#define def_k_chg_matrix_ig 3
|
|
#define def_k_chg_outputs 4
|
|
#define def_k_chg_activation 5
|
|
//---
|
|
#define def_k_UpdateWeightsMomentum 3
|
|
#define def_k_uwm_matrix_w 0
|
|
#define def_k_uwm_matrix_g 1
|
|
#define def_k_uwm_matrix_i 2
|
|
#define def_k_uwm_matrix_dw 3
|
|
#define def_k_uwm_inputs 4
|
|
#define def_k_uwm_learning_rates 5
|
|
#define def_k_uwm_momentum 6
|
|
#define def_k_uwm_optimizer 7
|
|
//---
|
|
#define def_k_UpdateWeightsAdam 4
|
|
#define def_k_uwa_matrix_w 0
|
|
#define def_k_uwa_matrix_g 1
|
|
#define def_k_uwa_matrix_i 2
|
|
#define def_k_uwa_matrix_m 3
|
|
#define def_k_uwa_matrix_v 4
|
|
#define def_k_uwa_inputs 5
|
|
#define def_k_uwa_l 6
|
|
#define def_k_uwa_b1 7
|
|
#define def_k_uwa_b2 8
|
|
//---
|
|
#define def_k_FeedForwardProof 15
|
|
#define def_k_ffp_matrix_i 0
|
|
#define def_k_ffp_matrix_o 1
|
|
#define def_k_ffp_inputs 2
|
|
#define def_k_ffp_window 3
|
|
#define def_k_ffp_step 4
|
|
//---
|
|
#define def_k_CalcInputGradientProof 16
|
|
#define def_k_cigp_matrix_i 0
|
|
#define def_k_cigp_matrix_g 1
|
|
#define def_k_cigp_matrix_o 2
|
|
#define def_k_cigp_matrix_ig 3
|
|
#define def_k_cigp_outputs 4
|
|
#define def_k_cigp_window 5
|
|
#define def_k_cigp_step 6
|
|
//---
|
|
#define def_k_FeedForwardConv 5
|
|
#define def_k_ffc_matrix_w 0
|
|
#define def_k_ffc_matrix_i 1
|
|
#define def_k_ffc_matrix_o 2
|
|
#define def_k_ffc_inputs 3
|
|
#define def_k_ffc_step 4
|
|
#define def_k_ffc_window_in 5
|
|
#define def_k_ffc_window_out 6
|
|
#define def_k_ffc_activation 7
|
|
//---
|
|
#define def_k_CalcHiddenGradientConv 6
|
|
#define def_k_chgc_matrix_w 0
|
|
#define def_k_chgc_matrix_g 1
|
|
#define def_k_chgc_matrix_o 2
|
|
#define def_k_chgc_matrix_ig 3
|
|
#define def_k_chgc_outputs 4
|
|
#define def_k_chgc_step 5
|
|
#define def_k_chgc_window_in 6
|
|
#define def_k_chgc_window_out 7
|
|
#define def_k_chgc_activation 8
|
|
//---
|
|
#define def_k_UpdateWeightsConvMomentum 7
|
|
#define def_k_uwcm_matrix_w 0
|
|
#define def_k_uwcm_matrix_g 1
|
|
#define def_k_uwcm_matrix_i 2
|
|
#define def_k_uwcm_matrix_dw 3
|
|
#define def_k_uwcm_inputs 4
|
|
#define def_k_uwcm_learning_rates 5
|
|
#define def_k_uwcm_momentum 6
|
|
#define def_k_uwcm_window_in 7
|
|
#define def_k_uwcm_window_out 8
|
|
#define def_k_uwcm_step 9
|
|
#define def_k_uwcm_optimizer 10
|
|
//---
|
|
#define def_k_UpdateWeightsConvAdam 8
|
|
#define def_k_uwca_matrix_w 0
|
|
#define def_k_uwca_matrix_g 1
|
|
#define def_k_uwca_matrix_i 2
|
|
#define def_k_uwca_matrix_m 3
|
|
#define def_k_uwca_matrix_v 4
|
|
#define def_k_uwca_inputs 5
|
|
#define def_k_uwca_l 6
|
|
#define def_k_uwca_b1 7
|
|
#define def_k_uwca_b2 8
|
|
#define def_k_uwca_window_in 9
|
|
#define def_k_uwca_window_out 10
|
|
#define def_k_uwca_step 11
|
|
//---
|
|
//--- LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient flows back into
|
|
//--- h_prev/c_prev from a prior step).
|
|
#define def_k_LSTM_Gates 9
|
|
#define def_k_lstmg_matrix_w 0
|
|
#define def_k_lstmg_hidden_prev 1
|
|
#define def_k_lstmg_inputs 2
|
|
#define def_k_lstmg_concatenated 3
|
|
#define def_k_lstmg_hidden_size 4
|
|
#define def_k_lstmg_input_size 5
|
|
//---
|
|
#define def_k_LSTM_State 10
|
|
#define def_k_lstms_concatenated 0
|
|
#define def_k_lstms_memory 1
|
|
#define def_k_lstms_hidden_prev 2
|
|
#define def_k_lstms_hidden_cache 3
|
|
#define def_k_lstms_output 4
|
|
#define def_k_lstms_hidden_size 5
|
|
//---
|
|
#define def_k_LSTM_GateGradient 11
|
|
#define def_k_lstmgg_gradient 0
|
|
#define def_k_lstmgg_memory 1
|
|
#define def_k_lstmgg_concatenated 2
|
|
#define def_k_lstmgg_concatenated_gradient 3
|
|
#define def_k_lstmgg_hidden_size 4
|
|
//---
|
|
#define def_k_LSTM_WeightsGradient 12
|
|
#define def_k_lstmwg_concatenated_gradient 0
|
|
#define def_k_lstmwg_hidden_cache 1
|
|
#define def_k_lstmwg_inputs 2
|
|
#define def_k_lstmwg_weights_gradient 3
|
|
#define def_k_lstmwg_hidden_size 4
|
|
#define def_k_lstmwg_input_size 5
|
|
//---
|
|
#define def_k_LSTM_InputsGradient 13
|
|
#define def_k_lstmig_concatenated_gradient 0
|
|
#define def_k_lstmig_matrix_w 1
|
|
#define def_k_lstmig_inputs_gradient 2
|
|
#define def_k_lstmig_hidden_size 3
|
|
#define def_k_lstmig_input_size 4
|
|
//---
|
|
#define def_k_LSTM_UpdateWeightsAdam 14
|
|
#define def_k_lstmuwa_matrix_w 0
|
|
#define def_k_lstmuwa_weights_gradient 1
|
|
#define def_k_lstmuwa_matrix_m 2
|
|
#define def_k_lstmuwa_matrix_v 3
|
|
#define def_k_lstmuwa_l 4
|
|
#define def_k_lstmuwa_b1 5
|
|
#define def_k_lstmuwa_b2 6
|
|
//---
|
|
// SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see
|
|
// AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body.
|
|
#define def_k_LSTM_UpdateWeightsMomentum 17
|
|
#define def_k_lstmuwm_matrix_w 0
|
|
#define def_k_lstmuwm_weights_gradient 1
|
|
#define def_k_lstmuwm_matrix_dw 2
|
|
#define def_k_lstmuwm_learning_rates 3
|
|
#define def_k_lstmuwm_momentum 4
|
|
#define def_k_lstmuwm_optimizer 5
|
|
//---
|
|
// Sequence LSTM. One launch PER TIMESTEP - the recurrence is sequential and OpenCL
|
|
// barriers only span a work-group, so the host loop is what provides the global
|
|
// ordering. See the block comment above LSTM_SeqStepForward in AI\Network.cl.
|
|
#define def_k_LSTM_SeqStepForward 18
|
|
#define def_k_lsf_matrix_w 0
|
|
#define def_k_lsf_inputs 1
|
|
#define def_k_lsf_cache_gates 2
|
|
#define def_k_lsf_cache_cell 3
|
|
#define def_k_lsf_cache_hidden 4
|
|
#define def_k_lsf_output 5
|
|
#define def_k_lsf_hidden_size 6
|
|
#define def_k_lsf_step_inputs 7
|
|
#define def_k_lsf_steps 8
|
|
#define def_k_lsf_t 9
|
|
//---
|
|
#define def_k_LSTM_SeqStepGateGrad 19
|
|
#define def_k_lsgg_out_gradient 0
|
|
#define def_k_lsgg_dh_buf 1
|
|
#define def_k_lsgg_dc_buf 2
|
|
#define def_k_lsgg_cache_gates 3
|
|
#define def_k_lsgg_cache_cell 4
|
|
#define def_k_lsgg_gate_grad 5
|
|
#define def_k_lsgg_hidden_size 6
|
|
#define def_k_lsgg_steps 7
|
|
#define def_k_lsgg_t 8
|
|
//---
|
|
#define def_k_LSTM_SeqStepWeightGrad 20
|
|
#define def_k_lswg_gate_grad 0
|
|
#define def_k_lswg_cache_hidden 1
|
|
#define def_k_lswg_inputs 2
|
|
#define def_k_lswg_weights_gradient 3
|
|
#define def_k_lswg_hidden_size 4
|
|
#define def_k_lswg_step_inputs 5
|
|
#define def_k_lswg_t 6
|
|
//---
|
|
#define def_k_LSTM_SeqStepInputGrad 21
|
|
#define def_k_lsig_gate_grad 0
|
|
#define def_k_lsig_matrix_w 1
|
|
#define def_k_lsig_inputs_gradient 2
|
|
#define def_k_lsig_dh_buf 3
|
|
#define def_k_lsig_hidden_size 4
|
|
#define def_k_lsig_step_inputs 5
|
|
#define def_k_lsig_t 6
|
|
//---
|
|
//--- Mini-batch gradient accumulation - see the block comment above AccumulateWeightGrad in
|
|
//--- AI\Network.cl.
|
|
#define def_k_AccumulateWeightGrad 22
|
|
#define def_k_awg_matrix_acc 0
|
|
#define def_k_awg_matrix_g 1
|
|
#define def_k_awg_matrix_i 2
|
|
#define def_k_awg_inputs 3
|
|
//---
|
|
#define def_k_AccumulateWeightGradConv 23
|
|
#define def_k_awgc_matrix_acc 0
|
|
#define def_k_awgc_matrix_g 1
|
|
#define def_k_awgc_matrix_i 2
|
|
#define def_k_awgc_inputs 3
|
|
#define def_k_awgc_window_in 4
|
|
#define def_k_awgc_window_out 5
|
|
#define def_k_awgc_step 6
|
|
//---
|
|
#define def_k_AccumulateBufferInto 24
|
|
#define def_k_abi_dst 0
|
|
#define def_k_abi_src 1
|
|
//---
|
|
//--- Mini-batch APPLY.
|
|
#define def_k_ApplyAccumAdam 25
|
|
#define def_k_aaa_matrix_w 0
|
|
#define def_k_aaa_matrix_acc 1
|
|
#define def_k_aaa_matrix_m 2
|
|
#define def_k_aaa_matrix_v 3
|
|
#define def_k_aaa_scale 4
|
|
#define def_k_aaa_l 5
|
|
#define def_k_aaa_b1 6
|
|
#define def_k_aaa_b2 7
|
|
//---
|
|
#define def_k_ApplyAccumMomentum 26
|
|
#define def_k_aam_matrix_w 0
|
|
#define def_k_aam_matrix_acc 1
|
|
#define def_k_aam_matrix_dw 2
|
|
#define def_k_aam_scale 3
|
|
#define def_k_aam_lr 4
|
|
#define def_k_aam_momentum 5
|
|
//--- One latch for the whole process, not per net: the apply kernels either built on this machine's
|
|
//--- OpenCL device or they did not, and that answer cannot change mid-run.
|
|
bool g_applyAccumKernelUsable = true;
|
|
//---
|
|
//--- Batch-norm kernels (2026-08-09) - see the BATCH NORM block in AI\Network.cl.
|
|
#define def_k_BatchNormForward 27
|
|
#define def_k_bnf_matrix_i 0
|
|
#define def_k_bnf_matrix_o 1
|
|
#define def_k_bnf_options 2
|
|
#define def_k_bnf_w 3
|
|
#define def_k_bnf_frozen 4
|
|
//---
|
|
#define def_k_BatchNormHiddenGrad 28
|
|
#define def_k_bnh_matrix_g 0
|
|
#define def_k_bnh_prev_o 1
|
|
#define def_k_bnh_prev_g 2
|
|
#define def_k_bnh_options 3
|
|
#define def_k_bnh_activation 4
|
|
//---
|
|
#define def_k_BatchNormAccumGammaBeta 29
|
|
#define def_k_bna_matrix_g 0
|
|
#define def_k_bna_options 1
|
|
#define def_k_bna_acc 2
|
|
//---
|
|
#define def_k_BatchNormApplyGammaBeta 30
|
|
#define def_k_bnp_options 0
|
|
#define def_k_bnp_acc 1
|
|
#define def_k_bnp_scale 2
|
|
#define def_k_bnp_lt 3
|
|
#define def_k_bnp_b1 4
|
|
#define def_k_bnp_b2 5
|
|
#define def_k_bnp_lr 6
|
|
#define def_k_bnp_momentum 7
|
|
#define def_k_bnp_optimizer 8
|
|
bool g_bnKernelUsable = true;
|
|
//---
|
|
// The Adam betas are ordinary inputs (book defaults 0.9/0.999) - see
|
|
// AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era
|
|
// same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root
|
|
// cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE
|
|
// gradient in backProp()/backPropOCL(), which addresses it more directly than shortening beta1 ever
|
|
// could. Both are passed as runtime parameters into every backend (not baked into compiled
|
|
// kernels - see DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl's UpdateWeightsAdam
|
|
// signatures), so they're safe to expose as ordinary inputs.
|
|
// Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had
|
|
// already gone unstable (real collapses were happening at weight magnitudes several orders of
|
|
// magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh
|
|
// engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually
|
|
// reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below.
|
|
#define MAX_WEIGHT 100.0
|
|
// Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in
|
|
// DirectML\WarriorCPU.cpp/WarriorDML.cpp/AI\Network.cl (all four backends kept in sync) - see
|
|
// WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright
|
|
// +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of
|
|
// training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi-
|
|
// hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles.
|
|
// 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates,
|
|
// ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's
|
|
// sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a
|
|
// weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the
|
|
// discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the
|
|
// calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the
|
|
// spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt
|
|
// dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay
|
|
// strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins
|
|
// argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth.
|
|
#define WEIGHT_DECAY 0.001
|
|
// Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full
|
|
// rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden
|
|
// Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras),
|
|
// most likely from 5x back-to-back oversampling replay building artificially correlated momentum.
|
|
// Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the
|
|
// post-update weight value and is far too loose (1e6) to prevent this.
|
|
#define MAX_WEIGHT_DELTA 0.1
|
|
//--- Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see
|
|
//--- SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near
|
|
//--- its activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes
|
|
//--- that neuron's entire backprop gradient contribution regardless of how wrong its output is.
|
|
#define MIN_ACTIVATION_DERIVATIVE 1.0e-3
|
|
//--- Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND
|
|
//--- read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored
|
|
//--- against a different distribution than it was trained on).
|
|
#define CLASS_LOGIT_SCALE 6.0
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#resource "Network.cl" as string cl_program
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
enum ENUM_ACTIVATION
|
|
{
|
|
NONE,
|
|
TANH,
|
|
SIGMOID,
|
|
PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Translates ENUM_ACTIVATION to the "activation" int code every |
|
|
//| native compute backend actually understands (Network.cl's |
|
|
//| kernels, and the mirrored Activation()/inline switches in |
|
|
//| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, |
|
|
//| and deliberately anything else (incl. |
|
|
//+------------------------------------------------------------------+
|
|
int NativeActivationCode(ENUM_ACTIVATION value)
|
|
{
|
|
switch(value)
|
|
{
|
|
case TANH: return 0;
|
|
case SIGMOID: return 1;
|
|
case PRELU: return 2;
|
|
default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches
|
|
}
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the |
|
|
//| load-time architecture repair (CNet::EnforceOutputActivation) so |
|
|
//| the log names the stale value it found instead of printing a bare |
|
|
//| integer nobody can decode months later. |
|
|
//+------------------------------------------------------------------+
|
|
string ActivationName(ENUM_ACTIVATION value)
|
|
{
|
|
switch(value)
|
|
{
|
|
case NONE: return "NONE";
|
|
case TANH: return "TANH";
|
|
case SIGMOID: return "SIGMOID";
|
|
case PRELU: return "PRELU";
|
|
}
|
|
return "UNKNOWN(" + IntegerToString((int)value) + ")";
|
|
}
|
|
//--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\
|
|
//--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead
|
|
//--- of this AI header - without a duplicate-definition error.
|
|
#ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED
|
|
#define WARRIOR_ENUM_OPTIMIZATION_DEFINED
|
|
//--- 2026-07-28: a third DFA entry was removed. SGD/ADAM keep their ordinal values 0/1 -
|
|
//--- m_optimizationAlgo feeds the weights-filename fingerprint, so these must never be renumbered.
|
|
enum ENUM_OPTIMIZATION
|
|
{
|
|
SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras)
|
|
ADAM // Adam (adaptive step, faster convergence, can overfit)
|
|
};
|
|
#endif
|
|
//---
|
|
enum ENUM_BUFFERS
|
|
{
|
|
WEIGHTS,
|
|
DELTA_WEIGHTS,
|
|
OUTPUT,
|
|
GRADIENT,
|
|
FIRST_MOMENTUM,
|
|
SECOND_MOMENTUM
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "NeuronPrimitives.mqh"
|
|
class CLayer;
|
|
//---
|
|
class CNeuronBase : public CObject
|
|
{
|
|
protected:
|
|
double outputVal;
|
|
double prevVal;
|
|
uint m_myIndex;
|
|
double gradient;
|
|
CArrayCon *Connections;
|
|
ENUM_ACTIVATION activation;
|
|
ENUM_OPTIMIZATION optimization;
|
|
int t;
|
|
//---
|
|
virtual bool feedForward(CLayer *prevLayer) { return false; }
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; }
|
|
virtual double activationFunction(double x);
|
|
virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); }
|
|
virtual double TanhFunction(double x) { return tanh(x); }
|
|
virtual CLayer *getOutputLayer(void) { return NULL; }
|
|
public:
|
|
CNeuronBase(void);
|
|
~CNeuronBase(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0);
|
|
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
|
|
//--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back
|
|
//--- what a Load() restored from disk without caring which neuron model the net was built from.
|
|
virtual ENUM_ACTIVATION Activation(void) { return activation; }
|
|
//---
|
|
static double alpha;
|
|
//---
|
|
virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; }
|
|
virtual double getOutputVal() { return outputVal; }
|
|
virtual double getPrevVal() { return prevVal; }
|
|
virtual void setGradient(double val) { gradient = val; }
|
|
virtual double getGradient() { return gradient; }
|
|
virtual CArrayCon *getConnections() { return Connections;}
|
|
virtual double activationFunctionDerivative(double x);
|
|
virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); }
|
|
virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); }
|
|
//---
|
|
virtual bool feedForward(CObject *&SourceObject);
|
|
virtual bool calcHiddenGradients(CObject *&TargetObject);
|
|
virtual bool updateInputWeights(CLayer *prevLayer) { return false; }
|
|
virtual bool updateInputWeights(CObject *SourceObject);
|
|
//---
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle)
|
|
{
|
|
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
|
|
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
return(Connections.Load(file_handle));
|
|
}
|
|
//--- Forget the optimizer's trajectory memory (Adam m/v, SGD momentum, step counter) while
|
|
//--- leaving the weights untouched - see CNet::ResetOptimizerState for when and why.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
t = 1;
|
|
if(CheckPointer(Connections) == POINTER_INVALID)
|
|
return true;
|
|
for(int i = 0; i < Connections.Total(); i++)
|
|
{
|
|
CConnection *con = Connections.At(i);
|
|
if(CheckPointer(con) == POINTER_INVALID)
|
|
continue;
|
|
con.deltaWeight = 0.0;
|
|
con.mt = 0.0;
|
|
con.vt = 0.0;
|
|
}
|
|
return true;
|
|
}
|
|
//---
|
|
virtual int Type(void) const { return defNeuronBase; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "NeuronCPU.mqh"
|
|
class COpenCLMy : public COpenCL
|
|
{
|
|
public:
|
|
COpenCLMy(void) {};
|
|
~COpenCLMy(void) {};
|
|
template<typename T>
|
|
int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
|
|
};
|
|
#include "NeuronDirectML.mqh"
|
|
class CLayer: public CArrayObj
|
|
{
|
|
private:
|
|
uint iOutputs;
|
|
int iFileHandle;
|
|
COpenCLMy *OpenCL;
|
|
CDirectMLMy *DirectML;
|
|
|
|
public:
|
|
CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CDirectMLMy *DirectML = NULL);
|
|
~CLayer(void) {};
|
|
//--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below.
|
|
bool CreateElementScaled(int const index, double weighScale);
|
|
//--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool
|
|
//--- CreateElement(const int index)` - because it is the real virtual override that
|
|
//--- CArrayObj::Load() dispatches through, and CArrayObj::Load() is how EVERY saved layer is
|
|
//--- read back (CLayer::Load -> CNet::Load).
|
|
virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); }
|
|
virtual void IncreaseTotal() { m_data_total++; }
|
|
virtual int Type(void) const { return defLayer; }
|
|
virtual bool Load(const int file_handle);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "ArrayLayer.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronPool : public CNeuronBase
|
|
{
|
|
protected:
|
|
CLayer *OutputLayer;
|
|
int iWindow;
|
|
int iStep;
|
|
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
|
|
public:
|
|
CNeuronPool(void) {};
|
|
~CNeuronPool(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
|
|
//---
|
|
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronPool; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronConv : public CNeuronPool
|
|
{
|
|
protected:
|
|
double param; //PReLU param
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual double activationFunction(double x);
|
|
virtual bool updateInputWeights(CLayer *prevLayer);
|
|
public:
|
|
CNeuronConv() : param(0.01) { };
|
|
~CNeuronConv(void) { };
|
|
//---
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
virtual double activationFunctionDerivative(double x);
|
|
virtual int Type(void) const { return defNeuronConv; }
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "LayerDescription.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNet
|
|
{
|
|
protected:
|
|
double dLogitAdjust[3];
|
|
bool bLogitAdjust;
|
|
void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0);
|
|
bool InitOpenCL(void);
|
|
bool InitDirectML(void);
|
|
//--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference.
|
|
bool feedForwardCPU(CArrayDouble *inputVals);
|
|
public:
|
|
CNet(CArrayObj *Description);
|
|
~CNet(void);
|
|
bool feedForward(CArrayDouble *inputVals);
|
|
//--- sampleWeight scales this example's output-layer gradient before it propagates back through the
|
|
//--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling
|
|
//--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same
|
|
//--- example multiple times).
|
|
void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0);
|
|
//--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class
|
|
//--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass
|
|
//--- only.
|
|
void SetLogitAdjustment(const double &offsets[]);
|
|
void ClearLogitAdjustment(void) { bLogitAdjust = false; }
|
|
void getResults(CArrayDouble *&resultVals) ;
|
|
double getRecentAverageError() { return recentAverageError; }
|
|
//--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see
|
|
//--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array
|
|
//--- when there is nothing to persist/restore.
|
|
bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]);
|
|
//--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it
|
|
//--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see
|
|
//--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud.
|
|
bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false);
|
|
//--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). In-place weight copy
|
|
//--- uses only getWeights/setWeights, which the per-era shadow blend already exercises
|
|
//--- successfully on that backend.
|
|
string LayerLearningReport(void);
|
|
bool CaptureWeights(void);
|
|
bool RestoreWeights(void);
|
|
//--- MINI-BATCH CONTROL (2026-08-09 audit, F4). SetBatchSize() is how training asks for
|
|
//--- accumulation; 1 restores the exact per-sample path the engine used before.
|
|
void SetBatchSize(int size) { m_batchSizeRequested = (size > 1 ? size : 1); }
|
|
int BatchSize(void);
|
|
bool FlushBatch(void);
|
|
//--- Zero every neuron's optimizer state - Adam first/second moments, SGD momentum deltas, the
|
|
//--- per-neuron bias-correction step counters, and batch-norm's gamma/beta moment slots - while
|
|
//--- leaving weights, activations and batch-norm running STATISTICS untouched (those belong to
|
|
//--- the model, not the optimizer).
|
|
bool ResetOptimizerState(void);
|
|
//--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another
|
|
//--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight +
|
|
//--- tau*live.weight.
|
|
bool BlendWeightsFrom(CNet &live, double tau);
|
|
//--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly
|
|
//--- initialized and carry the real learning signal) of each output neuron's incoming weight block,
|
|
//--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site
|
|
//--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to
|
|
//--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more
|
|
//--- non-majority classes than the true base rate warrants until backProp corrects it over many
|
|
//--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the
|
|
//--- OpenCL/DirectML batched neuron model (CNeuronBaseOCL) this project actually runs on - returns
|
|
//--- false (no-op) rather than corrupt anything if that assumption doesn't hold.
|
|
bool SeedOutputLayerBias(const double &biasValues[]);
|
|
//--- Pure-MQL5 (no OpenCL/DirectML/DLL) inference mode. Training/optimization never set this
|
|
//--- (they always want a backend), so their behaviour is unchanged.
|
|
void SetCpuInference(bool v) { m_cpuInference = v; }
|
|
bool CpuInference(void) const { return m_cpuInference; }
|
|
//--- Re-assert the output layer's activation after a Load(), and report what it used to be. WHY
|
|
//--- THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. Every reload restores
|
|
//--- whatever is on disk and the next Save() writes it back out - a wrong value can never heal
|
|
//--- on its own, while the source file reads as though it were already fixed.
|
|
bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous);
|
|
//--- Receptive field of the first conv layer as LOADED, for the stale-architecture check in
|
|
//--- CExpertSignalAIBase::EnforceTopologyContract. 0 when the net has no conv layer.
|
|
uint FirstConvWindow(void);
|
|
//--- Freeze/unfreeze every batch-normalization layer's running statistics
|
|
//--- (AI\NeuronBatchNorm.mqh). Frozen, a forward pass is a pure function of its input; unfrozen
|
|
//--- (the default) it also advances the statistics.
|
|
void SetBatchNormFrozen(bool frozen);
|
|
//--- Current freeze state (the first normalization layer's flag; they only ever move together).
|
|
//--- False on a net with no normalization layers.
|
|
bool GetBatchNormFrozen(void);
|
|
//---
|
|
static double recentAverageSmoothingFactor;
|
|
|
|
private:
|
|
CArrayLayer *layers;
|
|
COpenCLMy *opencl;
|
|
CDirectMLMy *directml;
|
|
double recentAverageError;
|
|
bool m_cpuInference;
|
|
//--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per
|
|
//--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture.
|
|
CArrayObj *m_weightSnapshot;
|
|
bool m_haveWeightSnapshot;
|
|
//--- Previous call's per-layer weight L2 norms, for LayerLearningReport(). Sized lazily to the layer
|
|
//--- count; -1 marks "no baseline yet" so the first report says (init) instead of a bogus 0% change.
|
|
double m_prevLayerNorm[];
|
|
//--- Previous call's per-layer weight VECTORS, for the |dW| term of LayerLearningReport(). One
|
|
//--- CArrayDouble per layer index (empty for layers that own no weights), host-only.
|
|
CArrayObj *m_prevLayerWeights;
|
|
//--- One-shot latch for BlendWeightsFrom's skip warning - it runs every era, and the condition it
|
|
//--- reports is permanent, so the second print would only be noise.
|
|
bool m_blendSkipLogged;
|
|
//--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). A host
|
|
//--- that HAS OpenCL never latches, so every CNet still gets its own COpenCLMy. Failure messages
|
|
//--- stay loud every time.
|
|
static bool s_openclUnavailable;
|
|
static bool s_computeTierLogged;
|
|
//--- Mini-batch state. m_batchSizeRequested is what training asked for; m_batchCount is how many
|
|
//--- samples have been accumulated into the current batch. m_batchKernelsOk records whether this
|
|
//--- net's OpenCL device actually built the accumulation kernels - false forces per-sample updates
|
|
//--- rather than failing, so an old device trains exactly as it did before (see InitOpenCL).
|
|
//--- m_batchBegun guards BeginBatch so the first sample of a batch zeroes the accumulators exactly
|
|
//--- once. m_batchWarned latches the one-time "cannot batch on this tier" notice.
|
|
int m_batchSizeRequested;
|
|
int m_batchCount;
|
|
bool m_batchKernelsOk;
|
|
//--- Whether the DEVICE-SIDE apply kernels built (def_k_ApplyAccumAdam). Conflating them would
|
|
//--- turn a missing optimisation into a change of optimizer.
|
|
bool m_applyKernelsOk;
|
|
bool m_batchBegun;
|
|
bool m_batchWarned;
|
|
//--- Zero every accumulator, starting a fresh batch.
|
|
bool BeginBatch(void);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronLSTM : public CNeuronPool
|
|
{
|
|
protected:
|
|
CLayer *ForgetGate;
|
|
CLayer *InputGate;
|
|
CLayer *OutputGate;
|
|
CLayer *NewContent;
|
|
CArrayDouble *Memory;
|
|
CArrayDouble *PrevMemory;
|
|
CArrayDouble *Input;
|
|
CArrayDouble *InputGradient;
|
|
//---
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual bool updateInputWeights(CLayer *prevLayer);
|
|
virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data);
|
|
virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type);
|
|
//--- Fills `result` rather than returning a fresh CArrayDouble. The old shape handed every caller
|
|
//--- an object to delete on each of its own error paths, and none of them did - see feedForward().
|
|
virtual bool CalculateGate(CLayer *gate, CArrayDouble *sequence, CArrayDouble &result);
|
|
bool AccumulateGateInputGradient(CLayer *gate, const int i, const int n, double &value);
|
|
|
|
public:
|
|
CNeuronLSTM(void);
|
|
~CNeuronLSTM(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
|
|
//---
|
|
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronLSTM; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
template<typename T>
|
|
int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags)
|
|
{
|
|
int result = -1;
|
|
for(int i = 0; i < m_buffers_total; i++)
|
|
{
|
|
if(m_buffers[i] != INVALID_HANDLE)
|
|
continue;
|
|
result = i;
|
|
break;
|
|
}
|
|
//---
|
|
if(result < 0)
|
|
{
|
|
if(ArrayResize(m_buffers, m_buffers_total + 1) > 0)
|
|
{
|
|
m_buffers_total = ArraySize(m_buffers);
|
|
result = m_buffers_total - 1;
|
|
m_buffers[result] = INVALID_HANDLE;
|
|
}
|
|
else
|
|
return result;
|
|
}
|
|
//---
|
|
if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags))
|
|
return -1;
|
|
//---
|
|
return result;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "BufferDouble.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronBaseOCL : public CObject
|
|
{
|
|
protected:
|
|
COpenCLMy *OpenCL;
|
|
CDirectMLMy *DirectML;
|
|
CBufferDouble *Output;
|
|
CBufferDouble *PrevOutput;
|
|
CBufferDouble *Weights;
|
|
CBufferDouble *DeltaWeights;
|
|
CBufferDouble *Gradient;
|
|
CBufferDouble *FirstMomentum;
|
|
CBufferDouble *SecondMomentum;
|
|
//--- MINI-BATCH ACCUMULATOR (2026-08-09 audit, F4). Same shape as Weights; holds the SUM of this
|
|
//--- weight block's per-sample gradients over the current batch.
|
|
CBufferDouble *GradAccum;
|
|
//---
|
|
const double alpha;
|
|
int t;
|
|
//---
|
|
ENUM_ACTIVATION activation;
|
|
ENUM_OPTIMIZATION optimization;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);
|
|
//--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only
|
|
//--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists -
|
|
//--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU).
|
|
bool BackendBufferCreate(CBufferDouble *buf)
|
|
{
|
|
if(CheckPointer(buf) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID)
|
|
return buf.BufferCreate(OpenCL);
|
|
if(CheckPointer(DirectML) != POINTER_INVALID)
|
|
return buf.BufferCreate(DirectML);
|
|
return true; // no backend: keep host m_data, skip device allocation
|
|
}
|
|
//--- THE optimizer step, and the only copy of it in the batched path: one weight block, one Adam
|
|
//--- or SGD+momentum update on `acc * scale`, then the accumulator is zeroed.
|
|
bool ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale, int total);
|
|
bool ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale);
|
|
//--- Lazily allocate the mini-batch accumulator to match `src` (Weights / WeightsConv /
|
|
//--- WeightsLSTM, whichever block the caller accumulates into).
|
|
bool EnsureGradAccumFor(CBufferDouble *&target, CBufferDouble *src)
|
|
{
|
|
if(CheckPointer(src) == POINTER_INVALID || src.Total() <= 0)
|
|
return false;
|
|
if(CheckPointer(target) != POINTER_INVALID && target.Total() == src.Total())
|
|
return true;
|
|
if(CheckPointer(target) != POINTER_INVALID)
|
|
delete target;
|
|
target = new CBufferDouble();
|
|
if(CheckPointer(target) == POINTER_INVALID)
|
|
return false;
|
|
if(!target.BufferInit(src.Total(), 0.0))
|
|
return false;
|
|
return BackendBufferCreate(target);
|
|
}
|
|
bool EnsureGradAccum(CBufferDouble *src) { return EnsureGradAccumFor(GradAccum, src); }
|
|
|
|
public:
|
|
CNeuronBaseOCL(void);
|
|
~CNeuronBaseOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
|
|
//---
|
|
virtual int getOutputIndex(void) { return Output.GetIndex(); }
|
|
virtual int getGradientIndex(void) { return Gradient.GetIndex(); }
|
|
virtual int getWeightsIndex(void) { return Weights.GetIndex(); }
|
|
virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); }
|
|
virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); }
|
|
virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();}
|
|
//---
|
|
virtual int getOutputVal(double &values[]) { return Output.GetData(values); }
|
|
virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); }
|
|
virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); }
|
|
virtual int getGradient(double &values[]) { return Gradient.GetData(values); }
|
|
//--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by
|
|
//--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call
|
|
//--- (which only computes the raw, unweighted delta) and before the backward pass reads this same
|
|
//--- buffer to propagate into the hidden layers.
|
|
virtual bool setGradient(const double &values[])
|
|
{
|
|
int count = ArraySize(values);
|
|
for(int i = 0; i < count; i++)
|
|
if(!Gradient.Update(i, values[i]))
|
|
return false;
|
|
return Gradient.BufferWrite();
|
|
}
|
|
// Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created,
|
|
// so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here.
|
|
virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); }
|
|
//--- Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment
|
|
//--- (see that method's declaration comment) - writes a full replacement weight array back to
|
|
//--- this buffer's device-side (DLL/OpenCL/DirectML) storage.
|
|
virtual bool setWeights(double &values[])
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
if(!Weights.AssignArray(values))
|
|
return false;
|
|
return Weights.BufferWrite();
|
|
}
|
|
//--- Forget the optimizer's trajectory memory: Adam first/second moments, SGD's previous-delta
|
|
//--- buffer, and the bias-correction step counter, weights untouched. See
|
|
//--- CNet::ResetOptimizerState for the restore/warm-restart rationale.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
t = 1;
|
|
bool ok = ZeroOptimizerBuffer(FirstMomentum);
|
|
ok = ZeroOptimizerBuffer(SecondMomentum) && ok;
|
|
ok = ZeroOptimizerBuffer(DeltaWeights) && ok;
|
|
return ok;
|
|
}
|
|
virtual int Neurons(void) { return Output.Total(); }
|
|
virtual ENUM_ACTIVATION Activation(void) { return activation; }
|
|
virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); }
|
|
//--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to
|
|
//--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/
|
|
//--- getOutputVal() which route through BufferRead() (a device read that fails without a backend).
|
|
double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; }
|
|
double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; }
|
|
int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; }
|
|
//--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only
|
|
//--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense
|
|
//--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math.
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL);
|
|
//--- Host-side input write for the CPU inference path's layer 0, and host-side output read for
|
|
//--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates.
|
|
bool SetInputsCPU(CArrayDouble *inputVals);
|
|
int GetOutputsCPU(CArrayDouble *values);
|
|
//---
|
|
virtual bool feedForward(CObject *SourceObject);
|
|
virtual bool calcHiddenGradients(CObject *TargetObject);
|
|
virtual bool calcOutputGradients(CArrayDouble *Target);
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool updateInputWeights(CObject *SourceObject);
|
|
//--- MINI-BATCH PAIR (2026-08-09 audit, F4). accumulateInputWeightGrads() adds THIS sample's
|
|
//--- per-weight gradient into GradAccum without touching the weights;
|
|
//--- ApplyAccumulatedGradients() then takes ONE optimizer step on the batch mean and clears the
|
|
//--- accumulator.
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool accumulateInputWeightGrads(CObject *SourceObject);
|
|
virtual bool ApplyAccumulatedGradients(double scale);
|
|
//--- Zero the accumulator at the start of a batch. Separate from ApplyAccumulatedGradients so a
|
|
//--- discarded partial batch (a stopped run) can be cleared without taking a step from it.
|
|
virtual bool BeginGradAccum(void);
|
|
//---
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
//---
|
|
virtual int Type(void) const { return defNeuronBaseOCL; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
#include "NeuronOCLConvPool.mqh"
|
|
#include "NeuronBatchNorm.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//--- Marks a .nnw LSTM record as the SEQUENCE format. Chosen so it cannot collide with any value the
|
|
//--- pre-sequence format could have written in that slot (an input width, i.e. -1 or a positive count).
|
|
#define LSTM_SEQ_SAVE_TAG (-424242)
|
|
//--- Initial bias of the FORGET gate (gate 0). Every other weight starts near zero; this one must
|
|
//--- not.
|
|
#define LSTM_FORGET_BIAS_INIT (1.0)
|
|
//+------------------------------------------------------------------+
|
|
//| GPU-accelerated LSTM layer (OpenCL + DirectML). Derived from |
|
|
//| scratch from the standard LSTM equations - NOT ported from the |
|
|
//| NeuroNet_DNG reference (see the note above the LSTM kernels in |
|
|
//| Network.cl for why). Single-timestep-truncated BPTT: gradient |
|
|
//| does not flow back into h_prev/c_prev from an earlier step. |
|
|
//| Adam-only - Init fails for any other optimization type. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronLSTMOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
int m_iInputs;
|
|
//--- Mini-batch running total of WeightsGradient across the samples of one batch - see the note on
|
|
//--- accumulateInputWeightGrads below for why WeightsGradient itself cannot serve as it.
|
|
CBufferDouble *GradAccumLSTM;
|
|
//--- Sequence shape. m_iStepInputs is the width of ONE timestep (the per-bar feature count
|
|
//--- reaching this layer), set from the layer descriptor by SetStepWidth() before the first
|
|
//--- feedForward; m_iSteps is then m_iInputs / m_iStepInputs.
|
|
int m_iStepInputs;
|
|
int m_iSteps;
|
|
//--- Per-timestep caches, required by backpropagation-through-time: the backward pass needs each
|
|
//--- step's gate activations, cell state and hidden state, which the single-buffer Concatenated/
|
|
//--- Memory/HiddenCache trio below cannot hold because every step overwrites the last.
|
|
CBufferDouble *CacheGates; // T * 4H, gate order [f,i,o,g]
|
|
CBufferDouble *CacheCell; // T * H, c_t
|
|
CBufferDouble *CacheHidden; // T * H, h_t
|
|
CBufferDouble *WeightsLSTM;
|
|
CBufferDouble *FirstMomentumLSTM;
|
|
CBufferDouble *SecondMomentumLSTM;
|
|
CBufferDouble *DeltaWeightsLSTM;
|
|
CBufferDouble *WeightsGradient;
|
|
CBufferDouble *Concatenated;
|
|
CBufferDouble *ConcatenatedGradient;
|
|
CBufferDouble *Memory;
|
|
CBufferDouble *HiddenCache;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool SetInputs(int count);
|
|
bool AllocateSequenceCaches(void);
|
|
|
|
public:
|
|
CNeuronLSTMOCL(void) : m_iInputs(-1), m_iStepInputs(-1), m_iSteps(-1)
|
|
{
|
|
GradAccumLSTM = NULL;
|
|
CacheGates = NULL;
|
|
CacheCell = NULL;
|
|
CacheHidden = NULL;
|
|
WeightsLSTM = NULL;
|
|
FirstMomentumLSTM = NULL;
|
|
SecondMomentumLSTM = NULL;
|
|
DeltaWeightsLSTM = NULL;
|
|
WeightsGradient = NULL;
|
|
Concatenated = NULL;
|
|
ConcatenatedGradient = NULL;
|
|
Memory = NULL;
|
|
HiddenCache = NULL;
|
|
}
|
|
~CNeuronLSTMOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CDirectMLMy *direct_ml, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
//--- Per-timestep input width, from CLayerDescription::window (see AddLstmStage). Must be called
|
|
//--- between Init() and the first feedForward; <= 0 keeps the legacy single-timestep behaviour.
|
|
//--- Not persisted from here - Save/Load carry it, so a loaded model does not depend on call order.
|
|
void SetStepWidth(int stepInputs) { m_iStepInputs = (stepInputs > 0 ? stepInputs : -1); }
|
|
bool IsSequenceMode(void) const { return (m_iStepInputs > 0 && m_iSteps > 1); }
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronLSTMOCL; }
|
|
// See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the
|
|
// base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
|
|
virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); }
|
|
//--- Build this layer's weight block to match `src`, for a net that was cloned from one whose
|
|
//--- LSTM had not yet run a forward pass.
|
|
virtual bool AdoptShapeFrom(CNeuronLSTMOCL &src)
|
|
{
|
|
if(src.m_iInputs <= 0 || Neurons() != src.Neurons())
|
|
return false;
|
|
m_iStepInputs = src.m_iStepInputs;
|
|
return SetInputs(src.m_iInputs);
|
|
}
|
|
virtual bool setWeightsLSTM(double &values[])
|
|
{
|
|
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
|
|
return false;
|
|
if(!WeightsLSTM.AssignArray(values))
|
|
return false;
|
|
return WeightsLSTM.BufferWrite();
|
|
}
|
|
//--- The LSTM's gate-weight block keeps its own moment/momentum buffers beside the base class's -
|
|
//--- see CNet::ResetOptimizerState.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::ResetOptimizerState();
|
|
ok = ZeroOptimizerBuffer(FirstMomentumLSTM) && ok;
|
|
ok = ZeroOptimizerBuffer(SecondMomentumLSTM) && ok;
|
|
ok = ZeroOptimizerBuffer(DeltaWeightsLSTM) && ok;
|
|
return ok;
|
|
}
|
|
//--- MINI-BATCH. Batching it is therefore just a running total. Hence a separate accumulator
|
|
//--- plus a generic elementwise add (AccumulateBufferInto), which also avoids changing the
|
|
//--- signature of an already-deployed export.
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool BeginGradAccum(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::BeginGradAccum();
|
|
if(CheckPointer(GradAccumLSTM) != POINTER_INVALID && GradAccumLSTM.Total() > 0)
|
|
ok = ZeroOptimizerBuffer(GradAccumLSTM) && ok;
|
|
return ok;
|
|
}
|
|
virtual bool ApplyAccumulatedGradients(double scale)
|
|
{
|
|
bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale);
|
|
ok = ApplyAccumToBlock(WeightsLSTM, GradAccumLSTM, FirstMomentumLSTM, SecondMomentumLSTM,
|
|
DeltaWeightsLSTM, scale) && ok;
|
|
if(optimization == ADAM)
|
|
t++;
|
|
return ok;
|
|
}
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Implementation bodies. |
|
|
//+------------------------------------------------------------------+
|
|
#include "Impl\NeuronBase.mqh"
|
|
#include "Impl\NeuronConvPool.mqh"
|
|
#include "Impl\NeuronLSTM.mqh"
|
|
#include "Impl\Layer.mqh"
|
|
#include "Impl\NetBuild.mqh"
|
|
#include "Impl\NetForward.mqh"
|
|
#include "Impl\NetPersistence.mqh"
|
|
#include "Impl\NetWeights.mqh"
|
|
#include "Impl\NeuronOCLBase.mqh"
|
|
#include "Impl\NeuronOCLLSTM.mqh"
|
|
//+------------------------------------------------------------------+
|