forked from MrBaro75/Warrior_EA
bd46374954
"Hundreds of times slower than a regular EA" decomposed into two
multiplied factors, both measured:
1. THE OPTIMIZER STEP RAN IN INTERPRETED MQL5. The CPU tier shipped
the F4 accumulate exports with deliberately no matching apply
(WarriorCPU.h said so), so on the DLL backend - this box - every
TRAIN_BATCH_SIZE=8 batch fell to the host loop in ApplyAccumToBlock:
a per-weight MQL5 pass through CBufferDouble.At()/Update() plus four
full weight-matrix BufferRead/Write round trips. The 2026-07-26
profile had already shown the per-sample Adam step at 81% of ALL
runtime (feedForward: 8%; feature building: 0.35%) - sqrt+divide
per weight vs one multiply-add; moving it into MQL5 made it worse.
New CPU_ApplyAccumAdam / CPU_ApplyAccumMomentum: one element-wise
ParallelFor takes the batch-mean step and zeroes the accumulator
DLL-side, generic over any flat block (dense/conv/LSTM/batch-norm -
all apply paths funnel through ApplyAccumToBlock, which now tries
the DLL first, with the same one-warning failure latch as the
OpenCL fast path). Math is the shipped step to the last clamp:
sqrt-stored v, ClampDelta, AdamW decay, ClampWeight.
batch_accum_check extended (check 6) and ALL PASS: apply == host
reference at B=8/B=4, accumulator zeroed, and B=1 accumulate+apply
== the unbatched Adam kernel BIT-EXACTLY (kernel-vs-kernel, no
transcription). DLL rebuilt with the shipped /fp:fast recipe.
2. A 24% DUTY CYCLE. Train sliced 120ms per 500ms timer period
(30ms/member x4), leaving the chart thread idle 76% of the time.
Now 300ms total (75ms/member): ~60% duty, ~2.5x, click latency
bounded at ~300ms while training runs - between the fully-reactive
120 and the documented "sticky drag" 480.
DEPLOYMENT COUPLING: the new .ex5 #imports the new exports, so it will
NOT LOAD against the old WarriorCPU.dll ("cannot find function"). Copy
DirectML\WarriorCPU.dll into MQL5\Libraries (terminal closed) in the
same step as deploying the new .ex5.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1503 lines
77 KiB
MQL5
1503 lines
77 KiB
MQL5
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include <Arrays\ArrayDouble.mqh>
|
|
#include <Arrays\ArrayInt.mqh>
|
|
#include <Arrays\ArrayObj.mqh>
|
|
#include <OpenCL\OpenCL.mqh>
|
|
#include "..\System\AtomicFile.mqh"
|
|
//--- 2nd-tier CPU fallback (used when OpenCL is unavailable, e.g. a VM with no GPU passthrough,
|
|
//--- or this machine's OpenCL init failed).
|
|
#define CPU_THREADS_PER_NETWORK 2
|
|
//+------------------------------------------------------------------+
|
|
//| The percentage CComputeDll needs in order to land on |
|
|
//| CPU_THREADS_PER_NETWORK workers, given the detected core count. |
|
|
//| (WarriorCPU.dll takes a percentage of cores, not a thread count.) |
|
|
//+------------------------------------------------------------------+
|
|
int EffectiveCpuLoadPercent()
|
|
{
|
|
int cores = (int)TerminalInfoInteger(TERMINAL_CPU_CORES);
|
|
//--- Unknown core count: assume a small machine rather than a large one. Guessing high here would
|
|
//--- reintroduce exactly the oversubscription this function exists to prevent.
|
|
if(cores <= 0)
|
|
cores = 4;
|
|
//--- Fewer cores than we would ask for: take the machine as it is. 100 also means "all cores" to the
|
|
//--- DLL, so this is the one value that needs no arithmetic.
|
|
if(cores <= CPU_THREADS_PER_NETWORK)
|
|
return 100;
|
|
//--- Round UP: the DLL truncates when turning this back into a thread count, and landing one thread
|
|
//--- short of the target is a worse error than landing one over.
|
|
int pct = (int)MathCeil(100.0 * (double)CPU_THREADS_PER_NETWORK / (double)cores);
|
|
return (int)MathMax(1, MathMin(100, pct));
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
//--- Adam (Kingma & Ba, 2014) hyperparameters. 3.0e-4 also happens to sit inside the noisy/non-
|
|
//--- stationary-trading-data range (0.0003-0.0005) this project had separately tuned lr to before
|
|
//--- this input existed, so no behavior conflict.
|
|
input double AdamLearningRate = 0.0003; // Adam learning rate
|
|
input double AdamBeta1 = 0.9; // Adam beta1
|
|
input double AdamBeta2 = 0.999; // Adam beta2
|
|
//--- SGD+momentum hyperparameters.
|
|
input double SgdLearningRate = 0.0003; // SGD learning rate
|
|
input double SgdMomentum = 0.9; // SGD momentum
|
|
//--- Live learning rate: starts at the Adam input, then decayed and restored by the training loop,
|
|
//--- so it is a mutable global rather than a constant.
|
|
//--- g_-PREFIXED, and it has to be: as a bare `eta` this collided with a local of the same name in
|
|
//--- the standard library's Math\Stat\Math.mqh (the incomplete-gamma branch), which the compiler
|
|
//--- reports as "declaration of 'eta' hides global variable". Same fault as the b1/b2/lr/momentum
|
|
//--- macros retired in ea2552e - a single-token global name in a header that library code is
|
|
//--- compiled beside. The ETA_DECAY_FACTOR/ETA_MIN/m_etaCeiling vocabulary around it is unchanged,
|
|
//--- so the symbol still reads as the learning rate everywhere it appears.
|
|
double g_eta = AdamLearningRate;
|
|
#define defConnect 0x7781
|
|
#define defArrayConnects 0x7782
|
|
#define defNeuronBase 0x7783
|
|
#define defNeuron 0x7784
|
|
#define defNeuronConv 0x7785
|
|
#define defNeuronPool 0x7786
|
|
#define defLayer 0x7787
|
|
#define defArrayLayer 0x7788
|
|
#define defNet 0x7789
|
|
#define defNeuronLSTM 0x7791
|
|
//--- Topology-descriptor form of the batch-normalization layer (CLayerDescription::type). Like
|
|
//--- defNeuronConv/defNeuronPool it has no scalar-CPU neuron class behind it - CNet's constructor maps
|
|
//--- it onto CNeuronBatchNormOCL, which is the only implementation. See AI\NeuronBatchNorm.mqh.
|
|
#define defNeuronBatchNorm 0x7792
|
|
//---
|
|
#define defBufferDouble 0x7882
|
|
#define defNeuronBaseOCL 0x7883
|
|
#define defNeuronLSTMOCL 0x7884
|
|
#define defNeuronConvOCL 0x7885
|
|
#define defNeuronPoolOCL 0x7886
|
|
#define defNeuronBatchNormOCL 0x7887
|
|
//---
|
|
#define def_k_FeedForward 0
|
|
#define def_k_ff_matrix_w 0
|
|
#define def_k_ff_matrix_i 1
|
|
#define def_k_ff_matrix_o 2
|
|
#define def_k_ff_inputs 3
|
|
#define def_k_ff_activation 4
|
|
//---
|
|
#define def_k_CaclOutputGradient 1
|
|
#define def_k_cog_matrix_t 0
|
|
#define def_k_cog_matrix_o 1
|
|
#define def_k_cog_matrix_ig 2
|
|
#define def_k_cog_activation 3
|
|
//---
|
|
#define def_k_CaclHiddenGradient 2
|
|
#define def_k_chg_matrix_w 0
|
|
#define def_k_chg_matrix_g 1
|
|
#define def_k_chg_matrix_o 2
|
|
#define def_k_chg_matrix_ig 3
|
|
#define def_k_chg_outputs 4
|
|
#define def_k_chg_activation 5
|
|
//---
|
|
#define def_k_UpdateWeightsMomentum 3
|
|
#define def_k_uwm_matrix_w 0
|
|
#define def_k_uwm_matrix_g 1
|
|
#define def_k_uwm_matrix_i 2
|
|
#define def_k_uwm_matrix_dw 3
|
|
#define def_k_uwm_inputs 4
|
|
#define def_k_uwm_learning_rates 5
|
|
#define def_k_uwm_momentum 6
|
|
#define def_k_uwm_optimizer 7
|
|
//---
|
|
#define def_k_UpdateWeightsAdam 4
|
|
#define def_k_uwa_matrix_w 0
|
|
#define def_k_uwa_matrix_g 1
|
|
#define def_k_uwa_matrix_i 2
|
|
#define def_k_uwa_matrix_m 3
|
|
#define def_k_uwa_matrix_v 4
|
|
#define def_k_uwa_inputs 5
|
|
#define def_k_uwa_l 6
|
|
#define def_k_uwa_b1 7
|
|
#define def_k_uwa_b2 8
|
|
//---
|
|
#define def_k_FeedForwardProof 15
|
|
#define def_k_ffp_matrix_i 0
|
|
#define def_k_ffp_matrix_o 1
|
|
#define def_k_ffp_inputs 2
|
|
#define def_k_ffp_window 3
|
|
#define def_k_ffp_step 4
|
|
//---
|
|
#define def_k_CalcInputGradientProof 16
|
|
#define def_k_cigp_matrix_i 0
|
|
#define def_k_cigp_matrix_g 1
|
|
#define def_k_cigp_matrix_o 2
|
|
#define def_k_cigp_matrix_ig 3
|
|
#define def_k_cigp_outputs 4
|
|
#define def_k_cigp_window 5
|
|
#define def_k_cigp_step 6
|
|
//---
|
|
#define def_k_FeedForwardConv 5
|
|
#define def_k_ffc_matrix_w 0
|
|
#define def_k_ffc_matrix_i 1
|
|
#define def_k_ffc_matrix_o 2
|
|
#define def_k_ffc_inputs 3
|
|
#define def_k_ffc_step 4
|
|
#define def_k_ffc_window_in 5
|
|
#define def_k_ffc_window_out 6
|
|
#define def_k_ffc_activation 7
|
|
//---
|
|
#define def_k_CalcHiddenGradientConv 6
|
|
#define def_k_chgc_matrix_w 0
|
|
#define def_k_chgc_matrix_g 1
|
|
#define def_k_chgc_matrix_o 2
|
|
#define def_k_chgc_matrix_ig 3
|
|
#define def_k_chgc_outputs 4
|
|
#define def_k_chgc_step 5
|
|
#define def_k_chgc_window_in 6
|
|
#define def_k_chgc_window_out 7
|
|
#define def_k_chgc_activation 8
|
|
//---
|
|
#define def_k_UpdateWeightsConvMomentum 7
|
|
#define def_k_uwcm_matrix_w 0
|
|
#define def_k_uwcm_matrix_g 1
|
|
#define def_k_uwcm_matrix_i 2
|
|
#define def_k_uwcm_matrix_dw 3
|
|
#define def_k_uwcm_inputs 4
|
|
#define def_k_uwcm_learning_rates 5
|
|
#define def_k_uwcm_momentum 6
|
|
#define def_k_uwcm_window_in 7
|
|
#define def_k_uwcm_window_out 8
|
|
#define def_k_uwcm_step 9
|
|
#define def_k_uwcm_optimizer 10
|
|
//---
|
|
#define def_k_UpdateWeightsConvAdam 8
|
|
#define def_k_uwca_matrix_w 0
|
|
#define def_k_uwca_matrix_g 1
|
|
#define def_k_uwca_matrix_i 2
|
|
#define def_k_uwca_matrix_m 3
|
|
#define def_k_uwca_matrix_v 4
|
|
#define def_k_uwca_inputs 5
|
|
#define def_k_uwca_l 6
|
|
#define def_k_uwca_b1 7
|
|
#define def_k_uwca_b2 8
|
|
#define def_k_uwca_window_in 9
|
|
#define def_k_uwca_window_out 10
|
|
#define def_k_uwca_step 11
|
|
//---
|
|
//--- LSTM (CNeuronLSTMOCL) - single-timestep-truncated BPTT (no gradient flows back into
|
|
//--- h_prev/c_prev from a prior step).
|
|
#define def_k_LSTM_Gates 9
|
|
#define def_k_lstmg_matrix_w 0
|
|
#define def_k_lstmg_hidden_prev 1
|
|
#define def_k_lstmg_inputs 2
|
|
#define def_k_lstmg_concatenated 3
|
|
#define def_k_lstmg_hidden_size 4
|
|
#define def_k_lstmg_input_size 5
|
|
//---
|
|
#define def_k_LSTM_State 10
|
|
#define def_k_lstms_concatenated 0
|
|
#define def_k_lstms_memory 1
|
|
#define def_k_lstms_hidden_prev 2
|
|
#define def_k_lstms_hidden_cache 3
|
|
#define def_k_lstms_output 4
|
|
#define def_k_lstms_hidden_size 5
|
|
//---
|
|
#define def_k_LSTM_GateGradient 11
|
|
#define def_k_lstmgg_gradient 0
|
|
#define def_k_lstmgg_memory 1
|
|
#define def_k_lstmgg_concatenated 2
|
|
#define def_k_lstmgg_concatenated_gradient 3
|
|
#define def_k_lstmgg_hidden_size 4
|
|
//---
|
|
#define def_k_LSTM_WeightsGradient 12
|
|
#define def_k_lstmwg_concatenated_gradient 0
|
|
#define def_k_lstmwg_hidden_cache 1
|
|
#define def_k_lstmwg_inputs 2
|
|
#define def_k_lstmwg_weights_gradient 3
|
|
#define def_k_lstmwg_hidden_size 4
|
|
#define def_k_lstmwg_input_size 5
|
|
//---
|
|
#define def_k_LSTM_InputsGradient 13
|
|
#define def_k_lstmig_concatenated_gradient 0
|
|
#define def_k_lstmig_matrix_w 1
|
|
#define def_k_lstmig_inputs_gradient 2
|
|
#define def_k_lstmig_hidden_size 3
|
|
#define def_k_lstmig_input_size 4
|
|
//---
|
|
#define def_k_LSTM_UpdateWeightsAdam 14
|
|
#define def_k_lstmuwa_matrix_w 0
|
|
#define def_k_lstmuwa_weights_gradient 1
|
|
#define def_k_lstmuwa_matrix_m 2
|
|
#define def_k_lstmuwa_matrix_v 3
|
|
#define def_k_lstmuwa_l 4
|
|
#define def_k_lstmuwa_b1 5
|
|
#define def_k_lstmuwa_b2 6
|
|
//---
|
|
// SGD+momentum counterpart to LSTM_UpdateWeightsAdam above - see
|
|
// AI\Network.cl's LSTM_UpdateWeightsMomentum for the kernel body.
|
|
#define def_k_LSTM_UpdateWeightsMomentum 17
|
|
#define def_k_lstmuwm_matrix_w 0
|
|
#define def_k_lstmuwm_weights_gradient 1
|
|
#define def_k_lstmuwm_matrix_dw 2
|
|
#define def_k_lstmuwm_learning_rates 3
|
|
#define def_k_lstmuwm_momentum 4
|
|
#define def_k_lstmuwm_optimizer 5
|
|
//---
|
|
// Sequence LSTM. One launch PER TIMESTEP - the recurrence is sequential and OpenCL
|
|
// barriers only span a work-group, so the host loop is what provides the global
|
|
// ordering. See the block comment above LSTM_SeqStepForward in AI\Network.cl.
|
|
#define def_k_LSTM_SeqStepForward 18
|
|
#define def_k_lsf_matrix_w 0
|
|
#define def_k_lsf_inputs 1
|
|
#define def_k_lsf_cache_gates 2
|
|
#define def_k_lsf_cache_cell 3
|
|
#define def_k_lsf_cache_hidden 4
|
|
#define def_k_lsf_output 5
|
|
#define def_k_lsf_hidden_size 6
|
|
#define def_k_lsf_step_inputs 7
|
|
#define def_k_lsf_steps 8
|
|
#define def_k_lsf_t 9
|
|
//---
|
|
#define def_k_LSTM_SeqStepGateGrad 19
|
|
#define def_k_lsgg_out_gradient 0
|
|
#define def_k_lsgg_dh_buf 1
|
|
#define def_k_lsgg_dc_buf 2
|
|
#define def_k_lsgg_cache_gates 3
|
|
#define def_k_lsgg_cache_cell 4
|
|
#define def_k_lsgg_gate_grad 5
|
|
#define def_k_lsgg_hidden_size 6
|
|
#define def_k_lsgg_steps 7
|
|
#define def_k_lsgg_t 8
|
|
//---
|
|
#define def_k_LSTM_SeqStepWeightGrad 20
|
|
#define def_k_lswg_gate_grad 0
|
|
#define def_k_lswg_cache_hidden 1
|
|
#define def_k_lswg_inputs 2
|
|
#define def_k_lswg_weights_gradient 3
|
|
#define def_k_lswg_hidden_size 4
|
|
#define def_k_lswg_step_inputs 5
|
|
#define def_k_lswg_t 6
|
|
//---
|
|
#define def_k_LSTM_SeqStepInputGrad 21
|
|
#define def_k_lsig_gate_grad 0
|
|
#define def_k_lsig_matrix_w 1
|
|
#define def_k_lsig_inputs_gradient 2
|
|
#define def_k_lsig_dh_buf 3
|
|
#define def_k_lsig_hidden_size 4
|
|
#define def_k_lsig_step_inputs 5
|
|
#define def_k_lsig_t 6
|
|
//---
|
|
//--- Mini-batch gradient accumulation - see the block comment above AccumulateWeightGrad in
|
|
//--- AI\Network.cl.
|
|
#define def_k_AccumulateWeightGrad 22
|
|
#define def_k_awg_matrix_acc 0
|
|
#define def_k_awg_matrix_g 1
|
|
#define def_k_awg_matrix_i 2
|
|
#define def_k_awg_inputs 3
|
|
//---
|
|
#define def_k_AccumulateWeightGradConv 23
|
|
#define def_k_awgc_matrix_acc 0
|
|
#define def_k_awgc_matrix_g 1
|
|
#define def_k_awgc_matrix_i 2
|
|
#define def_k_awgc_inputs 3
|
|
#define def_k_awgc_window_in 4
|
|
#define def_k_awgc_window_out 5
|
|
#define def_k_awgc_step 6
|
|
//---
|
|
#define def_k_AccumulateBufferInto 24
|
|
#define def_k_abi_dst 0
|
|
#define def_k_abi_src 1
|
|
//---
|
|
//--- Mini-batch APPLY.
|
|
#define def_k_ApplyAccumAdam 25
|
|
#define def_k_aaa_matrix_w 0
|
|
#define def_k_aaa_matrix_acc 1
|
|
#define def_k_aaa_matrix_m 2
|
|
#define def_k_aaa_matrix_v 3
|
|
#define def_k_aaa_scale 4
|
|
#define def_k_aaa_l 5
|
|
#define def_k_aaa_b1 6
|
|
#define def_k_aaa_b2 7
|
|
//---
|
|
#define def_k_ApplyAccumMomentum 26
|
|
#define def_k_aam_matrix_w 0
|
|
#define def_k_aam_matrix_acc 1
|
|
#define def_k_aam_matrix_dw 2
|
|
#define def_k_aam_scale 3
|
|
#define def_k_aam_lr 4
|
|
#define def_k_aam_momentum 5
|
|
//--- One latch for the whole process, not per net: the apply kernels either built on this machine's
|
|
//--- OpenCL device or they did not, and that answer cannot change mid-run.
|
|
bool g_applyAccumKernelUsable = true;
|
|
//--- Same latch for the CPU DLL's apply exports (CPU_ApplyAccumAdam/Momentum, 2026-08-25): a
|
|
//--- dispatch that fails once (bad handle geometry, DLL fault) will fail every batch, and the host
|
|
//--- step below it is correct - so one warning, then full speed on the fallback.
|
|
bool g_applyAccumDllUsable = true;
|
|
//---
|
|
//--- Batch-norm kernels (2026-08-09) - see the BATCH NORM block in AI\Network.cl.
|
|
#define def_k_BatchNormForward 27
|
|
#define def_k_bnf_matrix_i 0
|
|
#define def_k_bnf_matrix_o 1
|
|
#define def_k_bnf_options 2
|
|
#define def_k_bnf_w 3
|
|
#define def_k_bnf_frozen 4
|
|
//---
|
|
#define def_k_BatchNormHiddenGrad 28
|
|
#define def_k_bnh_matrix_g 0
|
|
#define def_k_bnh_prev_o 1
|
|
#define def_k_bnh_prev_g 2
|
|
#define def_k_bnh_options 3
|
|
#define def_k_bnh_activation 4
|
|
//---
|
|
#define def_k_BatchNormAccumGammaBeta 29
|
|
#define def_k_bna_matrix_g 0
|
|
#define def_k_bna_options 1
|
|
#define def_k_bna_acc 2
|
|
//---
|
|
#define def_k_BatchNormApplyGammaBeta 30
|
|
#define def_k_bnp_options 0
|
|
#define def_k_bnp_acc 1
|
|
#define def_k_bnp_scale 2
|
|
#define def_k_bnp_lt 3
|
|
#define def_k_bnp_b1 4
|
|
#define def_k_bnp_b2 5
|
|
#define def_k_bnp_lr 6
|
|
#define def_k_bnp_momentum 7
|
|
#define def_k_bnp_optimizer 8
|
|
bool g_bnKernelUsable = true;
|
|
//---
|
|
// The Adam betas are ordinary inputs (book defaults 0.9/0.999) - see
|
|
// AdamLearningRate's declaration comment for why the earlier 0.8 experiment (fighting a multi-era
|
|
// same-class-streak bug via a shorter momentum window) was reverted: that symptom's likely root
|
|
// cause was the independent-sigmoid+BCE output gradient, since replaced with a joint softmax+CCE
|
|
// gradient in backProp()/backPropOCL(), which addresses it more directly than shortening beta1 ever
|
|
// could. Both are passed as runtime parameters into every backend (not baked into compiled
|
|
// kernels - see DirectML\WarriorCPU.cpp/AI\Network.cl's UpdateWeightsAdam
|
|
// signatures), so they're safe to expose as ordinary inputs.
|
|
// Tightened from 1.0e6 - that ceiling was so loose it never actually engaged before training had
|
|
// already gone unstable (real collapses were happening at weight magnitudes several orders of
|
|
// magnitude below it). 100.0 matches the equivalent clamp in Dmitriy Gizlyk's reference NeuroNet.mqh
|
|
// engine (references\MQL5\Experts\NeuroNet_DNG\NeuroNet.mqh) and gives a hard ceiling that's actually
|
|
// reachable-and-meaningful given MAX_WEIGHT_DELTA=0.1 per step below.
|
|
#define MAX_WEIGHT 100.0
|
|
// Decoupled (AdamW-style) weight decay applied inside every Adam weight update below and in
|
|
// DirectML\WarriorCPU.cpp/AI\Network.cl (all three backends kept in sync) - see
|
|
// WarriorCPU.cpp's WEIGHT_DECAY comment for the full rationale: MAX_WEIGHT only stops outright
|
|
// +-Infinity blowups, it does nothing to stop weights slowly, unboundedly growing over hundreds of
|
|
// training eras on a fixed, heavily class-balance-oversampled dataset, which was producing multi-
|
|
// hour climb-to-90%+-then-collapse-to-single-digits OOS accuracy cycles.
|
|
// 0.001, NOT the 0.01 Loshchilov & Hutter default: decay here is applied per SAMPLE (online updates,
|
|
// ~20k+ steps per era), and AdamW's data term is invariant to gradient scale, so a weight's
|
|
// sustainable magnitude is roughly (its gradient stream's signal-to-noise ratio)/WEIGHT_DECAY. For a
|
|
// weak-signal domain like this one, 0.01 was observed (2026-07-19, SP500 H4) to grind the
|
|
// discriminative weights down until the per-bar logit spread (avg 0.19 at era 1) fell BELOW the
|
|
// calibration-capped class-prior offsets (~0.008): recall stayed healthy for ~30 eras while the
|
|
// spread decayed monotonically, then argmax degenerated to constant-Neutral once the evidence tilt
|
|
// dropped under the prior tilt. The prior offsets are capped by calibration regardless of decay
|
|
// strength; the evidence tilts scale with 1/WEIGHT_DECAY - so decay strength decides which one wins
|
|
// argmax. 0.001 lifts the evidence ceiling 10x while still bounding long-run weight growth.
|
|
#define WEIGHT_DECAY 0.001
|
|
// Per-step update clip - see WarriorCPU.cpp's matching MAX_WEIGHT_DELTA comment for the full
|
|
// rationale: weight decay alone didn't stop the collapse cycles, since they turned out to be sudden
|
|
// Adam overshoot events (OOS accuracy falling below the 3-class random-guess floor within ~20 eras),
|
|
// most likely from 5x back-to-back oversampling replay building artificially correlated momentum.
|
|
// Applied to the raw delta BEFORE it's added to the weight, unlike MAX_WEIGHT which only clamps the
|
|
// post-update weight value and is far too loose (1e6) to prevent this.
|
|
#define MAX_WEIGHT_DELTA 0.1
|
|
//--- Floor on |activationFunctionDerivative()| for saturated tanh/sigmoid units (see
|
|
//--- SigmoidFunctionDerivative/TanhFunctionDerivative below) - without this, a neuron pinned near
|
|
//--- its activation extremes (output near -1/0/1) produces a near-zero derivative, which zeroes
|
|
//--- that neuron's entire backprop gradient contribution regardless of how wrong its output is.
|
|
#define MIN_ACTIVATION_DERIVATIVE 1.0e-3
|
|
//--- Logit temperature for the 3-class softmax head (training gradient in backProp/backPropOCL AND
|
|
//--- read-time ApplyClassificationSoftmax - the two MUST stay in sync or the model is scored
|
|
//--- against a different distribution than it was trained on).
|
|
#define CLASS_LOGIT_SCALE 6.0
|
|
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#resource "Network.cl" as string cl_program
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
enum ENUM_ACTIVATION
|
|
{
|
|
NONE,
|
|
TANH,
|
|
SIGMOID,
|
|
PRELU // fixed param=0.01, matches CNeuronConv's CPU activationFunction
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Translates ENUM_ACTIVATION to the "activation" int code every |
|
|
//| native compute backend actually understands (Network.cl's |
|
|
//| kernels, and the mirrored Activation()/inline switches in |
|
|
//| WarriorCPU.cpp / WarriorDML.cpp): 0=TANH, 1=SIGMOID, 2=PRELU, |
|
|
//| and deliberately anything else (incl. |
|
|
//+------------------------------------------------------------------+
|
|
int NativeActivationCode(ENUM_ACTIVATION value)
|
|
{
|
|
switch(value)
|
|
{
|
|
case TANH: return 0;
|
|
case SIGMOID: return 1;
|
|
case PRELU: return 2;
|
|
default: return -1; // NONE (and anything unrecognized) - no kernel/DLL case matches
|
|
}
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| Human-readable ENUM_ACTIVATION, for diagnostics only. Used by the |
|
|
//| load-time architecture repair (CNet::EnforceOutputActivation) so |
|
|
//| the log names the stale value it found instead of printing a bare |
|
|
//| integer nobody can decode months later. |
|
|
//+------------------------------------------------------------------+
|
|
string ActivationName(ENUM_ACTIVATION value)
|
|
{
|
|
switch(value)
|
|
{
|
|
case NONE: return "NONE";
|
|
case TANH: return "TANH";
|
|
case SIGMOID: return "SIGMOID";
|
|
case PRELU: return "PRELU";
|
|
}
|
|
return "UNKNOWN(" + IntegerToString((int)value) + ")";
|
|
}
|
|
//--- Guarded so an identical copy can live in Enumerations\InputEnums.mqh too: that lets Variables\
|
|
//--- Inputs.mqh (which needs this type for the TrainingOptimizer input) be included FIRST - ahead
|
|
//--- of this AI header - without a duplicate-definition error.
|
|
#ifndef WARRIOR_ENUM_OPTIMIZATION_DEFINED
|
|
#define WARRIOR_ENUM_OPTIMIZATION_DEFINED
|
|
//--- 2026-07-28: a third DFA entry was removed. SGD/ADAM keep their ordinal values 0/1 -
|
|
//--- m_optimizationAlgo feeds the weights-filename fingerprint, so these must never be renumbered.
|
|
enum ENUM_OPTIMIZATION
|
|
{
|
|
SGD, // SGD + Momentum (heavy-ball, simpler, needs more eras)
|
|
ADAM // Adam (adaptive step, faster convergence, can overfit)
|
|
};
|
|
#endif
|
|
//---
|
|
enum ENUM_BUFFERS
|
|
{
|
|
WEIGHTS,
|
|
DELTA_WEIGHTS,
|
|
OUTPUT,
|
|
GRADIENT,
|
|
FIRST_MOMENTUM,
|
|
SECOND_MOMENTUM
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
//--- CConnection/CArrayCon - the per-synapse weight (+ Adam moment) storage and its owning array,
|
|
//--- used by the CPU-only CNeuronBase/CNeuron neuron family below as their fallback last-resort
|
|
//--- weight representation (no DLL import, no OpenCL). Bodies in AI\Impl\NeuronPrimitives.mqh.
|
|
class CConnection : public CObject
|
|
{
|
|
public:
|
|
double weight;
|
|
double deltaWeight;
|
|
double mt;
|
|
double vt;
|
|
|
|
CConnection(double w) { weight = w; deltaWeight = 0; mt = 0; vt = 0; }
|
|
~CConnection() {};
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defConnect; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CArrayCon : public CArrayObj
|
|
{
|
|
public:
|
|
CArrayCon(void) {};
|
|
~CArrayCon(void) {};
|
|
//--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below.
|
|
bool CreateElementScaled(int const index, double weighScale);
|
|
//--- MUST keep CArrayObj::CreateElement's EXACT signature so it really overrides the base virtual -
|
|
//--- CArrayObj::Load() dispatches through it when reading a saved CNeuronBase's connection array. In
|
|
//--- MQL5 an added parameter (even a defaulted one) turns this into a separate hiding method and
|
|
//--- leaves the base's `return(false)` stub in the vtable, which silently breaks every load. Same trap
|
|
//--- as CLayer::CreateElement - see the long note there.
|
|
virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); }
|
|
virtual void IncreaseTotal() { m_data_total++; }
|
|
virtual int Type(void) const { return defArrayConnects; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CLayer;
|
|
//---
|
|
class CNeuronBase : public CObject
|
|
{
|
|
protected:
|
|
double outputVal;
|
|
double prevVal;
|
|
uint m_myIndex;
|
|
double gradient;
|
|
CArrayCon *Connections;
|
|
ENUM_ACTIVATION activation;
|
|
ENUM_OPTIMIZATION optimization;
|
|
int t;
|
|
//---
|
|
virtual bool feedForward(CLayer *prevLayer) { return false; }
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer) { return false; }
|
|
virtual double activationFunction(double x);
|
|
virtual double SigmoidFunction(double x) { return MathPow(1 + exp(-x), -1); }
|
|
virtual double TanhFunction(double x) { return tanh(x); }
|
|
virtual CLayer *getOutputLayer(void) { return NULL; }
|
|
public:
|
|
CNeuronBase(void);
|
|
~CNeuronBase(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, ENUM_OPTIMIZATION optimization_type, double weighScale = -1.0);
|
|
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
|
|
//--- Mirrors CNeuronBaseOCL::Activation(). Needed so CNet::EnforceOutputActivation() can read back
|
|
//--- what a Load() restored from disk without caring which neuron model the net was built from.
|
|
virtual ENUM_ACTIVATION Activation(void) { return activation; }
|
|
//---
|
|
static double alpha;
|
|
//---
|
|
virtual void setOutputVal(double val) { prevVal = outputVal; outputVal = val; }
|
|
virtual double getOutputVal() { return outputVal; }
|
|
virtual double getPrevVal() { return prevVal; }
|
|
virtual void setGradient(double val) { gradient = val; }
|
|
virtual double getGradient() { return gradient; }
|
|
virtual CArrayCon *getConnections() { return Connections;}
|
|
virtual double activationFunctionDerivative(double x);
|
|
virtual double SigmoidFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, x * (1 - x)); }
|
|
virtual double TanhFunctionDerivative(double x) { return MathMax(MIN_ACTIVATION_DERIVATIVE, (1 + x) * (1 - x)); }
|
|
//---
|
|
virtual bool feedForward(CObject *&SourceObject);
|
|
virtual bool calcHiddenGradients(CObject *&TargetObject);
|
|
virtual bool updateInputWeights(CLayer *prevLayer) { return false; }
|
|
virtual bool updateInputWeights(CObject *SourceObject);
|
|
//---
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle)
|
|
{
|
|
activation = (ENUM_ACTIVATION)FileReadInteger(file_handle, INT_VALUE);
|
|
optimization = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
t = (ENUM_OPTIMIZATION)FileReadInteger(file_handle, INT_VALUE);
|
|
return(Connections.Load(file_handle));
|
|
}
|
|
//--- Forget the optimizer's trajectory memory (Adam m/v, SGD momentum, step counter) while
|
|
//--- leaving the weights untouched - see CNet::ResetOptimizerState for when and why.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
t = 1;
|
|
if(CheckPointer(Connections) == POINTER_INVALID)
|
|
return true;
|
|
for(int i = 0; i < Connections.Total(); i++)
|
|
{
|
|
CConnection *con = Connections.At(i);
|
|
if(CheckPointer(con) == POINTER_INVALID)
|
|
continue;
|
|
con.deltaWeight = 0.0;
|
|
con.mt = 0.0;
|
|
con.vt = 0.0;
|
|
}
|
|
return true;
|
|
}
|
|
//---
|
|
virtual int Type(void) const { return defNeuronBase; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
//--- CNeuron - the plain-CPU (no DLL/OpenCL) dense neuron, last-resort fallback tier. Bodies in
|
|
//--- AI\Impl\NeuronCPU.mqh.
|
|
class CNeuron : public CNeuronBase
|
|
{
|
|
private:
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual bool updateInputWeights(CLayer *prevLayer);
|
|
|
|
public:
|
|
CNeuron(void) {};
|
|
~CNeuron(void) { Connections.Shutdown(); }
|
|
//---
|
|
virtual bool calcOutputGradients(double targetVals);
|
|
virtual double sumDOW(CLayer *&nextLayer) ;
|
|
virtual int Type(void) const { return defNeuron; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class COpenCLMy : public COpenCL
|
|
{
|
|
public:
|
|
COpenCLMy(void) {};
|
|
~COpenCLMy(void) {};
|
|
template<typename T>
|
|
int AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags);
|
|
};
|
|
#include "ComputeDll.mqh"
|
|
class CLayer: public CArrayObj
|
|
{
|
|
private:
|
|
uint iOutputs;
|
|
int iFileHandle;
|
|
COpenCLMy *OpenCL;
|
|
CComputeDll *ComputeDll;
|
|
|
|
public:
|
|
CLayer(uint outputs = 0, int handle = INVALID_HANDLE, COpenCLMy *OpenCL = NULL, CComputeDll *ComputeDll = NULL);
|
|
~CLayer(void) {};
|
|
//--- Fan-in-scaled element factory. Deliberately NOT named CreateElement: see the override below.
|
|
bool CreateElementScaled(int const index, double weighScale);
|
|
//--- CRITICAL: this MUST keep CArrayObj::CreateElement's EXACT signature - `virtual bool
|
|
//--- CreateElement(const int index)` - because it is the real virtual override that
|
|
//--- CArrayObj::Load() dispatches through, and CArrayObj::Load() is how EVERY saved layer is
|
|
//--- read back (CLayer::Load -> CNet::Load).
|
|
virtual bool CreateElement(const int index) { return CreateElementScaled(index, -1.0); }
|
|
virtual void IncreaseTotal() { m_data_total++; }
|
|
virtual int Type(void) const { return defLayer; }
|
|
virtual bool Load(const int file_handle);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "ArrayLayer.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronPool : public CNeuronBase
|
|
{
|
|
protected:
|
|
CLayer *OutputLayer;
|
|
int iWindow;
|
|
int iStep;
|
|
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
|
|
public:
|
|
CNeuronPool(void) {};
|
|
~CNeuronPool(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
|
|
//---
|
|
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronPool; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronConv : public CNeuronPool
|
|
{
|
|
protected:
|
|
double param; //PReLU param
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual double activationFunction(double x);
|
|
virtual bool updateInputWeights(CLayer *prevLayer);
|
|
public:
|
|
CNeuronConv() : param(0.01) { };
|
|
~CNeuronConv(void) { };
|
|
//---
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
virtual double activationFunctionDerivative(double x);
|
|
virtual int Type(void) const { return defNeuronConv; }
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "LayerDescription.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNet
|
|
{
|
|
protected:
|
|
double dLogitAdjust[3];
|
|
bool bLogitAdjust;
|
|
void backPropOCL(CArrayDouble *targetVals, double sampleWeight = 1.0);
|
|
bool InitOpenCL(void);
|
|
bool InitComputeDll(void);
|
|
//--- Pure-MQL5 forward pass over OCL-format layers loaded host-only (no backend) - see SetCpuInference.
|
|
bool feedForwardCPU(CArrayDouble *inputVals);
|
|
public:
|
|
CNet(CArrayObj *Description);
|
|
~CNet(void);
|
|
bool feedForward(CArrayDouble *inputVals);
|
|
//--- sampleWeight scales this example's output-layer gradient before it propagates back through the
|
|
//--- hidden layers - see the matching declaration comment on ExpertSignalAIBase.mqh's oversampling
|
|
//--- replacement for why (inverse-class-frequency loss weighting instead of replaying the same
|
|
//--- example multiple times).
|
|
void backProp(CArrayDouble *targetVals, double sampleWeight = 1.0);
|
|
//--- LOGIT ADJUSTMENT (Menon et al. 2021, "Long-tail learning via logit adjustment"). Per-class
|
|
//--- additive offsets tau*log(prior_c) folded into the 3-class softmax during the BACKWARD pass
|
|
//--- only.
|
|
void SetLogitAdjustment(const double &offsets[]);
|
|
void ClearLogitAdjustment(void) { bLogitAdjust = false; }
|
|
void getResults(CArrayDouble *&resultVals) ;
|
|
double getRecentAverageError() { return recentAverageError; }
|
|
//--- indicatorParams: flattened AutoTuneIndicators "winning" AD indicator param values (see
|
|
//--- CExpertSignalAIBase::FlattenIndicatorParams/UnflattenIndicatorParams); pass an empty array
|
|
//--- when there is nothing to persist/restore.
|
|
bool Save(string file_name, double error, double undefine, double forecast, datetime time, bool common, long era, bool trainingComplete, const double &indicatorParams[]);
|
|
//--- `quiet` suppresses the on-reject diagnostic Prints for callers that EXPECT a miss and handle it
|
|
//--- gracefully (the EMA shadow-net bootstrap on a CPU-DLL box, which can't hold a 2nd full net - see
|
|
//--- EnsureShadowNet). The main-model load leaves it false so a real failure is still loud.
|
|
bool Load(string file_name, double &error, double &undefine, double &forecast, datetime &time, bool common, long &era, bool &trainingComplete, double &indicatorParams[], bool quiet=false);
|
|
//--- In-MEMORY weight checkpoint (host-only, zero extra device tensors). In-place weight copy
|
|
//--- uses only getWeights/setWeights, which the per-era shadow blend already exercises
|
|
//--- successfully on that backend.
|
|
string LayerLearningReport(void);
|
|
bool CaptureWeights(void);
|
|
bool RestoreWeights(void);
|
|
//--- MINI-BATCH CONTROL (2026-08-09 audit, F4). SetBatchSize() is how training asks for
|
|
//--- accumulation; 1 restores the exact per-sample path the engine used before.
|
|
void SetBatchSize(int size) { m_batchSizeRequested = (size > 1 ? size : 1); }
|
|
int BatchSize(void);
|
|
bool FlushBatch(void);
|
|
//--- Zero every neuron's optimizer state - Adam first/second moments, SGD momentum deltas, the
|
|
//--- per-neuron bias-correction step counters, and batch-norm's gamma/beta moment slots - while
|
|
//--- leaving weights, activations and batch-norm running STATISTICS untouched (those belong to
|
|
//--- the model, not the optimizer).
|
|
bool ResetOptimizerState(void);
|
|
//--- EMA shadow-weight deployment: blends this net's weights a small step (tau) toward another
|
|
//--- net's weights, layer by layer, neuron by neuron - this.weight = (1-tau)*this.weight +
|
|
//--- tau*live.weight.
|
|
bool BlendWeightsFrom(CNet &live, double tau);
|
|
//--- Cold-start fix: overwrites just the bias term (not the per-input weights, which stay randomly
|
|
//--- initialized and carry the real learning signal) of each output neuron's incoming weight block,
|
|
//--- on the layer immediately before the output layer - see ExpertSignalAIBase.mqh's call site
|
|
//--- (AdvanceLabelCachePrebuild()) for why: a freshly-initialized network's argmax is close to
|
|
//--- uniform noise across classes, so on a heavily imbalanced label distribution it fires far more
|
|
//--- non-majority classes than the true base rate warrants until backProp corrects it over many
|
|
//--- steps. biasValues.Size() must equal the output layer's neuron count. Only supports the
|
|
//--- OpenCL/CPU-DLL batched neuron model (CNeuronBaseOCL) this project actually runs on - returns
|
|
//--- false (no-op) rather than corrupt anything if that assumption doesn't hold.
|
|
bool SeedOutputLayerBias(const double &biasValues[]);
|
|
//--- Pure-MQL5 (no OpenCL/DLL) inference mode. Training/optimization never set this
|
|
//--- (they always want a backend), so their behaviour is unchanged.
|
|
void SetCpuInference(bool v) { m_cpuInference = v; }
|
|
bool CpuInference(void) const { return m_cpuInference; }
|
|
//--- Re-assert the output layer's activation after a Load(), and report what it used to be. WHY
|
|
//--- THIS EXISTS: a .nnw persists the ARCHITECTURE, not just the weights. Every reload restores
|
|
//--- whatever is on disk and the next Save() writes it back out - a wrong value can never heal
|
|
//--- on its own, while the source file reads as though it were already fixed.
|
|
bool EnforceOutputActivation(ENUM_ACTIVATION intended, ENUM_ACTIVATION &previous);
|
|
//--- Receptive field of the first conv layer as LOADED, for the stale-architecture check in
|
|
//--- CExpertSignalAIBase::EnforceTopologyContract. 0 when the net has no conv layer.
|
|
uint FirstConvWindow(void);
|
|
//--- Freeze/unfreeze every batch-normalization layer's running statistics
|
|
//--- (AI\NeuronBatchNorm.mqh). Frozen, a forward pass is a pure function of its input; unfrozen
|
|
//--- (the default) it also advances the statistics.
|
|
void SetBatchNormFrozen(bool frozen);
|
|
//--- Current freeze state (the first normalization layer's flag; they only ever move together).
|
|
//--- False on a net with no normalization layers.
|
|
bool GetBatchNormFrozen(void);
|
|
//---
|
|
static double recentAverageSmoothingFactor;
|
|
|
|
private:
|
|
CArrayLayer *layers;
|
|
COpenCLMy *opencl;
|
|
CComputeDll *computeDll;
|
|
double recentAverageError;
|
|
bool m_cpuInference;
|
|
//--- In-memory best-weights checkpoint (see CaptureWeights/RestoreWeights). One CArrayDouble per
|
|
//--- neuron in layer-major order; host-only, no device tensors. NULL/false until the first capture.
|
|
CArrayObj *m_weightSnapshot;
|
|
bool m_haveWeightSnapshot;
|
|
//--- Previous call's per-layer weight L2 norms, for LayerLearningReport(). Sized lazily to the layer
|
|
//--- count; -1 marks "no baseline yet" so the first report says (init) instead of a bogus 0% change.
|
|
double m_prevLayerNorm[];
|
|
//--- Previous call's per-layer weight VECTORS, for the |dW| term of LayerLearningReport(). One
|
|
//--- CArrayDouble per layer index (empty for layers that own no weights), host-only.
|
|
CArrayObj *m_prevLayerWeights;
|
|
//--- One-shot latch for BlendWeightsFrom's skip warning - it runs every era, and the condition it
|
|
//--- reports is permanent, so the second print would only be noise.
|
|
bool m_blendSkipLogged;
|
|
//--- PROCESS-WIDE compute-probe latches (shared by every CNet in this terminal process). A host
|
|
//--- that HAS OpenCL never latches, so every CNet still gets its own COpenCLMy. Failure messages
|
|
//--- stay loud every time.
|
|
static bool s_openclUnavailable;
|
|
static bool s_computeTierLogged;
|
|
//--- Mini-batch state. m_batchSizeRequested is what training asked for; m_batchCount is how many
|
|
//--- samples have been accumulated into the current batch. m_batchKernelsOk records whether this
|
|
//--- net's OpenCL device actually built the accumulation kernels - false forces per-sample updates
|
|
//--- rather than failing, so an old device trains exactly as it did before (see InitOpenCL).
|
|
//--- m_batchBegun guards BeginBatch so the first sample of a batch zeroes the accumulators exactly
|
|
//--- once. m_batchWarned latches the one-time "cannot batch on this tier" notice.
|
|
int m_batchSizeRequested;
|
|
int m_batchCount;
|
|
bool m_batchKernelsOk;
|
|
//--- Whether the DEVICE-SIDE apply kernels built (def_k_ApplyAccumAdam). Conflating them would
|
|
//--- turn a missing optimisation into a change of optimizer.
|
|
bool m_applyKernelsOk;
|
|
bool m_batchBegun;
|
|
bool m_batchWarned;
|
|
//--- Zero every accumulator, starting a fresh batch.
|
|
bool BeginBatch(void);
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronLSTM : public CNeuronPool
|
|
{
|
|
protected:
|
|
CLayer *ForgetGate;
|
|
CLayer *InputGate;
|
|
CLayer *OutputGate;
|
|
CLayer *NewContent;
|
|
CArrayDouble *Memory;
|
|
CArrayDouble *PrevMemory;
|
|
CArrayDouble *Input;
|
|
CArrayDouble *InputGradient;
|
|
//---
|
|
virtual bool feedForward(CLayer *prevLayer);
|
|
virtual bool calcHiddenGradients(CLayer *&nextLayer);
|
|
virtual bool updateInputWeights(CLayer *prevLayer);
|
|
virtual bool updateInputWeights(CLayer *gate, CArrayDouble *input_data);
|
|
virtual bool InitLayer(CLayer *layer, int numOutputs, int numUnits, ENUM_OPTIMIZATION optimization_type);
|
|
//--- Fills `result` rather than returning a fresh CArrayDouble. The old shape handed every caller
|
|
//--- an object to delete on each of its own error paths, and none of them did - see feedForward().
|
|
virtual bool CalculateGate(CLayer *gate, CArrayDouble *sequence, CArrayDouble &result);
|
|
bool AccumulateGateInputGradient(CLayer *gate, const int i, const int n, double &value);
|
|
|
|
public:
|
|
CNeuronLSTM(void);
|
|
~CNeuronLSTM(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, int window, int step, int units_count, ENUM_OPTIMIZATION optimization_type);
|
|
//---
|
|
virtual CLayer *getOutputLayer(void) { return OutputLayer; }
|
|
virtual bool calcInputGradients(CLayer *prevLayer) ;
|
|
virtual bool calcInputGradients(CNeuronBase *prevNeuron, uint index) ;
|
|
//--- methods for working with files
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronLSTM; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
template<typename T>
|
|
int COpenCLMy::AddBufferFromArray(T &data[], const uint data_array_offset, const uint data_array_count, const uint flags)
|
|
{
|
|
int result = -1;
|
|
for(int i = 0; i < m_buffers_total; i++)
|
|
{
|
|
if(m_buffers[i] != INVALID_HANDLE)
|
|
continue;
|
|
result = i;
|
|
break;
|
|
}
|
|
//---
|
|
if(result < 0)
|
|
{
|
|
if(ArrayResize(m_buffers, m_buffers_total + 1) > 0)
|
|
{
|
|
m_buffers_total = ArraySize(m_buffers);
|
|
result = m_buffers_total - 1;
|
|
m_buffers[result] = INVALID_HANDLE;
|
|
}
|
|
else
|
|
return result;
|
|
}
|
|
//---
|
|
if(!BufferFromArray(result, data, data_array_offset, data_array_count, flags))
|
|
return -1;
|
|
//---
|
|
return result;
|
|
}
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
#include "BufferDouble.mqh"
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronBaseOCL : public CObject
|
|
{
|
|
protected:
|
|
COpenCLMy *OpenCL;
|
|
CComputeDll *ComputeDll;
|
|
CBufferDouble *Output;
|
|
CBufferDouble *PrevOutput;
|
|
CBufferDouble *Weights;
|
|
CBufferDouble *DeltaWeights;
|
|
CBufferDouble *Gradient;
|
|
CBufferDouble *FirstMomentum;
|
|
CBufferDouble *SecondMomentum;
|
|
//--- MINI-BATCH ACCUMULATOR (2026-08-09 audit, F4). Same shape as Weights; holds the SUM of this
|
|
//--- weight block's per-sample gradients over the current batch.
|
|
CBufferDouble *GradAccum;
|
|
//---
|
|
const double alpha;
|
|
int t;
|
|
//---
|
|
ENUM_ACTIVATION activation;
|
|
ENUM_OPTIMIZATION optimization;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool calcHiddenGradients(CNeuronBaseOCL *NeuronOCL);
|
|
//--- Create a buffer's device-side storage on whichever backend is active, or leave it host-only
|
|
//--- (its CArrayDouble m_data already holds the values just Load()ed) when neither backend exists -
|
|
//--- the pure-MQL5 inference path (see CNet::SetCpuInference / feedForwardCPU).
|
|
bool BackendBufferCreate(CBufferDouble *buf)
|
|
{
|
|
if(CheckPointer(buf) == POINTER_INVALID)
|
|
return false;
|
|
if(CheckPointer(OpenCL) != POINTER_INVALID)
|
|
return buf.BufferCreate(OpenCL);
|
|
if(CheckPointer(ComputeDll) != POINTER_INVALID)
|
|
return buf.BufferCreate(ComputeDll);
|
|
return true; // no backend: keep host m_data, skip device allocation
|
|
}
|
|
//--- THE optimizer step, and the only copy of it in the batched path: one weight block, one Adam
|
|
//--- or SGD+momentum update on `acc * scale`, then the accumulator is zeroed.
|
|
bool ApplyAccumOnDevice(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale, int total);
|
|
bool ApplyAccumToBlock(CBufferDouble *w, CBufferDouble *acc, CBufferDouble *m,
|
|
CBufferDouble *v, CBufferDouble *dw, double scale);
|
|
//--- Lazily allocate the mini-batch accumulator to match `src` (Weights / WeightsConv /
|
|
//--- WeightsLSTM, whichever block the caller accumulates into).
|
|
bool EnsureGradAccumFor(CBufferDouble *&target, CBufferDouble *src)
|
|
{
|
|
if(CheckPointer(src) == POINTER_INVALID || src.Total() <= 0)
|
|
return false;
|
|
if(CheckPointer(target) != POINTER_INVALID && target.Total() == src.Total())
|
|
return true;
|
|
if(CheckPointer(target) != POINTER_INVALID)
|
|
delete target;
|
|
target = new CBufferDouble();
|
|
if(CheckPointer(target) == POINTER_INVALID)
|
|
return false;
|
|
if(!target.BufferInit(src.Total(), 0.0))
|
|
return false;
|
|
return BackendBufferCreate(target);
|
|
}
|
|
bool EnsureGradAccum(CBufferDouble *src) { return EnsureGradAccumFor(GradAccum, src); }
|
|
|
|
public:
|
|
CNeuronBaseOCL(void);
|
|
~CNeuronBaseOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual void SetActivationFunction(ENUM_ACTIVATION value) { activation = value; }
|
|
//---
|
|
virtual int getOutputIndex(void) { return Output.GetIndex(); }
|
|
virtual int getGradientIndex(void) { return Gradient.GetIndex(); }
|
|
virtual int getWeightsIndex(void) { return Weights.GetIndex(); }
|
|
virtual int getDeltaWeightsIndex(void) { return DeltaWeights.GetIndex(); }
|
|
virtual int getFirstMomentumIndex(void) { return FirstMomentum.GetIndex(); }
|
|
virtual int getSecondMomentumIndex(void) { return SecondMomentum.GetIndex();}
|
|
//---
|
|
virtual int getOutputVal(double &values[]) { return Output.GetData(values); }
|
|
virtual int getOutputVal(CArrayDouble *values) { return Output.GetData(values); }
|
|
virtual int getPrevVal(double &values[]) { return PrevOutput.GetData(values); }
|
|
virtual int getGradient(double &values[]) { return Gradient.GetData(values); }
|
|
//--- pushes locally-modified gradient values back to this buffer's GPU/CPU-DLL-side copy - used by
|
|
//--- CNet::backPropOCL() to apply per-sample loss weighting after the native CalcOutputGradient call
|
|
//--- (which only computes the raw, unweighted delta) and before the backward pass reads this same
|
|
//--- buffer to propagate into the hidden layers.
|
|
virtual bool setGradient(const double &values[])
|
|
{
|
|
int count = ArraySize(values);
|
|
for(int i = 0; i < count; i++)
|
|
if(!Gradient.Update(i, values[i]))
|
|
return false;
|
|
return Gradient.BufferWrite();
|
|
}
|
|
// Guarded: output-layer neurons (numOutputs==0) have Weights deleted in Init() but not re-created,
|
|
// so BlendWeightsFrom() walking every neuron would otherwise dereference a dead pointer here.
|
|
virtual int getWeights(double &values[]) { return (CheckPointer(Weights) == POINTER_INVALID ? 0 : Weights.GetData(values)); }
|
|
//--- Paired with getWeights() above for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment
|
|
//--- (see that method's declaration comment) - writes a full replacement weight array back to
|
|
//--- this buffer's device-side (OpenCL/CPU-DLL) storage.
|
|
virtual bool setWeights(double &values[])
|
|
{
|
|
if(CheckPointer(Weights) == POINTER_INVALID)
|
|
return false;
|
|
if(!Weights.AssignArray(values))
|
|
return false;
|
|
return Weights.BufferWrite();
|
|
}
|
|
//--- Forget the optimizer's trajectory memory: Adam first/second moments, SGD's previous-delta
|
|
//--- buffer, and the bias-correction step counter, weights untouched. See
|
|
//--- CNet::ResetOptimizerState for the restore/warm-restart rationale.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
t = 1;
|
|
bool ok = ZeroOptimizerBuffer(FirstMomentum);
|
|
ok = ZeroOptimizerBuffer(SecondMomentum) && ok;
|
|
ok = ZeroOptimizerBuffer(DeltaWeights) && ok;
|
|
return ok;
|
|
}
|
|
virtual int Neurons(void) { return Output.Total(); }
|
|
virtual ENUM_ACTIVATION Activation(void) { return activation; }
|
|
virtual int getConnections(void) { return (CheckPointer(Weights) != POINTER_INVALID && CheckPointer(Gradient) != POINTER_INVALID && Gradient.Total() > 0 ? Weights.Total() / Gradient.Total() : 0); }
|
|
//--- Host-side (no device round-trip) buffer element access for the pure-MQL5 inference path. Safe to
|
|
//--- call after Load() with no backend: the CArrayDouble m_data holds the values, unlike getWeights()/
|
|
//--- getOutputVal() which route through BufferRead() (a device read that fails without a backend).
|
|
double OutputHost(int i) { return (CheckPointer(Output) != POINTER_INVALID && i >= 0 && i < Output.Total()) ? Output.At(i) : 0.0; }
|
|
double WeightHost(int i) { return (CheckPointer(Weights) != POINTER_INVALID && i >= 0 && i < Weights.Total()) ? Weights.At(i) : 0.0; }
|
|
int WeightsCount(void) { return (CheckPointer(Weights) != POINTER_INVALID) ? Weights.Total() : 0; }
|
|
//--- Pure-MQL5, double-precision forward pass mirroring Network.cl's FeedForward kernel, reading only
|
|
//--- host buffers. Used exclusively when no compute backend exists (CNet::SetCpuInference). Dense
|
|
//--- (fully-connected) here; conv/pool/LSTM subclasses override with their own kernel math.
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL);
|
|
//--- Host-side input write for the CPU inference path's layer 0, and host-side output read for
|
|
//--- getResults() - both bypass the device buffer that the CPU path deliberately never allocates.
|
|
bool SetInputsCPU(CArrayDouble *inputVals);
|
|
int GetOutputsCPU(CArrayDouble *values);
|
|
//---
|
|
virtual bool feedForward(CObject *SourceObject);
|
|
virtual bool calcHiddenGradients(CObject *TargetObject);
|
|
virtual bool calcOutputGradients(CArrayDouble *Target);
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool updateInputWeights(CObject *SourceObject);
|
|
//--- MINI-BATCH PAIR (2026-08-09 audit, F4). accumulateInputWeightGrads() adds THIS sample's
|
|
//--- per-weight gradient into GradAccum without touching the weights;
|
|
//--- ApplyAccumulatedGradients() then takes ONE optimizer step on the batch mean and clears the
|
|
//--- accumulator.
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool accumulateInputWeightGrads(CObject *SourceObject);
|
|
virtual bool ApplyAccumulatedGradients(double scale);
|
|
//--- Zero the accumulator at the start of a batch. Separate from ApplyAccumulatedGradients so a
|
|
//--- discarded partial batch (a stopped run) can be cleared without taking a step from it.
|
|
virtual bool BeginGradAccum(void);
|
|
//---
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
//---
|
|
virtual int Type(void) const { return defNeuronBaseOCL; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//--- Accelerated convolution layer (OpenCL + CPU-DLL). Ported from the NeuroNet_DNG reference
|
|
//--- library's CNeuronConvOCL/CNeuronProofOCL, adapted to this project's double-precision
|
|
//--- CBufferDouble/COpenCLMy/CComputeDll conventions. A single (window+1)*window_out weight block is
|
|
//--- shared across every sliding position - unlike CNeuronBaseOCL, where every output has its own
|
|
//--- private weight vector. Bodies in AI\Impl\NeuronOCLConvPool.mqh.
|
|
class CNeuronConvOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
uint iWindow;
|
|
uint iStep;
|
|
uint iWindowOut;
|
|
CBufferDouble *WeightsConv;
|
|
CBufferDouble *DeltaWeightsConv;
|
|
CBufferDouble *FirstMomentumConv;
|
|
CBufferDouble *SecondMomentumConv;
|
|
//--- Mini-batch accumulator for the CONVOLUTION KERNEL block. Distinct from the base class's
|
|
//--- GradAccum, which shadows the base Weights - i.e. the outgoing dense matrix that the layer ABOVE
|
|
//--- accumulates into. A conv neuron owns both tensors, so it needs both accumulators; sharing one
|
|
//--- slot would have the two layers writing each other's gradients.
|
|
CBufferDouble *GradAccumConv;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of FeedForwardConv
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
|
|
public:
|
|
CNeuronConvOCL(void) : iWindow(1), iStep(1), iWindowOut(1)
|
|
{
|
|
WeightsConv = NULL;
|
|
DeltaWeightsConv = NULL;
|
|
FirstMomentumConv = NULL;
|
|
SecondMomentumConv = NULL;
|
|
GradAccumConv = NULL;
|
|
}
|
|
~CNeuronConvOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint window_in, uint step, uint window_out, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronConvOCL; }
|
|
// A .nnw persists the window it was BUILT with, so an older build's model keeps that receptive
|
|
// field forever on load - this lets EnforceTopologyContract() detect a stale one instead of
|
|
// training on silently. See CNet::FirstConvWindow.
|
|
uint Window(void) const { return iWindow; }
|
|
// See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsConv instead of the
|
|
// base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
|
|
virtual int getWeightsConv(double &values[]) { return (CheckPointer(WeightsConv) == POINTER_INVALID ? 0 : WeightsConv.GetData(values)); }
|
|
virtual bool setWeightsConv(double &values[])
|
|
{
|
|
if(CheckPointer(WeightsConv) == POINTER_INVALID)
|
|
return false;
|
|
if(!WeightsConv.AssignArray(values))
|
|
return false;
|
|
return WeightsConv.BufferWrite();
|
|
}
|
|
//--- The conv kernel block keeps its own moment/momentum buffers beside the base class's - see
|
|
//--- CNet::ResetOptimizerState.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::ResetOptimizerState();
|
|
ok = ZeroOptimizerBuffer(FirstMomentumConv) && ok;
|
|
ok = ZeroOptimizerBuffer(SecondMomentumConv) && ok;
|
|
ok = ZeroOptimizerBuffer(DeltaWeightsConv) && ok;
|
|
return ok;
|
|
}
|
|
//--- Both accumulators - the base one for the outgoing dense matrix, this class's for the conv
|
|
//--- kernel. See GradAccumConv's declaration for why they cannot share a slot.
|
|
virtual bool BeginGradAccum(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::BeginGradAccum();
|
|
if(CheckPointer(GradAccumConv) != POINTER_INVALID && GradAccumConv.Total() > 0)
|
|
ok = ZeroOptimizerBuffer(GradAccumConv) && ok;
|
|
return ok;
|
|
}
|
|
virtual bool ApplyAccumulatedGradients(double scale)
|
|
{
|
|
//--- Both blocks step on the SAME batch, so t advances once here rather than once per block.
|
|
bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale);
|
|
ok = ApplyAccumToBlock(WeightsConv, GradAccumConv, FirstMomentumConv, SecondMomentumConv,
|
|
DeltaWeightsConv, scale) && ok;
|
|
if(optimization == ADAM)
|
|
t++;
|
|
return ok;
|
|
}
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Accelerated max-pooling layer (OpenCL + CPU-DLL). No weights,|
|
|
//| so no updateInputWeights work - just a sliding max. Ported from |
|
|
//| the NeuroNet_DNG reference's CNeuronProofOCL kernels. Bodies in |
|
|
//| AI\Impl\NeuronOCLConvPool.mqh. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronPoolOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
uint iWindow;
|
|
uint iStep;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of FeedForwardProof
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL) { return true; }
|
|
|
|
public:
|
|
CNeuronPoolOCL(void) : iWindow(2), iStep(1) {}
|
|
~CNeuronPoolOCL(void) {}
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint window, uint step, uint units_count, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronPoolOCL; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| CNeuronBatchNormOCL - batch normalization (Ioffe & Szegedy 2015). |
|
|
//| Bodies in AI\Impl\NeuronBatchNorm.mqh. |
|
|
//+------------------------------------------------------------------+
|
|
//--- Per-neuron slot layout inside BatchOptions.
|
|
#define BN_OPT_STRIDE 9
|
|
#define BN_OPT_MEAN 0 // running mean
|
|
#define BN_OPT_VAR 1 // running variance
|
|
#define BN_OPT_NX 2 // normalized input, cached from the forward pass for the backward pass
|
|
#define BN_OPT_GAMMA 3 // learned scale, init 1
|
|
#define BN_OPT_BETA 4 // learned shift, init 0
|
|
#define BN_OPT_MG 5 // gamma: Adam first momentum, or SGD previous delta
|
|
#define BN_OPT_MB 6 // beta: Adam first momentum, or SGD previous delta
|
|
#define BN_OPT_VG 7 // gamma: Adam second momentum (stored already square-rooted, as the
|
|
#define BN_OPT_VB 8 // beta: ... UpdateWeightsAdam kernels in this engine also do)
|
|
//--- Variance floor. Applied to the standard deviation (not the variance) so it reads as "no unit
|
|
//--- is amplified by more than 1e4", which is the property that actually matters.
|
|
#define BN_EPSILON 1.0e-10
|
|
#define BN_MIN_STD 1.0e-4
|
|
//--- HARD BOUND ON THE NORMALIZED VALUE. BN_MIN_STD alone caps the per-unit gain at 1/1e-4 = 1e4,
|
|
//--- and the comment above states that as if it were a safety property.
|
|
#define BN_MAX_NX 8.0
|
|
//--- Sanity ceiling on a single input value, applied before it can touch the running statistics
|
|
//--- below. See NormalizeHost() for the failure this exists to stop.
|
|
#define BN_MAX_INPUT 1.0e6
|
|
//+------------------------------------------------------------------+
|
|
//| |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronBatchNormOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
int iBatchSize; // EMA window length; <=1 disables normalization entirely
|
|
//--- When true the running statistics are USED but not UPDATED, i.e. classic batch-norm
|
|
//--- inference semantics. Off by default (see the note on adaptation in the class header).
|
|
bool bStatsFrozen;
|
|
//--- Number of samples seen. Used only to ramp the effective window up from 1 to iBatchSize over
|
|
//--- the first iBatchSize samples (standard EMA bias correction).
|
|
int iSamplesSeen;
|
|
//--- Persisted through Save/Load like any other parameter - gamma/beta are learned, and the
|
|
//--- running statistics ARE the layer's inference behaviour, so a model that loses them is not
|
|
//--- the model that was trained.
|
|
CBufferDouble *BatchOptions;
|
|
//--- Mini-batch running sums of dL/dgamma and dL/dbeta, one entry per unit. Host-only and NOT
|
|
//--- persisted: transient within a batch, and every save point flushes first. See
|
|
//--- accumulateInputWeightGrads for why they are not extra BatchOptions slots.
|
|
double m_accGamma[];
|
|
double m_accBeta[];
|
|
//--- PER-SAMPLE TRANSFER CACHES.
|
|
double m_fwdInputCache[];
|
|
bool m_fwdInputCached;
|
|
double m_gradCache[];
|
|
bool m_gradCached;
|
|
//--- DEVICE PATH (OpenCL only, 2026-08-09). m_bnAcc holds the mini-batch gamma/beta gradient sums
|
|
//--- on the device (2 floats per unit: gamma then beta), the kernel twin of m_accGamma/m_accBeta.
|
|
//--- m_bnDeviceAuthoritative says the DEVICE copy of BatchOptions is the truth (kernels have
|
|
//--- written it since the last host sync); the m_bnChecked* flags latch each kernel's one-time
|
|
//--- self-check against its host twin. The checks are the whole safety story for shipping kernels
|
|
//--- that could not be built on the dev machine: a transcription or dispatch-binding error is
|
|
//--- caught on its first use, the layer resyncs from the good copy, latches the kernels off
|
|
//--- process-wide, and training continues host-side - a warning and some speed, never a poisoned
|
|
//--- .nnw.
|
|
CBufferDouble *m_bnAcc;
|
|
bool m_bnDeviceAuthoritative;
|
|
bool m_bnCheckedFwd;
|
|
bool m_bnCheckedGrad;
|
|
bool m_bnCheckedAccum;
|
|
bool m_bnCheckedApply;
|
|
//--- eligibility + buffer management for the kernel path
|
|
bool BnDeviceEligible(void);
|
|
bool EnsureBnDeviceBuffers(void);
|
|
//--- read-only pull of the device statistics into the host mirror (checkpoints/saves mid-training);
|
|
//--- device stays authoritative
|
|
void SyncOptionsToHost(void);
|
|
//--- full handover to the host path: pull statistics, drain the device accumulator into
|
|
//--- m_accGamma/m_accBeta so a mid-batch handover loses nothing, clear the flag
|
|
void EnsureHostAuthoritative(void);
|
|
//--- one-way process-wide latch + this layer's handover, with the reason printed once
|
|
void LatchBnKernelsOff(const string reason);
|
|
//--- kernel dispatches (return false on any SetArgument/Execute failure, no logging - the caller
|
|
//--- decides between latching and falling back)
|
|
bool DispatchBnForward(CNeuronBaseOCL *NeuronOCL, double w);
|
|
bool DispatchBnHiddenGrad(CNeuronBaseOCL *NeuronOCL);
|
|
bool DispatchBnAccum(void);
|
|
bool DispatchBnApply(double scale, double lt);
|
|
//--- one-time kernel-vs-host comparisons; each returns the OPERATION's result (true = the work got
|
|
//--- done correctly, by whichever path survived), never "the kernel matched"
|
|
bool SelfCheckBnForward(CNeuronBaseOCL *NeuronOCL);
|
|
bool SelfCheckBnHiddenGrad(CNeuronBaseOCL *NeuronOCL);
|
|
bool SelfCheckBnAccum(void);
|
|
bool SelfCheckBnApply(double scale, double lt);
|
|
//--- normalized disagreement: |got-ref| / (1e-3 * max(1,|ref|)), <=1 passes. DBL_MAX when
|
|
//--- exactly one side is non-finite.
|
|
double BnDiffScore(double ref, double got);
|
|
//--- host twin of the backward elementwise math, factored out of calcInputGradients so the
|
|
//--- self-check compares against literally the same code the host path runs
|
|
void HiddenGradHost(const double &grad[], const double &prevOut[],
|
|
ENUM_ACTIVATION act, double &ig[], int n);
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
//--- one unit's gamma/beta step, shared by the per-sample and per-batch paths
|
|
bool StepGammaBeta(int shift, double gGamma, double gBeta, double lt);
|
|
|
|
public:
|
|
virtual bool BeginGradAccum(void);
|
|
virtual bool ApplyAccumulatedGradients(double scale);
|
|
protected:
|
|
//--- shared by feedForward/feedForwardCPU: the whole forward transform for one already-read input
|
|
//--- vector, writing straight into the host mirror of Output.
|
|
bool NormalizeHost(const double &inputs[], int count);
|
|
//--- ensures BatchOptions exists and is sized/seeded for `neurons` units
|
|
bool InitOptions(int neurons);
|
|
|
|
public:
|
|
CNeuronBatchNormOCL(void);
|
|
~CNeuronBatchNormOCL(void);
|
|
//---
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint numNeurons, uint batchSize, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
//--- see bStatsFrozen. Deliberately NOT persisted: it is a transient evaluation mode, not model state.
|
|
void SetStatsFrozen(bool v) { bStatsFrozen = v; }
|
|
//--- Read-back for save/restore bracketing (CNet::GetBatchNormFrozen): a display-only forward
|
|
//--- must put the flag back the way it FOUND it, not assume the trainer wanted it unfrozen -
|
|
//--- pass 3 and the online-learning probes hold it frozen across their whole scan.
|
|
bool StatsFrozen(void) const { return bStatsFrozen; }
|
|
//--- Checkpoint/blend support. Without this the plateau ladder's "restore best checkpoint" would
|
|
//--- put the dense weights back while leaving this layer's parameters at whatever the diverged
|
|
//--- era left behind - a silently mismatched pair.
|
|
virtual int getWeightsBN(double &values[]);
|
|
virtual bool setWeightsBN(double &values[]);
|
|
//--- How many of the trailing entries in getWeightsBN's array are BatchOptions rather than the
|
|
//--- outgoing dense matrix.
|
|
int BatchOptionsTotal(void) const
|
|
{
|
|
return (CheckPointer(BatchOptions) == POINTER_INVALID) ? 0 : BatchOptions.Total();
|
|
}
|
|
//--- Zero ONLY the gamma/beta moment slots (BN_OPT_MG/MB/VG/VB) plus the base class's buffers
|
|
//--- for the outgoing dense matrix. See CNet::ResetOptimizerState.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::ResetOptimizerState();
|
|
if(CheckPointer(BatchOptions) != POINTER_INVALID)
|
|
{
|
|
//--- Kernel-mode discipline: this zeroes SOME slots of a block whose truth may live on the
|
|
//--- device, so pull first (or the untouched slots would be written back stale), zero, push.
|
|
SyncOptionsToHost();
|
|
int totalSlots = BatchOptions.Total();
|
|
for(int shift = 0; shift + BN_OPT_VB < totalSlots; shift += BN_OPT_STRIDE)
|
|
{
|
|
ok = BatchOptions.Update(shift + BN_OPT_MG, 0.0) && ok;
|
|
ok = BatchOptions.Update(shift + BN_OPT_MB, 0.0) && ok;
|
|
ok = BatchOptions.Update(shift + BN_OPT_VG, 0.0) && ok;
|
|
ok = BatchOptions.Update(shift + BN_OPT_VB, 0.0) && ok;
|
|
}
|
|
if(BatchOptions.GetIndex() >= 0)
|
|
ok = BatchOptions.BufferWrite() && ok;
|
|
}
|
|
return ok;
|
|
}
|
|
//---
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
//---
|
|
virtual int Type(void) const { return defNeuronBatchNormOCL; }
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//--- Marks a .nnw LSTM record as the SEQUENCE format. Chosen so it cannot collide with any value the
|
|
//--- pre-sequence format could have written in that slot (an input width, i.e. -1 or a positive count).
|
|
#define LSTM_SEQ_SAVE_TAG (-424242)
|
|
//--- Initial bias of the FORGET gate (gate 0). Every other weight starts near zero; this one must
|
|
//--- not.
|
|
#define LSTM_FORGET_BIAS_INIT (1.0)
|
|
//+------------------------------------------------------------------+
|
|
//| GPU/CPU-DLL-accelerated LSTM layer (OpenCL + CPU-DLL). Derived |
|
|
//| from scratch from the standard LSTM equations - NOT ported from |
|
|
//| the NeuroNet_DNG reference (see the note above the LSTM kernels |
|
|
//| in Network.cl for why). Single-timestep-truncated BPTT: gradient |
|
|
//| does not flow back into h_prev/c_prev from an earlier step. |
|
|
//| Adam-only - Init fails for any other optimization type. |
|
|
//+------------------------------------------------------------------+
|
|
class CNeuronLSTMOCL : public CNeuronBaseOCL
|
|
{
|
|
protected:
|
|
int m_iInputs;
|
|
//--- Mini-batch running total of WeightsGradient across the samples of one batch - see the note on
|
|
//--- accumulateInputWeightGrads below for why WeightsGradient itself cannot serve as it.
|
|
CBufferDouble *GradAccumLSTM;
|
|
//--- Sequence shape. m_iStepInputs is the width of ONE timestep (the per-bar feature count
|
|
//--- reaching this layer), set from the layer descriptor by SetStepWidth() before the first
|
|
//--- feedForward; m_iSteps is then m_iInputs / m_iStepInputs.
|
|
int m_iStepInputs;
|
|
int m_iSteps;
|
|
//--- Per-timestep caches, required by backpropagation-through-time: the backward pass needs each
|
|
//--- step's gate activations, cell state and hidden state, which the single-buffer Concatenated/
|
|
//--- Memory/HiddenCache trio below cannot hold because every step overwrites the last.
|
|
CBufferDouble *CacheGates; // T * 4H, gate order [f,i,o,g]
|
|
CBufferDouble *CacheCell; // T * H, c_t
|
|
CBufferDouble *CacheHidden; // T * H, h_t
|
|
CBufferDouble *WeightsLSTM;
|
|
CBufferDouble *FirstMomentumLSTM;
|
|
CBufferDouble *SecondMomentumLSTM;
|
|
CBufferDouble *DeltaWeightsLSTM;
|
|
CBufferDouble *WeightsGradient;
|
|
CBufferDouble *Concatenated;
|
|
CBufferDouble *ConcatenatedGradient;
|
|
CBufferDouble *Memory;
|
|
CBufferDouble *HiddenCache;
|
|
//---
|
|
virtual bool feedForward(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool feedForwardCPU(CNeuronBaseOCL *NeuronOCL); // pure-MQL5 mirror of LSTM_Gates + LSTM_State
|
|
virtual bool updateInputWeights(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool SetInputs(int count);
|
|
bool AllocateSequenceCaches(void);
|
|
|
|
public:
|
|
CNeuronLSTMOCL(void) : m_iInputs(-1), m_iStepInputs(-1), m_iSteps(-1)
|
|
{
|
|
GradAccumLSTM = NULL;
|
|
CacheGates = NULL;
|
|
CacheCell = NULL;
|
|
CacheHidden = NULL;
|
|
WeightsLSTM = NULL;
|
|
FirstMomentumLSTM = NULL;
|
|
SecondMomentumLSTM = NULL;
|
|
DeltaWeightsLSTM = NULL;
|
|
WeightsGradient = NULL;
|
|
Concatenated = NULL;
|
|
ConcatenatedGradient = NULL;
|
|
Memory = NULL;
|
|
HiddenCache = NULL;
|
|
}
|
|
~CNeuronLSTMOCL(void);
|
|
virtual bool Init(uint numOutputs, uint myIndex, COpenCLMy *open_cl, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
virtual bool Init(uint numOutputs, uint myIndex, CComputeDll *compute_dll, uint numNeurons, ENUM_OPTIMIZATION optimization_type);
|
|
//--- Per-timestep input width, from CLayerDescription::window (see AddLstmStage). Must be called
|
|
//--- between Init() and the first feedForward; <= 0 keeps the legacy single-timestep behaviour.
|
|
//--- Not persisted from here - Save/Load carry it, so a loaded model does not depend on call order.
|
|
void SetStepWidth(int stepInputs) { m_iStepInputs = (stepInputs > 0 ? stepInputs : -1); }
|
|
bool IsSequenceMode(void) const { return (m_iStepInputs > 0 && m_iSteps > 1); }
|
|
virtual bool calcInputGradients(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool Save(int const file_handle);
|
|
virtual bool Load(int const file_handle);
|
|
virtual int Type(void) const { return defNeuronLSTMOCL; }
|
|
// See CNeuronBaseOCL::getWeights/setWeights - same pair, targeting WeightsLSTM instead of the
|
|
// base class's Weights, for CNet::BlendWeightsFrom()'s EMA shadow-weight deployment.
|
|
virtual int getWeightsLSTM(double &values[]) { return (CheckPointer(WeightsLSTM) == POINTER_INVALID ? 0 : WeightsLSTM.GetData(values)); }
|
|
//--- Build this layer's weight block to match `src`, for a net that was cloned from one whose
|
|
//--- LSTM had not yet run a forward pass.
|
|
virtual bool AdoptShapeFrom(CNeuronLSTMOCL &src)
|
|
{
|
|
if(src.m_iInputs <= 0 || Neurons() != src.Neurons())
|
|
return false;
|
|
m_iStepInputs = src.m_iStepInputs;
|
|
return SetInputs(src.m_iInputs);
|
|
}
|
|
virtual bool setWeightsLSTM(double &values[])
|
|
{
|
|
if(CheckPointer(WeightsLSTM) == POINTER_INVALID)
|
|
return false;
|
|
if(!WeightsLSTM.AssignArray(values))
|
|
return false;
|
|
return WeightsLSTM.BufferWrite();
|
|
}
|
|
//--- The LSTM's gate-weight block keeps its own moment/momentum buffers beside the base class's -
|
|
//--- see CNet::ResetOptimizerState.
|
|
virtual bool ResetOptimizerState(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::ResetOptimizerState();
|
|
ok = ZeroOptimizerBuffer(FirstMomentumLSTM) && ok;
|
|
ok = ZeroOptimizerBuffer(SecondMomentumLSTM) && ok;
|
|
ok = ZeroOptimizerBuffer(DeltaWeightsLSTM) && ok;
|
|
return ok;
|
|
}
|
|
//--- MINI-BATCH. Batching it is therefore just a running total. Hence a separate accumulator
|
|
//--- plus a generic elementwise add (AccumulateBufferInto), which also avoids changing the
|
|
//--- signature of an already-deployed export.
|
|
virtual bool accumulateInputWeightGrads(CNeuronBaseOCL *NeuronOCL);
|
|
virtual bool BeginGradAccum(void)
|
|
{
|
|
bool ok = CNeuronBaseOCL::BeginGradAccum();
|
|
if(CheckPointer(GradAccumLSTM) != POINTER_INVALID && GradAccumLSTM.Total() > 0)
|
|
ok = ZeroOptimizerBuffer(GradAccumLSTM) && ok;
|
|
return ok;
|
|
}
|
|
virtual bool ApplyAccumulatedGradients(double scale)
|
|
{
|
|
bool ok = ApplyAccumToBlock(Weights, GradAccum, FirstMomentum, SecondMomentum, DeltaWeights, scale);
|
|
ok = ApplyAccumToBlock(WeightsLSTM, GradAccumLSTM, FirstMomentumLSTM, SecondMomentumLSTM,
|
|
DeltaWeightsLSTM, scale) && ok;
|
|
if(optimization == ADAM)
|
|
t++;
|
|
return ok;
|
|
}
|
|
};
|
|
//+------------------------------------------------------------------+
|
|
//| Implementation bodies. |
|
|
//+------------------------------------------------------------------+
|
|
#include "Impl\NeuronPrimitives.mqh"
|
|
#include "Impl\NeuronBase.mqh"
|
|
#include "Impl\NeuronCPU.mqh"
|
|
#include "Impl\NeuronConvPool.mqh"
|
|
#include "Impl\NeuronLSTM.mqh"
|
|
#include "Impl\Layer.mqh"
|
|
#include "Impl\NetBuild.mqh"
|
|
#include "Impl\NetForward.mqh"
|
|
#include "Impl\NetPersistence.mqh"
|
|
#include "Impl\NetWeights.mqh"
|
|
#include "Impl\NeuronOCLBase.mqh"
|
|
#include "Impl\NeuronOCLConvPool.mqh"
|
|
#include "Impl\NeuronBatchNorm.mqh"
|
|
#include "Impl\NeuronOCLLSTM.mqh"
|
|
//+------------------------------------------------------------------+
|